diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..ce0de81 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,48 @@ +## What and why + + + +## Contract classification (VER-3) + + + +- [ ] **None** — this PR touches no contract. +- [ ] **Additive** — new optional field, new flag, new subcommand. Consumers that + ignore unknown fields are unaffected. +- [ ] **Deprecating** — something is scheduled for removal. Notice, migration note + and removal date below. +- [ ] **Breaking** — removed or renamed field, changed type, changed exit code. + Needs an ADR and a migration window. +- [ ] **Semantic** — same wire format, different meaning. Treated as breaking: + version bump, parity test, replay decision. + +ADR: + +Migration notes: + +## Testing + + + +- [ ] `make lint` passes +- [ ] `make test` passes +- [ ] `make test-race` passes + +## Checklist + +- [ ] Commit messages follow Conventional Commits +- [ ] `CHANGELOG.md` updated if this is user-visible +- [ ] Docs updated (README, `docs/`, `--help` text) +- [ ] No secrets, credentials or customer data in the diff diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 1b5e4f4..674404e 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -9,43 +9,55 @@ on: permissions: contents: read +# The merge gate from §9 of the OBSRVR Engineering Standard. Every job here +# blocks the merge: +# +# lint gofmt, go vet, staticcheck +# test build and the full suite +# race go test -race (TEST-4) +# vulncheck govulncheck (SEC-9) +# secrets gitleaks (SEC-6) +# proto buf lint and buf breaking (VER-5) +# smoke onboarding smoke tests jobs: - vet: - name: Vet + lint: + name: Lint runs-on: ubuntu-latest - timeout-minutes: 20 - + timeout-minutes: 15 steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5 with: go-version-file: go.mod cache: true - - name: Download dependencies - run: go mod download - - - name: Run go vet + - name: Check formatting run: | - PKGS=$(find . \ - \( -path './.git' -o -path './data' -o -path './bin' -o -path './internal/testfixtures/source' -o -path './internal/testfixtures/processor' -o -path './internal/testfixtures/sink' \) -prune \ - -o -name '*.go' -print | xargs -r -n1 dirname | sort -u | sed 's#^#./#' | sed 's#\.//#./#') - go vet $PKGS + unformatted=$(gofmt -l . | grep -v '^data/' || true) + if [ -n "$unformatted" ]; then + echo "These files need gofmt:" + echo "$unformatted" + exit 1 + fi + + - name: go vet + run: go vet ./... + + - name: staticcheck + uses: dominikh/staticcheck-action@v1 + with: + version: latest + install-go: false test: name: Build and test runs-on: ubuntu-latest timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5 with: go-version-file: go.mod cache: true @@ -53,29 +65,145 @@ jobs: - name: Download dependencies run: go mod download + - name: Verify go.mod is tidy + run: | + go mod tidy + git diff --exit-code go.mod go.sum || { + echo "go.mod/go.sum are not tidy; run 'go mod tidy' and commit the result" + exit 1 + } + - name: Verify build run: make build - name: Run test suite run: make test + race: + name: Race detector + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # TEST-4. The pipeline runner and control plane are heavily concurrent; + # this job found a real escaped-pointer race in GetComponentStatus the + # first time it was switched on. + - name: Run tests with the race detector + run: make test-race + + vulncheck: + name: Vulnerability scan + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # SEC-9: fail on reachable known vulnerabilities. Findings that genuinely + # do not affect flowctl are exempted one at a time, with a written + # justification, in .vex/flowctl.openvex.json (SEC-12). The script fails + # on anything not covered there, so the gate cannot rot into a no-op. + - name: govulncheck + run: ./scripts/govulncheck.sh + + secrets: + name: Secret scan + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + # gitleaks needs history to scan the commits a PR adds. + fetch-depth: 0 + + # SEC-6: secrets must never reach git. + # + # The gitleaks *binary* is open source; only the GitHub Action wrapper + # requires a paid licence for organization repos. Running the binary + # directly keeps the gate without the licence. + - name: Install gitleaks + env: + GITLEAKS_VERSION: 8.21.2 + run: | + curl -sSfL -o /tmp/gitleaks.tar.gz \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + tar -xzf /tmp/gitleaks.tar.gz -C /tmp gitleaks + sudo install -m 0755 /tmp/gitleaks /usr/local/bin/gitleaks + gitleaks version + + - name: Scan the working tree + run: gitleaks dir . --redact --verbose + + - name: Scan the commits this PR adds + if: github.event_name == 'pull_request' + run: | + gitleaks git . --redact --verbose \ + --log-opts="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}" + + proto: + name: Protobuf contracts + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: bufbuild/buf-setup-action@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: buf lint + run: buf lint + + # VER-5: field numbers and wire types must not change, and removed + # numbers must stay reserved. Compared against the last release rather + # than against main, because the released tag is what deployed + # components are actually speaking. + - name: buf breaking + run: | + latest=$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null || true) + if [ -z "$latest" ]; then + echo "No release tag found; skipping the breaking-change check." + exit 0 + fi + echo "Comparing protobuf against $latest" + buf breaking --against ".git#tag=$latest" + + # DATA-5: generated code is regenerated and checked, never hand-edited. + - name: Verify generated code is current + run: | + buf generate + git diff --exit-code -- proto/ || { + echo "Generated protobuf code is stale; run 'make generate' and commit the result." + echo "Plugin versions are pinned in buf.gen.yaml, so this diff means" + echo "the .proto files changed without the generated code being updated." + exit 1 + } + smoke: name: Onboarding smoke tests runs-on: ubuntu-latest timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 - - name: Set up Go - uses: actions/setup-go@v5 + - uses: actions/setup-go@v5 with: go-version-file: go.mod cache: true - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.x' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c908b97..59a9302 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,9 @@ on: permissions: contents: write + # SEC-10: lets cosign exchange a GitHub OIDC token for a Sigstore + # certificate, so releases are signed without a long-lived key. + id-token: write jobs: release: @@ -28,9 +31,34 @@ jobs: - name: Download dependencies run: go mod download + # Release gate (§9): the merge gate plus the extras below. These repeat + # the PR jobs because a tag can be pushed to a commit that never went + # through a PR. + - name: Lint + run: make lint + - name: Run tests run: make test + - name: Run tests with the race detector + run: make test-race + + - name: Vulnerability scan + run: ./scripts/govulncheck.sh + + # BUILD-3: a release that cannot be rebuilt byte-for-byte from its tag + # is not a release. + - name: Verify the build is reproducible + run: make verify-repro + + # SEC-10: SBOM generation and keyless signing, both driven by + # .goreleaser.yaml. + - name: Install syft + uses: anchore/sbom-action/download-syft@v0 + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + - name: Run GoReleaser uses: goreleaser/goreleaser-action@v6 with: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 2617f51..dcfe394 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -18,12 +18,18 @@ builds: goarch: - amd64 - arm64 + # BUILD-3: no absolute build-machine paths in the binary, and the only + # timestamp embedded is the commit's, so building a tag twice gives the + # same bytes. + flags: + - -trimpath + mod_timestamp: "{{ .CommitTimestamp }}" ldflags: - >- -s -w -X github.com/withobsrvr/flowctl/cmd/version.Version={{ .Version }} -X github.com/withobsrvr/flowctl/cmd/version.Commit={{ .Commit }} - -X github.com/withobsrvr/flowctl/cmd/version.BuildDate={{ .Date }} + -X github.com/withobsrvr/flowctl/cmd/version.BuildDate={{ .CommitDate }} archives: - id: default @@ -32,16 +38,59 @@ archives: files: - README.md - CHANGELOG.md + - LICENSE + - SECURITY.md checksum: name_template: "checksums.txt" +# SEC-10: every release ships an SBOM alongside the archives. +sboms: + - id: archive + artifacts: archive + +# SEC-10: keyless Sigstore signing of the checksum file, which transitively +# covers every artifact listed in it. Requires `id-token: write` in the release +# workflow so GitHub can mint the OIDC token cosign exchanges for a certificate. +signs: + - cmd: cosign + certificate: "${artifact}.pem" + args: + - sign-blob + - "--output-certificate=${certificate}" + - "--output-signature=${signature}" + - "${artifact}" + - "--yes" + artifacts: checksum + output: true + +release: + footer: | + ## Verifying this release + + Checksums: + + ``` + sha256sum --check checksums.txt + ``` + + Signature (keyless Sigstore, no key to distribute): + + ``` + cosign verify-blob checksums.txt \ + --certificate checksums.txt.pem \ + --signature checksums.txt.sig \ + --certificate-identity-regexp 'https://github\.com/withObsrvr/flowctl/\.github/workflows/.+' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com + ``` + changelog: use: git sort: asc filters: exclude: - - '^docs:' - - '^test:' - - '^chore:' - - '^ci:' + - "^docs:" + - "^test:" + - "^chore:" + - "^ci:" + - "^style:" diff --git a/.vex/flowctl.openvex.json b/.vex/flowctl.openvex.json new file mode 100644 index 0000000..88f1c8a --- /dev/null +++ b/.vex/flowctl.openvex.json @@ -0,0 +1,39 @@ +{ + "@context": "https://openvex.dev/ns/v0.2.0", + "@id": "https://github.com/withObsrvr/flowctl/.vex/flowctl.openvex.json", + "author": "Tillman Mosley III ", + "role": "Maintainer", + "timestamp": "2026-10-20T00:00:00Z", + "version": 1, + "tooling": "hand-authored; see scripts/govulncheck.sh", + "statements": [ + { + "vulnerability": { + "name": "GO-2026-4887", + "description": "Moby has AuthZ plugin bypass when provided oversized request bodies" + }, + "products": [ + { "@id": "pkg:golang/github.com/withObsrvr/flowctl" } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "The vulnerability is in the Moby daemon's HTTP API authorization path. flowctl imports github.com/docker/docker only as a client (api/types/container, client, pkg/stdcopy in pkg/runner/docker_cli_client.go) to talk to a Docker daemon the operator already runs. flowctl does not run a Moby daemon, does not serve the Docker API, and does not use AuthZ plugins. govulncheck reaches the module through package init functions only, not through any vulnerable symbol. Upstream reports no fixed version.", + "action_statement": "Re-evaluate when docker/docker publishes a fixed release, or if flowctl ever serves a Docker-compatible API.", + "action_statement_timestamp": "2026-10-20T00:00:00Z" + }, + { + "vulnerability": { + "name": "GO-2026-4883", + "description": "Moby has an off-by-one error in its plugin privilege validation" + }, + "products": [ + { "@id": "pkg:golang/github.com/withObsrvr/flowctl" } + ], + "status": "not_affected", + "justification": "vulnerable_code_not_in_execute_path", + "impact_statement": "The vulnerability is in the Moby daemon's plugin privilege validation. flowctl neither installs nor validates Docker plugins; it starts and stops containers through the client API. As above, govulncheck reaches the module only through package init. Upstream reports no fixed version.", + "action_statement": "Re-evaluate when docker/docker publishes a fixed release, or if flowctl ever manages Docker plugins.", + "action_statement_timestamp": "2026-10-20T00:00:00Z" + } + ] +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..dbbd6ef --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,141 @@ +# AGENTS.md + +Instructions for AI agents working in this repository. Humans: see +[CONTRIBUTING.md](CONTRIBUTING.md) — this file is a subset written for automated tools. + +## What this is + +flowctl is a control plane for Stellar blockchain data pipelines. It runs sources, +processors and sinks as separate processes or containers, wires them together over +gRPC, and tracks their health and run history. + +It is open source and must stay fully usable without any hosted OBSRVR service. + +## Setup + +```bash +nix develop # or: direnv allow +make test +``` + +## Commands + +| Task | Command | +|---|---| +| Build | `make build` → `./bin/flowctl` | +| Test (all) | `make test` | +| Test (one) | `go test -v ./internal/core -run TestPipelineDAG` | +| Test (race) | `go test -race ./...` | +| Format | `make fmt` | +| Lint | `make lint` | +| Regenerate protobuf | `make generate` | +| Multi-platform build | `make build-all` | + +Run a pipeline locally: + +```bash +./bin/flowctl validate examples/getting-started/pipeline.yaml +./bin/flowctl run examples/getting-started/pipeline.yaml +``` + +The sandbox needs Docker or nerdctl installed: + +```bash +./bin/flowctl sandbox start --pipeline examples/sandbox-pipeline.yaml --services examples/sandbox.yaml +./bin/flowctl sandbox status +./bin/flowctl sandbox logs +./bin/flowctl sandbox stop +``` + +## Architecture + +``` +Sources → Processors → Sinks + ↓ ↓ ↓ + └──────────┴─────────┘ + ↓ + Control Plane + (gRPC API) +``` + +Components register with the control plane at startup and heartbeat for health +monitoring. Two execution models: simple (linear) and DAG (complex topologies). + +| Path | Holds | +|---|---| +| `cmd/` | Cobra commands. Adapters only — no business logic. | +| `internal/core/` | Pipeline execution, DAG resolution | +| `internal/runner/` | Pipeline lifecycle, orchestration, supervision | +| `internal/api/` | Control plane gRPC service | +| `internal/config/` | Kubernetes-style YAML config, TLS config | +| `internal/translator/` | Pipeline YAML → docker-compose / local | +| `internal/storage/` | BoltDB and in-memory backends | +| `internal/cliout/` | Output rendering: table, json, ndjson | +| `internal/exitcode/` | Exit code vocabulary and error mapping | +| `internal/sanitize/` | Control-character stripping for untrusted strings | +| `proto/` | gRPC service and message definitions | +| `schemas/cue/schema.cue` | Pipeline config schema (authoritative) | + +## Rules that are easy to break by accident + +These cause real bugs or standard violations. Check them before proposing a change. + +1. **stdout is results only.** Logs, progress, hints and prompts go to stderr, so + `flowctl … | jq` always works. Use `internal/cliout` for results and + `internal/utils/logger` for everything else. Never add a bare `fmt.Println` that + prints status to stdout. + +2. **Every machine-readable output carries a schema identifier.** When you add a + command that emits JSON or NDJSON, give it a schema name (`flowctl..v1`) and + add a golden test. Changing an existing shape is a breaking change. + +3. **Exit codes are a contract.** Return errors wrapped by `internal/exitcode` — + 2 usage, 3 not found, 4 unauthorized, 5 conflict, 6 timeout, 7 dependency + unavailable, 8 data validation. Do not add a bare `os.Exit(1)`. + +4. **Anything from a registered component is untrusted.** Service names, metadata and + relayed errors arrive from processes flowctl did not start. A crafted service name + can otherwise inject terminal escape sequences. Rendering through `internal/cliout` + handles this for you — it strips control characters as it formats a cell. Only reach + for `sanitize.Message` or `sanitize.Line` when writing to a terminal outside cliout, + as `status` does for its grouped layout. Never sanitise what goes into JSON: + `encoding/json` escapes control characters already, and stripping them corrupts data + the consumer is entitled to receive intact. + +5. **Standalone ingestion never depends on the control plane.** A running pipeline must + keep processing and checkpointing when the control plane is unreachable. Losing the + control plane degrades observability, never delivery. Treat any change to + `internal/runner` or `pkg/console/heartbeat` that violates this as a bug. + +6. **The two orchestrators must behave identically.** Process and container + orchestration must produce the same lifecycle, run states and exit behaviour. A fix + to one usually needs the same fix in the other, and a test covering both. + +7. **Never default to pubnet.** The Stellar network is always explicit configuration. + +8. **Protobuf changes keep field numbers and wire types**, and `reserve` removed numbers + and names. `buf breaking` runs in CI. + +## Testing expectations + +- `go test -race` must pass; the runner is heavily concurrent. +- A bug fix needs a test that fails without the fix. +- Parsers of external input need fuzz tests (`internal/config`, `internal/translator`). +- Prefer real captured data. Invented data is for injecting failures only. + +## Contracts and decisions + +- Contracts flowctl owns are listed in [`docs/engineering-profile.md`](docs/engineering-profile.md), + along with the current conformance gaps. Read it before changing a public surface. +- Decisions that change a contract, the architecture or the security posture need an + ADR in `docs/adr/`. Copy `docs/adr/0000-template.md`. Do not edit an accepted ADR — + supersede it. +- Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/). + +## Things to leave alone unless asked + +- `proto/` package naming. Renaming `flowctl` → `flowctl.v1` is a wire-breaking change + that needs coordination with flowctl-sdk and every deployed component. It is tracked + as GAP-7 with an ADR pending. +- The control plane's lack of authentication. It is a known gap (GAP-12) with a design + decision outstanding, not an oversight to patch ad hoc. diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ce63a4..0f0c2d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,39 +2,173 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +flowctl is pre-1.0: per SemVer, a minor bump may break. Breaking changes are +called out below and, where they change a contract, carry an ADR in `docs/adr/`. -### Removed +## [Unreleased] -- **BREAKING:** Removed `flowctl apply` command - use `flowctl translate` instead -- **BREAKING:** Removed `flowctl new` command - use `flowctl init` instead -- **BREAKING:** Removed `flowctl context` command - feature was not implemented -- Removed redundant `flowctl help` command - use `--help` flag instead +### Added -### Why? +- **Machine-readable output on every command that produces records.** + `--output json` emits one JSON document and `--output ndjson` one object per + line, each carrying a schema identifier. + ([ADR 0001](docs/adr/0001-cli-output-and-exit-code-contract.md)) -These commands were either incomplete stubs (`apply`), redundant (`new`, `help`), -or premature features (`context`). Removing them reduces user confusion, -eliminates false promises, and reduces maintenance burden by ~500 lines of code. + | Command | Schema | + |---|---| + | `version` | `flowctl.version.v1` | + | `list` | `flowctl.service_list.v1` / `flowctl.service.v1` | + | `config show` | `flowctl.config_setting_list.v1` / `flowctl.config_setting.v1` | + | `status` | `flowctl.pipeline_status.v1` | + | `pipelines list` | `flowctl.pipeline_list.v1` / `flowctl.pipeline.v1` | + | `pipelines runs` / `active` / `run-info` / `stop` | `flowctl.pipeline_run_list.v1` / `flowctl.pipeline_run.v1` | + | `chunks list` / `show` | `flowctl.chunk_run_list.v1` / `flowctl.chunk_run.v1` | + | `processors list` / `find` / `show` | `flowctl.processor_list.v1` / `flowctl.processor.v1` | + | `sandbox status` | `flowctl.sandbox_container_list.v1` / `flowctl.sandbox_container.v1` | + | `validate` | `flowctl.validation_result.v1` | + | `translate` | `flowctl.translation.v1` | + | `init` | `flowctl.pipeline_init.v1` | -### Migration Guide + Machine output carries full values where the table condenses: every event + type rather than "first (+2)", raw counts, RFC 3339 timestamps, and status as + a stable lowercase string. The table layout is explicitly not a contract. -| Old Command | New Alternative | -|-------------|----------------| -| `flowctl apply -f file.yaml --translate-only` | `flowctl translate -f file.yaml` | -| `flowctl new pipeline foo` | `flowctl init` (interactive wizard) | -| `flowctl context --show` | Use `--context` flag on commands | -| `flowctl help ` | `flowctl --help` | + `dashboard` is an interactive terminal UI and rejects a machine format, + pointing at `flowctl status` instead. `run` and `server` are long-running and + produce no record; their progress now goes to stderr. +- **Exit codes.** Failures now exit with a code that says why: 2 usage, 3 not + found, 4 not permitted, 5 conflict, 6 timed out, 7 dependency unavailable, + 8 data validation. A clean shutdown on SIGINT or SIGTERM exits 130 or 143. +- `flowctl config show` prints every setting with the source it came from — + flag, environment variable, config file or default. +- `LICENSE` (MPL-2.0), `SECURITY.md`, `CONTRIBUTING.md` and `AGENTS.md`. +- `docs/engineering-profile.md`: the contracts this repo owns and its + conformance gaps against the OBSRVR Engineering Standard v2.0. +- `docs/adr/`: architecture decision records, with a MADR template. +- Merge gate in CI: `gofmt`, `go vet`, `staticcheck`, `go test -race`, + `govulncheck`, `gitleaks` secret scanning, `buf lint` and `buf breaking`. +- `make test-race`, `make vulncheck` and `make verify-repro`. ### Changed -- Fixed linting issues in `flowctl init` command (redundant newlines in fmt.Println) +- **BREAKING: `flowctl init --output ` is now `--file` / `-f`.** `-o` and + `--output` select an output *format* everywhere in flowctl, and reusing the + name for a filename is forbidden by CLI-3. No alias is kept: a local + `--output` would shadow the global one and silently write to a file named + `json`. +- **BREAKING: `flowctl processors find --output ` is now `--output-type`.** + Same reason, same absence of an alias. +- **BREAKING: `flowctl translate --output ` is now `--target` / `-t`.** + It selected a deployment target, not an encoding — the third instance of the + same collision. `flowctl translate -f p.yaml -t docker-compose`. +- **BREAKING: `flowctl list --table` / `-t` removed.** Table is the default; + use `--output json` for machine-readable output. +- **BREAKING: `--output yaml` and `--output wide` are now errors.** They were + advertised and never implemented; they used to be silently ignored. +- **BREAKING: failures no longer always exit 1.** A script testing `$? -eq 1` + should test `$? -ne 0` instead. +- **BREAKING: `apiVersion` is validated consistently.** The CUE schema required + `flowctl/v1` but the Go loaders accepted any non-empty string, so + `apiVersion: v99` was rejected by `flowctl validate` and silently run as v1 by + `flowctl run`. All three paths now share one validator and reject anything + that is not `flowctl/v1`, with an actionable error (VER-7). Pipelines using a + bare `v1` must change to `flowctl/v1`. +- Progress messages, hints and prompts now go to stderr, leaving stdout for + results so that `flowctl … | jq` always works. That includes the `flowctl run` + startup banner, the `--show-status` live monitor, and component download + progress, none of which were results. +- `flowctl init` refuses to start its wizard when stdin is not a terminal, + instead of reading EOF for every answer and silently building a pipeline from + defaults (CLI-8). Use `--preset`, or `--non-interactive` with `--network` and + `--destination`. In table mode it now prints just the created path on stdout, + so `flowctl run "$(flowctl init --preset testnet-duckdb)"` works. +- `flowctl status --watch` is a usage error when combined with a machine format, + since redrawing a terminal has no meaning for a consumer reading one document. +- `flowctl validate` always emits a result document in machine mode, including + when the file fails to load — a caller never has to handle two shapes. +- Colour is suppressed when output is not a terminal, when `NO_COLOR` is set, + or when `TERM=dumb`. +- Logs are JSON unless attached to a colour-capable terminal, carry `service` + and `version` on every line, and timestamp in RFC 3339 UTC. +- Environment variables are read with a `FLOWCTL_` prefix. Previously viper's + `AutomaticEnv` ran with no prefix, binding bare `OUTPUT`, `CONTEXT`, + `NAMESPACE` and `LOG_LEVEL` from the ambient environment. +- Configuration honours `$XDG_CONFIG_HOME`, and a project `flowctl.yaml` in the + working directory layers over the user file. +- Release builds are reproducible: `-trimpath`, and the only embedded timestamp + is the commit's. `make verify-repro` builds twice and compares hashes. +- Container images run as a non-root user and carry OCI source, revision and + version annotations. + +### Fixed + +- **Data race in the control plane.** `GetComponentStatus` and `ListComponents` + returned pointers into the service map that the health janitor mutates, so + gRPC serialised a struct while the janitor was writing to it. Both now return + a clone. Found by switching on `go test -race`. +- **Terminal escape injection.** Service names, metadata, event-type strings and + relayed error messages come from processes flowctl did not start and were + printed to the terminal unescaped, letting a crafted name emit its own OSC + sequences — set the window title, plant a misleading hyperlink, or write to + the clipboard. Every display path is now covered: commands rendering through + the new output layer sanitise each formatted cell, and the commands not yet + converted — `status`, `pipelines`, `chunks`, `processors` and `dashboard` — + sanitise each control-plane response on arrival. JSON output is deliberately + unaffected and still round-trips the original bytes, since `encoding/json` + escapes control characters anyway. +- `flowctl status` wrote raw ANSI colour and a screen-clear escape + unconditionally, so `flowctl status > file` embedded escape codes and + `flowctl status | grep RUNNING` could fail to match. Colour now follows the + same rule as everywhere else: off when stdout is not a terminal, when + `NO_COLOR` is set, or when `TERM=dumb`. +- **SIGINT during startup killed the process.** The signal handler was installed + only once the pipeline was running, so an interrupt during startup bypassed + graceful shutdown and orphaned child components. It is now installed before + anything starts. +- Errors from `src.Events` and `snk.Write` in the simple pipeline were silently + discarded behind a `// TODO: Handle error`. A dropped sink write is data loss; + both are now logged. +- A malformed config file is now a usage error instead of being ignored. +- `parseFileMode` had a comparison that could never be true (`byte < 0`). The + validation was still correct — the upper bound caught every bad input — but + the dead half has been removed. +- `flowctl pipelines` sliced `run.RunId[:8]` for its short-id column, which + panics on any run id shorter than eight bytes. The id arrives over the wire + from the control plane, which is a trust boundary; it now slices by rune count + and tolerates short input. +- `flowctl validate` exited via a bare `os.Exit(1)`, bypassing the exit-code + contract. It now exits 8 when validation fails and 3 when the file is missing. +- `flowctl translate` reported an incomplete pipeline — a missing image, say — + as a generic failure. It is a data validation failure and now exits 8. + +### Removed + +- Dead code: eleven unused helpers in `internal/translator/utils.go`, unused + test mocks, an unused `sinkChan` field, and a vestigial builder stage in the + duckdb-consumer Dockerfile that downloaded a Go toolchain and discarded it. --- -## Previous Releases +## [0.0.12] and earlier + +Released before this changelog was maintained. See the +[git history](https://github.com/withObsrvr/flowctl/commits/main) and the +[release list](https://github.com/withObsrvr/flowctl/releases). + +Notable breaking changes in that period: -_No previous releases documented_ +- Removed `flowctl apply` — use `flowctl translate`. +- Removed `flowctl new` — use `flowctl init`. +- Removed `flowctl context` — the feature was never implemented; use the + `--context` flag. +- Removed `flowctl help` — use `--help`. + +| Old command | New alternative | +|---|---| +| `flowctl apply -f file.yaml --translate-only` | `flowctl translate -f file.yaml` | +| `flowctl new pipeline foo` | `flowctl init` | +| `flowctl context --show` | `--context` flag on commands | +| `flowctl help ` | `flowctl --help` | diff --git a/CLAUDE.md b/CLAUDE.md index 1164232..ef81ab6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,129 +1,16 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Agent instructions for this repository live in **[AGENTS.md](AGENTS.md)**. -## Commands +Read that file. It is the single source of truth for build commands, architecture, +the rules that are easy to break by accident, and testing expectations. This file +exists only so that tools looking for `CLAUDE.md` find their way there, and is kept +deliberately thin so the two cannot drift apart. -### Build Commands -- `make build` - Build the binary with CGO disabled and install schema files -- `make build-all` - Build for multiple platforms (linux-amd64, linux-arm64, darwin-amd64) -- `make clean` - Clean build artifacts +See also: -### Test Commands -- `make test` - Run all tests with verbose output -- `go test -v ./...` - Run all tests directly -- `./test/test_registration.sh` - Test registration functionality - -### Development Commands -- `make deps` - Update Go module dependencies -- `make run-example` - Build and run the example pipeline -- `./scripts/generate.sh` - Generate Go code from proto files - -### Sandbox Commands -The sandbox provides a local containerized environment for development: - -**Prerequisites**: Docker or nerdctl must be installed on your system. - -```bash -# Start sandbox -./bin/flowctl sandbox start --pipeline examples/sandbox-pipeline.yaml --services examples/sandbox.yaml - -# Check status -./bin/flowctl sandbox status - -# View logs -./bin/flowctl sandbox logs - -# Stop sandbox -./bin/flowctl sandbox stop -``` - -**Note**: The `--use-system-runtime` flag is still supported but no longer required (it's now the default behavior). - -### Running a Single Test -```bash -go test -v ./internal/core -run TestPipelineDAG -``` - -## Architecture Overview - -### Command Structure -The application is a CLI tool built with Cobra framework: -- **Root**: `flowctl` - Base command with global flags -- **Core Commands**: `init`, `run`, `validate`, `translate` -- **Monitoring**: `dashboard`, `status`, `list`, `pipelines`, `processors` -- **Development**: `sandbox` -- **Utility**: `version` -- **Hidden**: `server` (standalone control plane mode) - -### Core Components - -**Pipeline Execution** (`internal/core/`) -- Two execution models: Simple (linear) and DAG (complex topologies) -- Components (sources, processors, sinks) run as separate containers/processes -- Communication via gRPC with typed events - -**Configuration** (`internal/config/`) -- Kubernetes-style YAML (apiVersion, kind, metadata, spec) -- CUE schema validation in `schemas/cue/schema.cue` -- TLS/mTLS configuration support - -**Translation System** (`internal/translator/`) -- Converts pipeline YAML to deployment formats -- Supports Docker Compose, Kubernetes (planned), Nomad (planned), Local execution -- Uses CUE for validation during translation - -**Control Plane** (`internal/api/`) -- gRPC-based service for component registration -- Health monitoring and service discovery -- Storage backends: BoltDB (embedded) or in-memory - -### Key Design Patterns - -1. **Plugin Architecture**: Sources, processors, and sinks are pluggable components implementing standard interfaces -2. **Event-Driven**: Components communicate via typed events with buffered channels for backpressure -3. **Interface-Based**: Heavy use of Go interfaces for extensibility -4. **Cloud-Native**: Designed for containerized deployment with multiple orchestrator support - -### Component Interaction Flow -``` -Sources → Processors → Sinks - ↓ ↓ ↓ - └──────────┴─────────┘ - ↓ - Control Plane - (gRPC API) -``` - -Components register with the control plane on startup and send periodic heartbeats for health monitoring. - -### Important Files -- Pipeline examples: `examples/getting-started/` -- Schema definition: `schemas/cue/schema.cue` -- Proto definitions: `proto/` directory -- Storage interface: `internal/storage/storage.go` - -## Available Commands - -### Pipeline Management -- `flowctl init` - Interactive wizard to create a new Stellar pipeline -- `flowctl validate ` - Validate pipeline configuration before running -- `flowctl run ` - Run a pipeline with embedded control plane -- `flowctl translate -f -o ` - Translate pipeline to docker-compose or local format - -### Monitoring & Observability -- `flowctl dashboard` - Interactive TUI dashboard for real-time monitoring -- `flowctl status [file]` - Show component health and status -- `flowctl pipelines` - Manage pipeline runs (list, active, runs, stop) -- `flowctl processors` - Discover and inspect processors (list, find, show) -- `flowctl list` - List all registered services - -### Development Environment -- `flowctl sandbox start` - Start local containerized dev environment -- `flowctl sandbox status` - Check sandbox status -- `flowctl sandbox logs` - View sandbox logs -- `flowctl sandbox stop` - Stop sandbox environment - -### Utility -- `flowctl version` - Print version information -- `flowctl server` - Run standalone control plane (hidden command) \ No newline at end of file +- [`docs/engineering-profile.md`](docs/engineering-profile.md) — the contracts this repo + owns and its current conformance gaps. Read before changing a public surface. +- [`CONTRIBUTING.md`](CONTRIBUTING.md) — the human-facing version, with commit + conventions and the contract-change classification. +- [`docs/adr/`](docs/adr/) — architecture decision records. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..dd024da --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,109 @@ +# Contributing to flowctl + +Thanks for your interest. flowctl is a control plane for Stellar data pipelines; it is +open source and meant to stay usable without any hosted OBSRVR service. + +## Before you start + +- **Found a security problem?** Do not open an issue. See [SECURITY.md](SECURITY.md). +- **Large or cross-cutting change?** Open an issue describing the problem before writing + code. A change that alters a contract — CLI flags, JSON output, exit codes, config + format, the gRPC API, or event schemas — needs an + [ADR](docs/adr/) accepted first. See "Changing a contract" below. +- **Small fix?** Just open a PR. + +## Development setup + +Two commands from clone to passing tests: + +```bash +nix develop # or: direnv allow +make test +``` + +Without Nix, you need Go (the version in `go.mod`) and `protoc` with the Go plugins +(`make install-proto-tools`). The Nix shell is what CI uses, so it is the supported path. + +Useful targets: + +```bash +make build # build ./bin/flowctl +make test # run the test suite +make fmt # gofmt the tree +make lint # golangci-lint +make generate # regenerate protobuf code +``` + +## Making a change + +1. Branch from `main`. +2. Write the change and a test that fails without it. For a bug fix this is required — + the test must demonstrate the bug. +3. Run `make fmt && make test` before pushing. CI runs `gofmt`, `go vet`, + `golangci-lint`, `go test -race`, `govulncheck`, secret scanning and `buf lint`; + all of them block the merge. +4. Open a PR describing what changed and why. Link the issue or ADR. + +### Commit messages + +Use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) — release +notes and version bumps are generated from them. + +``` +feat(cli): add --output ndjson to pipelines list +fix(runner): stop leaking a goroutine when a sink disconnects +docs: document the control plane trust boundary +chore(deps): bump grpc to 1.79.2 +``` + +Types in use: `feat`, `fix`, `docs`, `test`, `refactor`, `perf`, `build`, `ci`, `chore`, +`style`. Breaking changes get a `!` (`feat(cli)!: …`) and a `BREAKING CHANGE:` footer. + +### Changing a contract + +flowctl's contracts are listed in [`docs/engineering-profile.md`](docs/engineering-profile.md). +They include CLI flags and arguments, exit codes, `--output json`/`ndjson` shapes, the +pipeline config format, the gRPC API, the event envelope, and the on-disk run store. + +Every change to one must be classified in the PR description: + +| Class | Example | What it needs | +|---|---|---| +| **Additive** | New optional field, new flag, new subcommand | Minor release. Consumers must ignore unknown fields. | +| **Deprecating** | A flag or field scheduled for removal | Deprecation notice, migration note, removal date | +| **Breaking** | Removed or renamed field, changed type, changed exit code | Major release (pre-1.0: minor), an ADR, a migration window | +| **Semantic** | Same wire format, different meaning | Treated as breaking: version bump, parity test, replay decision | + +Protobuf changes must keep field numbers and wire types, and must `reserve` removed +numbers and names. `buf breaking` runs in CI against the last release. + +### Architecture Decision Records + +Decisions that change a contract, the architecture, or the security posture go in +`docs/adr/NNNN-title.md` in [MADR](https://adr.github.io/madr/) form. Copy +`docs/adr/0000-template.md`. ADRs are not edited after acceptance — supersede them with +a new one. + +## Testing expectations + +- Concurrent code must pass `go test -race`. +- Parsers of external input (pipeline YAML, CUE, anything off the wire) need fuzz tests. +- A bug fix needs a test that fails without the fix. +- Prefer real captured data over invented data. Invented data is for injecting failures + — timeouts, malformed input, storage errors — not for showing that a feature works. + +## Code style + +- `gofmt` is enforced; `golangci-lint` config lives in `.golangci.yml`. +- Logs go to stderr, structured, via `internal/utils/logger`. Results go to stdout via + `internal/cliout`. Never print results with bare `fmt.Println` — it breaks + `flowctl … | jq`. +- Anything that arrives from a registered component is untrusted. Sanitize it with + `internal/sanitize` before it reaches a terminal, a log, or persisted state. +- Return errors wrapped with an exit code (`internal/exitcode`) so the CLI exits with + something a script can branch on. + +## Licence + +flowctl is [MPL-2.0](LICENSE). By contributing you agree your contribution is licensed +under it. There is no CLA. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e87a115 --- /dev/null +++ b/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/Makefile b/Makefile index a0f826a..57645ef 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,24 @@ -.PHONY: all generate proto build build-all install test clean run server deps fmt lint help +.PHONY: all generate proto proto-lint proto-breaking build build-all install test test-race clean run server deps fmt lint vulncheck verify-repro help SHELL := /usr/bin/env bash # Version information for reproducible release builds. +# +# BUILD_DATE is the *commit* date, not the wall clock: BUILD-3 requires release +# builds to embed no build time other than the commit's, so that building the +# same tag twice produces the same bytes. SOURCE_DATE_EPOCH overrides it, per the +# reproducible-builds.org convention. VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo none) -BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +# Derived from the commit, never the wall clock. Note this deliberately does not +# read an ambient SOURCE_DATE_EPOCH: the Nix dev shell exports 315532800 (1980), +# which would stamp every local build with a date that tells you nothing. The +# commit date is both reproducible and meaningful. +COMMIT_EPOCH := $(shell git log -1 --pretty=%ct 2>/dev/null || echo 0) +BUILD_DATE ?= $(shell date -u -d @$(COMMIT_EPOCH) +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -r $(COMMIT_EPOCH) +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown) +export SOURCE_DATE_EPOCH = $(COMMIT_EPOCH) +# -trimpath keeps absolute paths from the build machine out of the binary. +GOFLAGS_REPRO := -trimpath LDFLAGS := -s -w \ -X github.com/withobsrvr/flowctl/cmd/version.Version=$(VERSION) \ -X github.com/withobsrvr/flowctl/cmd/version.Commit=$(COMMIT) \ @@ -17,18 +30,31 @@ all: generate build # Generate protobuf files generate: proto +# buf is the single generator (see buf.gen.yaml). Plugin versions are pinned +# there, so this produces the same bytes on every machine and in CI — which is +# what lets CI check that the committed output is current (DATA-5). +BUF ?= go run github.com/bufbuild/buf/cmd/buf@v1.47.2 + proto: @echo "Generating protobuf files..." - @protoc --go_out=. --go_opt=paths=source_relative \ - --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - proto/*.proto + @$(BUF) generate @echo "Protobuf generation complete!" +# Lint and breaking-change checks for the protobuf contracts (VER-5). +proto-lint: + @$(BUF) lint + +proto-breaking: + @latest=$$(git describe --tags --abbrev=0 --match 'v*' 2>/dev/null); \ + if [ -z "$$latest" ]; then echo "No release tag; skipping."; exit 0; fi; \ + echo "Comparing protobuf against $$latest"; \ + $(BUF) breaking --against ".git#tag=$$latest" + # Build the binary build: @echo "Building flowctl $(VERSION)..." @mkdir -p bin - @CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/flowctl . + @CGO_ENABLED=0 go build $(GOFLAGS_REPRO) -ldflags "$(LDFLAGS)" -o bin/flowctl . @echo "Binary built at bin/flowctl" # Build release binaries for common platforms @@ -39,29 +65,38 @@ build-all: for goarch in amd64 arm64; do \ out="dist/flowctl_$(VERSION)_$${goos}_$${goarch}"; \ echo " -> $$out"; \ - CGO_ENABLED=0 GOOS=$${goos} GOARCH=$${goarch} go build -ldflags "$(LDFLAGS)" -o "$$out" .; \ + CGO_ENABLED=0 GOOS=$${goos} GOARCH=$${goarch} go build $(GOFLAGS_REPRO) -ldflags "$(LDFLAGS)" -o "$$out" .; \ done; \ done install: @echo "Installing flowctl $(VERSION)..." - @CGO_ENABLED=0 go install -ldflags "$(LDFLAGS)" . + @CGO_ENABLED=0 go install $(GOFLAGS_REPRO) -ldflags "$(LDFLAGS)" . @echo "Installed flowctl to $$GOPATH/bin (or ~/go/bin)" +# Package list for the test targets. +# +# Built from the filesystem rather than ./... so that a local runtime directory +# — an unreadable ./data mount, say — cannot break package discovery. The +# testfixtures subpackages carry a `test` build tag and are excluded here for +# the same reason `go list ./...` omits them. +GOFILES = $(shell find . \ + \( -path './.git' -o -path './data' -o -path './bin' -o -path './dist' \ + -o -path './internal/testfixtures/source' -o -path './internal/testfixtures/processor' \ + -o -path './internal/testfixtures/sink' \) -prune \ + -o -name '*.go' -print) +PKGS = $(shell printf '%s\n' $(GOFILES) | xargs -r -n1 dirname | sort -u | sed 's#^#./#' | sed 's#\.//#./#') + # Run tests -# Use a filesystem-based package list so local runtime directories (for example -# unreadable data mounts under ./data) do not break package discovery. test: @echo "Running tests..." - @GOFILES=$$(find . \ - \( -path './.git' -o -path './data' -o -path './bin' -o -path './internal/testfixtures/source' -o -path './internal/testfixtures/processor' -o -path './internal/testfixtures/sink' \) -prune \ - -o -name '*.go' -print); \ - if [ -n "$$GOFILES" ]; then \ - PKGS=$$(printf '%s\n' "$$GOFILES" | xargs -n1 dirname | sort -u | sed 's#^#./#' | sed 's#\.//#./#'); \ - go test -v $$PKGS; \ - else \ - echo "No Go packages found."; \ - fi + @go test -v $(PKGS) + +# Run tests with the race detector (TEST-4). The runner and control plane are +# heavily concurrent, so this is a merge gate, not an optional extra. +test-race: + @echo "Running tests with -race..." + @go test -race $(PKGS) # Run the application run: @@ -84,37 +119,54 @@ deps: @go mod download @go mod tidy -# Install protoc plugins -install-proto-tools: - @echo "Installing protobuf Go plugins..." - @go install google.golang.org/protobuf/cmd/protoc-gen-go@latest - @go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest - @echo "Protobuf tools installed!" - # Format code fmt: @echo "Formatting code..." @go fmt ./... @echo "Formatting complete!" -# Lint code +# Lint code. These are the §9 merge-gate checks; CI runs the same set. lint: - @echo "Linting code..." - @golangci-lint run + @echo "Checking formatting..." + @test -z "$$(gofmt -l . | grep -v '^data/')" || { echo "gofmt needed:"; gofmt -l . | grep -v '^data/'; exit 1; } + @echo "Running go vet..." + @go vet ./... + @echo "Running staticcheck..." + @staticcheck ./... @echo "Linting complete!" +# Scan dependencies for known vulnerabilities (SEC-9). Findings that genuinely +# do not affect flowctl are exempted one at a time, with a justification, in +# .vex/flowctl.openvex.json (SEC-12) — never by ignoring the exit code. +vulncheck: + @./scripts/govulncheck.sh + +# Verify a release build is reproducible: build twice, compare hashes (BUILD-3). +verify-repro: + @echo "Building twice and comparing..." + @$(MAKE) --no-print-directory build && sha256sum bin/flowctl > /tmp/flowctl.sha1 + @rm -f bin/flowctl + @$(MAKE) --no-print-directory build && sha256sum bin/flowctl | sed 's#bin/flowctl#bin/flowctl#' > /tmp/flowctl.sha2 + @diff <(cut -d" " -f1 /tmp/flowctl.sha1) <(cut -d" " -f1 /tmp/flowctl.sha2) \ + && echo "Reproducible: hashes match" \ + || { echo "NOT reproducible: hashes differ"; exit 1; } + # Help target help: @echo "Available targets:" - @echo " make generate - Generate protobuf files" + @echo " make generate - Generate protobuf files (via buf)" + @echo " make proto-lint - Lint the protobuf contracts" @echo " make build - Build the flowctl binary" @echo " make build-all - Build release binaries for common platforms" @echo " make install - Install flowctl to GOPATH/bin" @echo " make test - Run tests" + @echo " make test-race - Run tests with the race detector" + @echo " make lint - gofmt, go vet and staticcheck (the CI merge gate)" + @echo " make vulncheck - Scan dependencies with govulncheck" + @echo " make verify-repro- Build twice and compare hashes" @echo " make run - Run flowctl (use ARGS='server' for server mode)" @echo " make server - Run flowctl server" @echo " make clean - Clean build artifacts" @echo " make deps - Install Go dependencies" @echo " make fmt - Format Go code" - @echo " make lint - Lint Go code" @echo " make help - Show this help message" \ No newline at end of file diff --git a/README.md b/README.md index 3739092..ec8119d 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ Common examples: ```bash flowctl init flowctl init --preset testnet-duckdb -flowctl init --non-interactive --network mainnet --destination postgres -o prod-pipeline.yaml +flowctl init --non-interactive --network mainnet --destination postgres -f prod-pipeline.yaml ``` Destination prerequisites: @@ -419,7 +419,7 @@ Discover and inspect processors registered with the control plane at runtime. ./bin/flowctl processors find --input stellar.ledger.v1 # Find processors that produce token.transfer events -./bin/flowctl processors find --output stellar.token.transfer.v1 +./bin/flowctl processors find --output-type stellar.token.transfer.v1 # Find processors for a specific network ./bin/flowctl processors find --metadata network=testnet diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..59dcc84 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,83 @@ +# Security Policy + +## Reporting a vulnerability + +**Do not open a public issue for a security vulnerability.** + +Report it privately through either channel: + +- **GitHub Security Advisories** — [open a private advisory](https://github.com/withObsrvr/flowctl/security/advisories/new) + on this repository. This is preferred: it gives us a private fork to develop and test + the fix in, and it issues the CVE. +- **Email** — security@withobsrvr.com. If you want to encrypt, say so in a first + contactless message and we will send a key. + +Please include, as far as you can determine it: + +- the flowctl version (`flowctl version`) and how it was installed; +- the component involved — CLI, embedded control plane, standalone `flowctl server`, + sandbox, or the translation layer; +- what an attacker gains, and what access they need to get it; +- a reproduction, ideally a pipeline YAML plus the commands you ran. + +## What to expect + +| Stage | Target | +|---|---| +| Acknowledgement of your report | 3 business days | +| Initial assessment and severity | 10 business days | +| Fix released, or a dated plan if it needs longer | 90 days from acknowledgement | + +We will tell you when we disagree that something is a vulnerability, and why. We credit +reporters in the advisory unless you ask us not to. + +flowctl is maintained by a very small team. If you have not heard from us within the +acknowledgement window, please escalate by emailing tillman@withobsrvr.com directly — +it means something went wrong with routing, not that we are ignoring you. + +## Scope + +In scope: + +- The `flowctl` binary and everything under this repository. +- The control plane gRPC API, embedded or standalone. +- The pipeline configuration parser and the CUE schema, including + denial-of-service and injection through crafted pipeline YAML. +- The translation layer's generated output (`docker-compose`, local runner scripts), + including command or argument injection from pipeline configuration. +- Terminal escape-sequence injection through values that reach a user's terminal. + +Out of scope: + +- Vulnerabilities in the components flowctl runs. Report those to the component's own + repository. flowctl orchestrates arbitrary user-supplied container images and + processes, and runs them with the privileges you give it; that is by design. +- The absence of authentication on the control plane. **This is a known, documented + limitation, not a vulnerability report we need** — see below. +- Findings that require an attacker to already have local code execution as the user + running flowctl. + +## Known limitations + +These are deliberate, documented gaps, tracked in +[`docs/engineering-profile.md`](docs/engineering-profile.md). Reporting them is not +necessary — but reporting a *bypass* of a control we claim to have is. + +- **The control plane gRPC API has no authentication or authorization** (gap GAP-12). + Any process that can reach the port can register services, read run state, and stop + runs. Do not expose the control plane port to an untrusted network. Bind it to + localhost, or put it behind mTLS (see "Secure Communication with TLS" in the README) + and a trusted network boundary. +- **Released binaries are not yet signed, and carry no SBOM or provenance** (gap GAP-11). + Verify downloads against `checksums.txt` on the release until that lands. + +## Supported versions + +flowctl is pre-1.0. Only the latest released minor version receives security fixes. +Once 1.0 ships, this table will state a support window per the OBSRVR Engineering +Standard (VER-6). + +| Version | Supported | +|---|---| +| Latest release | Yes | +| Anything older | No — upgrade | diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000..80ee180 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,22 @@ +version: v2 + +# buf is the single generator for this repo. Previously `make generate` shelled +# out to a locally installed protoc, so the checked-in output depended on +# whichever plugin versions a given developer happened to have — which made the +# "generated code is current" check in CI unstable. +# +# Plugin versions are pinned for the same reason the Go toolchain is (BUILD-2): +# regenerating must produce the same bytes everywhere. Bumping one is a normal +# dependency update, and the resulting diff should be committed with it. +managed: + enabled: false + +plugins: + - remote: buf.build/protocolbuffers/go:v1.36.12 + out: . + opt: + - paths=source_relative + - remote: buf.build/grpc/go:v1.6.2 + out: . + opt: + - paths=source_relative diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 0000000..148bb95 --- /dev/null +++ b/buf.yaml @@ -0,0 +1,45 @@ +version: v2 + +modules: + - path: . + +lint: + use: + - STANDARD + except: + # The four rules below all demand renaming existing messages or services, + # which changes their fully-qualified names on the wire and breaks every + # deployed component — the same migration deferred in ADR 0002, and they + # should be done in it rather than piecemeal. + # + # RPC_REQUEST_STANDARD_NAME / RPC_RESPONSE_STANDARD_NAME want + # Request / Response. flowctl's RPCs take and return + # domain messages (Event, RawLedger, SinkAck). + - RPC_REQUEST_STANDARD_NAME + - RPC_RESPONSE_STANDARD_NAME + # RPC_REQUEST_RESPONSE_UNIQUE forbids sharing a message between RPCs; + # Ping/Shutdown deliberately share Empty, Pong and ShutdownRequest across + # the source, processor and sink services. + - RPC_REQUEST_RESPONSE_UNIQUE + # SERVICE_SUFFIX wants every service named *Service. + - SERVICE_SUFFIX + # ENUM_ZERO_VALUE_SUFFIX wants *_UNSPECIFIED; RunStatus, ChunkStatus and + # FailureClass use *_UNKNOWN. The binary encoding is unaffected (zero is + # still zero) but the Go constant and the JSON name both change, so it + # breaks source compatibility for flowctl-sdk consumers. Same migration, + # same ADR. + - ENUM_ZERO_VALUE_SUFFIX + # flowctl's protobuf package is `flowctl`, not `flowctl.v1`. Renaming it + # changes the fully-qualified name of every message and the gRPC path of + # every method, breaking every deployed component and every build of + # flowctl-sdk at once. The rename is correct and is a release blocker for + # 1.0; it is deferred deliberately, with the exception recorded under + # CONF-1 in docs/adr/0002-defer-protobuf-package-versioning.md. + - PACKAGE_VERSION_SUFFIX + # Directory structure is expected to match the package name, which the + # same rename would fix. Same ADR, same expiry. + - PACKAGE_DIRECTORY_MATCH + +breaking: + use: + - FILE diff --git a/cmd/chunks.go b/cmd/chunks.go index fb2a803..73b1ad8 100644 --- a/cmd/chunks.go +++ b/cmd/chunks.go @@ -3,16 +3,105 @@ package cmd import ( "context" "fmt" - "os" "sort" "strings" - "text/tabwriter" "time" "github.com/spf13/cobra" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" flowctlpb "github.com/withobsrvr/flowctl/proto" ) +// Schema identifiers for this command group's machine-readable output. +const ( + chunkListSchema = "flowctl.chunk_run_list.v1" + chunkSchema = "flowctl.chunk_run.v1" +) + +// chunkRow converts a chunk run into a record. Machine output carries the +// numbers and maps as themselves; the table condenses them. +func chunkRow(chunk *flowctlpb.ChunkRun) cliout.Row { + row := cliout.Row{ + "chunk_id": chunk.GetChunkId(), + "pipeline_run_id": chunk.GetPipelineRunId(), + "component_id": chunk.GetComponentId(), + "chunk_start": chunk.GetChunkStart(), + "chunk_end": chunk.GetChunkEnd(), + "attempt": chunk.GetAttempt(), + "status": formatChunkStatus(chunk.GetStatus()), + "phase": chunk.GetPhase(), + "total_rows": totalRows(chunk.GetRowCounts()), + } + if chunk.GetFailureClass() != flowctlpb.FailureClass_FAILURE_CLASS_UNKNOWN { + row["failure_class"] = formatFailureClass(chunk.GetFailureClass()) + } + if v := chunk.GetError(); v != "" { + row["error"] = v + } + if v := chunk.GetRecommendedAction(); v != "" { + row["recommended_action"] = v + } + if ts := chunk.GetStartedAt(); ts != nil { + row["started_at"] = ts.AsTime() + } + if ts := chunk.GetCompletedAt(); ts != nil { + row["completed_at"] = ts.AsTime() + } + if ts := chunk.GetVerifiedAt(); ts != nil { + row["verified_at"] = ts.AsTime() + } + if len(chunk.GetRowCounts()) > 0 { + row["row_counts"] = chunk.GetRowCounts() + } + if len(chunk.GetVerification()) > 0 { + row["verification"] = chunk.GetVerification() + } + return row +} + +// chunkRange renders the ledger range for a table cell. +func chunkRange(row cliout.Row) string { + return fmt.Sprintf("%v-%v", row["chunk_start"], row["chunk_end"]) +} + +// sortedCounts renders an int64 map for a detail view, keys sorted so output +// is stable between runs. +func sortedCounts(v any) string { + m, ok := v.(map[string]int64) + if !ok || len(m) == 0 { + return "-" + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, fmt.Sprintf("%s=%d", cliout.Value(k), m[k])) + } + return strings.Join(parts, ", ") +} + +// sortedStrings renders a string map for a detail view, keys sorted. +func sortedStrings(v any) string { + m, ok := v.(map[string]string) + if !ok || len(m) == 0 { + return "-" + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, cliout.Value(k)+"="+cliout.Value(m[k])) + } + return strings.Join(parts, ", ") +} + var ( chunksRunID string chunksComponentID string @@ -63,13 +152,11 @@ var chunksListCmd = &cobra.Command{ Limit: chunksLimit, }) if err != nil { - return fmt.Errorf("failed to list chunk runs: %w", err) + return err } - if len(resp.Chunks) == 0 { - fmt.Println("No chunk runs found") - return nil - } + // No sanitize.Message here: cliout strips control characters as it + // formats each cell, and machine output keeps the original bytes. sort.Slice(resp.Chunks, func(i, j int) bool { if resp.Chunks[i].ChunkStart == resp.Chunks[j].ChunkStart { @@ -78,28 +165,47 @@ var chunksListCmd = &cobra.Command{ return resp.Chunks[i].ChunkStart < resp.Chunks[j].ChunkStart }) - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "CHUNK ID\tCOMPONENT\tRANGE\tATTEMPT\tSTATUS\tFAILURE\tPHASE\tROWS") + rows := make([]cliout.Row, 0, len(resp.Chunks)) for _, chunk := range resp.Chunks { - fmt.Fprintf(w, "%s\t%s\t%d-%d\t%d\t%s\t%s\t%s\t%d\n", - chunk.ChunkId, - chunk.ComponentId, - chunk.ChunkStart, - chunk.ChunkEnd, - chunk.Attempt, - formatChunkStatus(chunk.Status), - formatFailureClass(chunk.FailureClass), - chunk.Phase, - totalRows(chunk.RowCounts), - ) + rows = append(rows, chunkRow(chunk)) + } + + p := cliout.FromContext(cmd.Context()) + if err := p.List(cliout.List{ + Schema: chunkListSchema, + ItemSchema: chunkSchema, + Columns: []cliout.Column{ + {Header: "Chunk ID", Key: "chunk_id"}, + {Header: "Component", Key: "component_id"}, + {Header: "Range", Key: "_range"}, + {Header: "Attempt", Key: "attempt"}, + {Header: "Status", Key: "status"}, + {Header: "Failure", Key: "failure_class"}, + {Header: "Phase", Key: "phase"}, + {Header: "Rows", Key: "total_rows"}, + }, + Rows: withRange(rows), + Empty: "No chunk runs found.", + }); err != nil { + return err } - w.Flush() - fmt.Printf("\n%d chunk run(s) found\n", len(resp.Chunks)) + if !p.Machine() && len(rows) > 0 { + p.Info("\n%d chunk run(s) found", len(rows)) + } return nil }, } +// withRange adds the derived display-only range cell. It is not part of the +// record: machine consumers get chunk_start and chunk_end as numbers. +func withRange(rows []cliout.Row) []cliout.Row { + for _, row := range rows { + row["_range"] = chunkRange(row) + } + return rows +} + var chunksShowCmd = &cobra.Command{ Use: "show ", Short: "Show detailed information about a chunk run", @@ -116,57 +222,37 @@ var chunksShowCmd = &cobra.Command{ chunk, err := client.GetChunkRun(ctx, &flowctlpb.GetChunkRunRequest{ChunkId: args[0]}) if err != nil { - return fmt.Errorf("failed to get chunk run: %w", err) + return exitcode.Errorf(exitcode.NotFound, + "chunk run %q not found: %v", args[0], err) } - fmt.Printf("Chunk ID: %s\n", chunk.ChunkId) - fmt.Printf("Run ID: %s\n", chunk.PipelineRunId) - fmt.Printf("Component: %s\n", chunk.ComponentId) - fmt.Printf("Range: %d-%d\n", chunk.ChunkStart, chunk.ChunkEnd) - fmt.Printf("Attempt: %d\n", chunk.Attempt) - fmt.Printf("Status: %s\n", formatChunkStatus(chunk.Status)) - fmt.Printf("Phase: %s\n", chunk.Phase) - if chunk.FailureClass != flowctlpb.FailureClass_FAILURE_CLASS_UNKNOWN { - fmt.Printf("Failure: %s\n", formatFailureClass(chunk.FailureClass)) - } - if chunk.Error != "" { - fmt.Printf("Error: %s\n", chunk.Error) - } - if chunk.RecommendedAction != "" { - fmt.Printf("Next action: %s\n", chunk.RecommendedAction) - } - if chunk.StartedAt != nil { - fmt.Printf("Started: %s\n", chunk.StartedAt.AsTime().Format("2006-01-02 15:04:05 MST")) - } - if chunk.CompletedAt != nil { - fmt.Printf("Completed: %s\n", chunk.CompletedAt.AsTime().Format("2006-01-02 15:04:05 MST")) - } - if chunk.VerifiedAt != nil { - fmt.Printf("Verified: %s\n", chunk.VerifiedAt.AsTime().Format("2006-01-02 15:04:05 MST")) - } - if len(chunk.RowCounts) > 0 { - fmt.Println("\nRow counts:") - keys := make([]string, 0, len(chunk.RowCounts)) - for k := range chunk.RowCounts { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - fmt.Printf(" %s: %d\n", k, chunk.RowCounts[k]) - } - } - if len(chunk.Verification) > 0 { - fmt.Println("\nVerification:") - keys := make([]string, 0, len(chunk.Verification)) - for k := range chunk.Verification { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - fmt.Printf(" %s: %s\n", k, chunk.Verification[k]) - } - } - return nil + // No sanitize.Message here: cliout strips control characters as it + // formats each cell, and machine output keeps the original bytes. + + row := chunkRow(chunk) + row["_range"] = chunkRange(row) + + return cliout.FromContext(cmd.Context()).Object(cliout.Object{ + Schema: chunkSchema, + Fields: []cliout.Field{ + {Label: "Chunk ID", Key: "chunk_id"}, + {Label: "Run ID", Key: "pipeline_run_id"}, + {Label: "Component", Key: "component_id"}, + {Label: "Range", Key: "_range"}, + {Label: "Attempt", Key: "attempt"}, + {Label: "Status", Key: "status"}, + {Label: "Phase", Key: "phase"}, + {Label: "Failure", Key: "failure_class"}, + {Label: "Error", Key: "error"}, + {Label: "Next action", Key: "recommended_action"}, + {Label: "Started", Key: "started_at"}, + {Label: "Completed", Key: "completed_at"}, + {Label: "Verified", Key: "verified_at"}, + {Label: "Row counts", Key: "row_counts", Format: sortedCounts}, + {Label: "Verification", Key: "verification", Format: sortedStrings}, + }, + Data: row, + }) }, } @@ -193,13 +279,6 @@ func formatFailureClass(failure flowctlpb.FailureClass) string { return strings.ToLower(strings.TrimPrefix(failure.String(), "FAILURE_CLASS_")) } -func shortChunkID(chunkID string) string { - if len(chunkID) <= 18 { - return chunkID - } - return chunkID[:18] -} - func totalRows(rows map[string]int64) int64 { var total int64 for _, count := range rows { diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..7161ff9 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,109 @@ +package cmd + +import ( + "os" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/withobsrvr/flowctl/internal/cliout" +) + +// settings are the configuration keys flowctl resolves through the precedence chain. +// Each is a global flag bound to viper in root.go. +var settings = []string{"context", "namespace", "output", "log-level"} + +// newConfigCommand builds `flowctl config`. +func newConfigCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "config", + Short: "Inspect flowctl configuration", + Long: `Inspect the configuration flowctl resolved for this invocation. + +Configuration is resolved in this order, highest priority first: + + 1. Command-line flags + 2. FLOWCTL_* environment variables + 3. A flowctl.yaml in the current directory + 4. A flowctl.yaml in $XDG_CONFIG_HOME/flowctl (or ~/.config/flowctl) + 5. Built-in defaults`, + } + cmd.AddCommand(newConfigShowCommand()) + return cmd +} + +func newConfigShowCommand() *cobra.Command { + return &cobra.Command{ + Use: "show", + Short: "Show each setting and where its value came from", + Long: `Print every configuration setting, its resolved value, and the source +that supplied it. + +Examples: + # See what flowctl thinks its configuration is + flowctl config show + + # Check which file is being read + flowctl config show --output json | jq -r '.items[] | select(.source=="config file")'`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + p := cliout.FromContext(cmd.Context()) + + rows := make([]cliout.Row, 0, len(settings)) + for _, key := range settings { + value, source, origin := resolve(cmd, key) + rows = append(rows, cliout.Row{ + "key": key, + "value": value, + "source": source, + "origin": origin, + }) + } + + if file := viper.ConfigFileUsed(); file != "" { + p.Info("Config file: %s", file) + } else { + p.Info("Config file: none found") + } + + return p.List(cliout.List{ + Schema: "flowctl.config_setting_list.v1", + ItemSchema: "flowctl.config_setting.v1", + Columns: []cliout.Column{ + {Header: "Key", Key: "key"}, + {Header: "Value", Key: "value"}, + {Header: "Source", Key: "source"}, + {Header: "Origin", Key: "origin"}, + }, + Rows: rows, + }) + }, + } +} + +// resolve reports a setting's value, which layer supplied it, and the specific +// origin within that layer — the flag name, the environment variable, or the file. +func resolve(cmd *cobra.Command, key string) (value, source, origin string) { + value = viper.GetString(key) + + if f := cmd.Flags().Lookup(key); f != nil && f.Changed { + return value, "flag", "--" + key + } + if f := cmd.Root().PersistentFlags().Lookup(key); f != nil && f.Changed { + return value, "flag", "--" + key + } + + env := envPrefix + "_" + strings.ToUpper(strings.NewReplacer("-", "_", ".", "_").Replace(key)) + if _, set := os.LookupEnv(env); set { + return value, "environment", env + } + + if viper.InConfig(key) { + if file := viper.ConfigFileUsed(); file != "" { + return value, "config file", file + } + return value, "config file", "unknown" + } + + return value, "default", "built-in" +} diff --git a/cmd/dashboard.go b/cmd/dashboard.go index 8d4ef0f..a9b5fd4 100644 --- a/cmd/dashboard.go +++ b/cmd/dashboard.go @@ -3,14 +3,18 @@ package cmd import ( "context" "fmt" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" + "github.com/withobsrvr/flowctl/internal/sanitize" + "os" "strings" "time" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/spf13/cobra" - flowctlpb "github.com/withobsrvr/flowctl/proto" flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" + flowctlpb "github.com/withobsrvr/flowctl/proto" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -32,7 +36,23 @@ Keyboard shortcuts: ↑/↓ - Navigate (future) The dashboard auto-refreshes every second by default.`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { + printer := cliout.FromContext(cmd.Context()) + + // The dashboard draws a full-screen terminal UI; there is no document + // it could emit instead. Say so rather than rendering escape codes into + // a pipe (CLI-6, CLI-7). + if printer.Machine() { + return exitcode.Errorf(exitcode.Usage, + "the dashboard is interactive and cannot produce --output %s; use 'flowctl status --output %s' instead", + printer.Format(), printer.Format()) + } + if !cliout.IsTerminal(os.Stdout) { + return exitcode.Errorf(exitcode.Usage, + "the dashboard needs a terminal; use 'flowctl status' when output is redirected") + } + // Create the bubbletea program p := tea.NewProgram(initialModel(), tea.WithAltScreen()) @@ -194,9 +214,13 @@ func fetchDataCmd(addr string, port int) tea.Cmd { return dataMsg{err: fmt.Errorf("failed to list components: %w", err)} } + // Both lists are written by components flowctl did not start and are + // rendered into the operator's terminal, so strip control characters + // first (CLI-22). A refreshing TUI is a particularly attractive + // injection target, since the payload is redrawn continuously. return dataMsg{ - pipelines: runsResp.Runs, - components: compsResp.Components, + pipelines: sanitize.Message(runsResp).Runs, + components: sanitize.Message(compsResp).Components, } } } @@ -284,52 +308,52 @@ func countRunning(pipelines []*flowctlpb.PipelineRun) int { var ( // Colors - primaryColor = lipgloss.Color("#7D56F4") - successColor = lipgloss.Color("#04B575") - errorColor = lipgloss.Color("#FF0000") - dimColor = lipgloss.Color("#666666") + primaryColor = lipgloss.Color("#7D56F4") + successColor = lipgloss.Color("#04B575") + errorColor = lipgloss.Color("#FF0000") + dimColor = lipgloss.Color("#666666") // Text styles headerStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(primaryColor). - PaddingLeft(2) + Bold(true). + Foreground(primaryColor). + PaddingLeft(2) sectionTitleStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(primaryColor) + Bold(true). + Foreground(primaryColor) boxStyle = lipgloss.NewStyle(). - Border(lipgloss.RoundedBorder()). - BorderForeground(primaryColor). - Padding(1, 2). - Width(80) + Border(lipgloss.RoundedBorder()). + BorderForeground(primaryColor). + Padding(1, 2). + Width(80) boldStyle = lipgloss.NewStyle(). - Bold(true) + Bold(true) dimStyle = lipgloss.NewStyle(). - Foreground(dimColor) + Foreground(dimColor) activeStyle = lipgloss.NewStyle(). - Foreground(successColor). - Bold(true) + Foreground(successColor). + Bold(true) healthyStyle = lipgloss.NewStyle(). - Foreground(successColor). - Bold(true) + Foreground(successColor). + Bold(true) unhealthyStyle = lipgloss.NewStyle(). - Foreground(errorColor). - Bold(true) + Foreground(errorColor). + Bold(true) errorStyle = lipgloss.NewStyle(). - Foreground(errorColor). - Bold(true) + Foreground(errorColor). + Bold(true) helpStyle = lipgloss.NewStyle(). - Foreground(dimColor). - Italic(true) + Foreground(dimColor). + Italic(true) ) func init() { diff --git a/cmd/init.go b/cmd/init.go index 7aed361..59b19ba 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -5,6 +5,10 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/withobsrvr/flowctl/internal/apiversion" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" + "io" "net/http" "os" "strconv" @@ -38,7 +42,7 @@ Examples: flowctl init # Specify output file - flowctl init --output my-pipeline.yaml + flowctl init --file my-pipeline.yaml # Non-interactive mode (use flags) flowctl init --non-interactive --network testnet --destination duckdb @@ -52,17 +56,43 @@ Examples: func init() { rootCmd.AddCommand(initCmd) - initCmd.Flags().StringVarP(&outputFile, "output", "o", "", "output file (default: .yaml)") + // Renamed from --output, which CLI-3 reserves for selecting an output format. + // No deprecated alias: a local --output would shadow the global one, so + // `--output json` would silently be read as a filename — the exact confusion + // the rule exists to prevent. Pre-1.0, VER-2 permits the clean break. + initCmd.Flags().StringVarP(&outputFile, "file", "f", "", "file to write (default: .yaml)") initCmd.Flags().BoolVar(&initNonInteractive, "non-interactive", false, "non-interactive mode (requires --network/--destination unless --preset is used)") initCmd.Flags().StringVar(&networkFlag, "network", "", "network (testnet, mainnet)") initCmd.Flags().StringVar(&destinationFlag, "destination", "", "destination (postgres, duckdb)") initCmd.Flags().StringVar(&presetFlag, "preset", "", "starter preset (testnet-duckdb, testnet-postgres, mainnet-duckdb, mainnet-postgres)") } +// wizardOut is where the interactive wizard writes. Prompts, progress and the +// closing summary are all messages, not results, so they go to stderr and +// leave stdout for the record emitted at the end (CLI-5). +var wizardOut io.Writer = os.Stderr + func runInit(cmd *cobra.Command, args []string) error { - if !initNonInteractive && presetFlag == "" { - fmt.Println("\n🚀 Welcome to flowctl!") - fmt.Println("Let's create your first Stellar data pipeline.") + p := cliout.FromContext(cmd.Context()) + wizardOut = p.Err() + + interactive := !initNonInteractive && presetFlag == "" + + // CLI-8: a tool must run without prompts when stdin is not a terminal. + // Without this the wizard reads EOF for every answer and silently builds a + // pipeline from defaults. + if interactive && !cliout.IsTerminal(os.Stdin) { + return exitcode.Errorf(exitcode.Usage, + "flowctl init needs a terminal to prompt; use --preset, or --non-interactive with --network and --destination") + } + if interactive && p.Machine() { + return exitcode.Errorf(exitcode.Usage, + "--output %s cannot be combined with the interactive wizard; use --preset or --non-interactive", p.Format()) + } + + if interactive { + fmt.Fprintln(wizardOut, "\n🚀 Welcome to flowctl!") + fmt.Fprintln(wizardOut, "Let's create your first Stellar data pipeline.") } reader := bufio.NewReader(os.Stdin) @@ -121,12 +151,35 @@ func runInit(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to write pipeline file: %w", err) } - fmt.Printf("\n✓ Created %s\n\n", filename) + fmt.Fprintf(wizardOut, "\n✓ Created %s\n\n", filename) // Show summary printSummary(intent, filename, blockchain, network, destination) - return nil + row := cliout.Row{ + "file": filename, + "name": name, + "blockchain": blockchain, + "network": network, + "destination": destination, + "api_version": apiversion.Current, + } + if startLedger != "" { + row["start_ledger"] = startLedger + } + if endLedger != "" { + row["end_ledger"] = endLedger + } + row["continuous"] = startLedger == "" + + // In table mode the summary above already told the operator everything, so + // stdout carries just the path — which makes + // `flowctl run "$(flowctl init --preset …)"` work. + if !p.Machine() { + _, err := fmt.Fprintln(p.Out(), filename) + return err + } + return p.Object(cliout.Object{Schema: "flowctl.pipeline_init.v1", Data: row}) } func resolveInitPreset(preset string) (network string, destination string, err error) { @@ -147,65 +200,65 @@ func resolveInitPreset(preset string) (network string, destination string, err e func promptNetwork(reader *bufio.Reader) string { options := []string{"Testnet (recommended for learning)", "Mainnet"} - fmt.Println("? Which network?") + fmt.Fprintln(wizardOut, "? Which network?") for i, opt := range options { prefix := " " if i == 0 { prefix = ">" } - fmt.Printf(" %s %d. %s\n", prefix, i+1, opt) + fmt.Fprintf(wizardOut, " %s %d. %s\n", prefix, i+1, opt) } - fmt.Print("Enter number [1]: ") + fmt.Fprint(wizardOut, "Enter number [1]: ") choice := readIntWithDefault(reader, 1, 1, len(options)) // Extract network name (strip description) networkMap := []string{"testnet", "mainnet"} selected := networkMap[choice-1] - fmt.Printf("Selected: %s\n\n", options[choice-1]) + fmt.Fprintf(wizardOut, "Selected: %s\n\n", options[choice-1]) return selected } func promptDestination(reader *bufio.Reader) string { options := []string{"PostgreSQL database", "DuckDB file"} - fmt.Println("? What do you want to do with the data?") + fmt.Fprintln(wizardOut, "? What do you want to do with the data?") for i, opt := range options { prefix := " " if i == 0 { prefix = ">" } - fmt.Printf(" %s %d. %s\n", prefix, i+1, opt) + fmt.Fprintf(wizardOut, " %s %d. %s\n", prefix, i+1, opt) } - fmt.Print("Enter number [1]: ") + fmt.Fprint(wizardOut, "Enter number [1]: ") choice := readIntWithDefault(reader, 1, 1, len(options)) // Map display names to event types destMap := []string{"postgres", "duckdb"} selected := destMap[choice-1] - fmt.Printf("Selected: %s\n\n", options[choice-1]) + fmt.Fprintf(wizardOut, "Selected: %s\n\n", options[choice-1]) return selected } func promptLedgerRange(reader *bufio.Reader) [2]string { - fmt.Println("? Ledger range (optional - press Enter to skip for continuous processing):") + fmt.Fprintln(wizardOut, "? Ledger range (optional - press Enter to skip for continuous processing):") - fmt.Print(" Start ledger [leave empty for continuous]: ") + fmt.Fprint(wizardOut, " Start ledger [leave empty for continuous]: ") startLedger, _ := reader.ReadString('\n') startLedger = strings.TrimSpace(startLedger) var endLedger string if startLedger != "" { - fmt.Print(" End ledger [leave empty for continuous]: ") + fmt.Fprint(wizardOut, " End ledger [leave empty for continuous]: ") endLedger, _ = reader.ReadString('\n') endLedger = strings.TrimSpace(endLedger) } if startLedger != "" && endLedger != "" { - fmt.Printf("Selected: Ledgers %s to %s\n\n", startLedger, endLedger) + fmt.Fprintf(wizardOut, "Selected: Ledgers %s to %s\n\n", startLedger, endLedger) } else { - fmt.Println("Selected: Continuous processing") + fmt.Fprintln(wizardOut, "Selected: Continuous processing") } return [2]string{startLedger, endLedger} @@ -213,7 +266,7 @@ func promptLedgerRange(reader *bufio.Reader) [2]string { func promptName(reader *bufio.Reader, blockchain string) string { defaultName := blockchain + "-pipeline" - fmt.Printf("? Pipeline name [%s]: ", defaultName) + fmt.Fprintf(wizardOut, "? Pipeline name [%s]: ", defaultName) name, _ := reader.ReadString('\n') name = strings.TrimSpace(name) @@ -222,7 +275,7 @@ func promptName(reader *bufio.Reader, blockchain string) string { name = defaultName } - fmt.Printf("Selected: %s\n\n", name) + fmt.Fprintf(wizardOut, "Selected: %s\n\n", name) return name } @@ -236,7 +289,7 @@ func readIntWithDefault(reader *bufio.Reader, defaultVal, min, max int) int { choice, err := strconv.Atoi(input) if err != nil || choice < min || choice > max { - fmt.Printf("Invalid choice. Using default: %d\n", defaultVal) + fmt.Fprintf(wizardOut, "Invalid choice. Using default: %d\n", defaultVal) return defaultVal } @@ -418,12 +471,12 @@ func writeIntentToFile(intent map[string]interface{}, filename string) error { func printSummary(intent map[string]interface{}, filename, blockchain, network, destination string) { spec := intent["spec"].(map[string]interface{}) - fmt.Println("Pipeline summary") - fmt.Println("----------------") - fmt.Printf("Source: %s ledgers (%s)\n", blockchain, network) - fmt.Println("Processor: contract-events") - fmt.Printf("Destination: %s\n", destination) - fmt.Printf("File: %s\n", filename) + fmt.Fprintln(wizardOut, "Pipeline summary") + fmt.Fprintln(wizardOut, "----------------") + fmt.Fprintf(wizardOut, "Source: %s ledgers (%s)\n", blockchain, network) + fmt.Fprintln(wizardOut, "Processor: contract-events") + fmt.Fprintf(wizardOut, "Destination: %s\n", destination) + fmt.Fprintf(wizardOut, "File: %s\n", filename) // Show configuration details if sources, ok := spec["sources"].([]map[string]interface{}); ok && len(sources) > 0 { @@ -432,34 +485,34 @@ func printSummary(intent map[string]interface{}, filename, blockchain, network, endLedger := configValueString(config["end_ledger"]) if startLedger != "" { if endLedger != "" { - fmt.Printf("Ledgers: %s to %s\n", startLedger, endLedger) + fmt.Fprintf(wizardOut, "Ledgers: %s to %s\n", startLedger, endLedger) } else { - fmt.Printf("Start at: %s\n", startLedger) + fmt.Fprintf(wizardOut, "Start at: %s\n", startLedger) } } else { - fmt.Println("Mode: continuous") + fmt.Fprintln(wizardOut, "Mode: continuous") } } } - fmt.Println("Format: flowctl/v1") + fmt.Fprintln(wizardOut, "Format: flowctl/v1") - fmt.Println("\nNext:") - fmt.Printf(" flowctl validate %s\n", filename) - fmt.Printf(" flowctl run %s\n", filename) - fmt.Println("\nThen, in another terminal:") - fmt.Println(" flowctl status") - fmt.Println(" flowctl pipelines active") + fmt.Fprintln(wizardOut, "\nNext:") + fmt.Fprintf(wizardOut, " flowctl validate %s\n", filename) + fmt.Fprintf(wizardOut, " flowctl run %s\n", filename) + fmt.Fprintln(wizardOut, "\nThen, in another terminal:") + fmt.Fprintln(wizardOut, " flowctl status") + fmt.Fprintln(wizardOut, " flowctl pipelines active") if destination == "duckdb" { duckdbFile := duckDBPathFromIntent(spec) - fmt.Println("\nOptional, after stopping the pipeline:") - fmt.Printf(" duckdb %s \"SELECT event_type, COUNT(*) FROM contract_events GROUP BY event_type\"\n", duckdbFile) - fmt.Println("\nDuckDB is the recommended first-run path and does not require a local Docker daemon.") + fmt.Fprintln(wizardOut, "\nOptional, after stopping the pipeline:") + fmt.Fprintf(wizardOut, " duckdb %s \"SELECT event_type, COUNT(*) FROM contract_events GROUP BY event_type\"\n", duckdbFile) + fmt.Fprintln(wizardOut, "\nDuckDB is the recommended first-run path and does not require a local Docker daemon.") } else { - fmt.Println("\nNote: ensure PostgreSQL is running and the sink credentials in the pipeline file are correct.") + fmt.Fprintln(wizardOut, "\nNote: ensure PostgreSQL is running and the sink credentials in the pipeline file are correct.") } - fmt.Println("\nComponents are downloaded automatically on first run and cached under ~/.flowctl/.") + fmt.Fprintln(wizardOut, "\nComponents are downloaded automatically on first run and cached under ~/.flowctl/.") } func configValueString(v interface{}) string { diff --git a/cmd/list/list.go b/cmd/list/list.go index 0da9f2b..20fa4d6 100644 --- a/cmd/list/list.go +++ b/cmd/list/list.go @@ -2,10 +2,7 @@ package list import ( "context" - "fmt" - "os" "strings" - "text/tabwriter" "time" "github.com/spf13/cobra" @@ -13,13 +10,15 @@ import ( "google.golang.org/grpc/credentials/insecure" "google.golang.org/protobuf/types/known/emptypb" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" pb "github.com/withobsrvr/flowctl/proto" ) var ( endpoint string waitForServer bool - tableFormat bool + timeout time.Duration ) // formatServiceType converts a ServiceType to a user-friendly string @@ -58,100 +57,93 @@ func NewCommand() *cobra.Command { Use: "list", Short: "List registered services", Long: `List all services registered with the control plane. -If a pipeline is running, it will connect to the embedded control plane automatically.`, +If a pipeline is running, it will connect to the embedded control plane automatically. + +Examples: + # Human-readable table + flowctl list + + # Every unhealthy service, for a monitoring check + flowctl list --output json | jq '.items[] | select(.healthy | not)' + + # Stream records as they are produced + flowctl list --output ndjson`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - // If endpoint is default, check if embedded control plane is running - if endpoint == "localhost:8080" && waitForServer { - fmt.Println("Checking for embedded control plane...") + p := cliout.FromContext(cmd.Context()) + + // Only wait for the embedded control plane when a person is watching. + // A script that asked for JSON wants an answer or an error, not a + // two-second sleep (CLI-8). + if endpoint == "localhost:8080" && waitForServer && !p.Machine() { + p.Info("Checking for embedded control plane...") time.Sleep(2 * time.Second) } - // Connect to control plane - conn, err := grpc.Dial(endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + cliout.UnreachableInterceptor(endpoint), + ) if err != nil { - return fmt.Errorf("failed to connect to control plane at %s: %w\nHint: Make sure 'flowctl run' is running or start 'flowctl server'", endpoint, err) + return exitcode.Errorf(exitcode.Unavailable, + "cannot reach the control plane at %s: %v\nHint: start 'flowctl run' or 'flowctl server' first", endpoint, err) } defer conn.Close() - // Create client client := pb.NewControlPlaneClient(conn) - // Set timeout - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(cmd.Context(), timeout) defer cancel() - // List services resp, err := client.ListServices(ctx, &emptypb.Empty{}) if err != nil { - return fmt.Errorf("failed to list services: %w", err) - } - - // Print services - if len(resp.Services) == 0 { - fmt.Println("No services registered with the control plane.") - return nil + // exitcode.Of maps the gRPC status, so a dead control plane + // exits 7 and a deadline exits 6 without classifying here. + return err } - if tableFormat { - // Table format output - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "ID\tTYPE\tSTATUS\tLAST HEARTBEAT") - fmt.Fprintln(w, "----\t----\t------\t--------------") - - for _, service := range resp.Services { - status := "UNHEALTHY" - if service.IsHealthy { - status = "HEALTHY" - } - - lastHeartbeat := "Never" - if service.LastHeartbeat != nil { - lastHeartbeat = service.LastHeartbeat.AsTime().Format("2006-01-02 15:04:05") - } - - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", - service.ServiceId, - formatServiceType(service.ServiceType), - status, - lastHeartbeat, - ) + rows := make([]cliout.Row, 0, len(resp.Services)) + for _, service := range resp.Services { + row := cliout.Row{ + "id": service.ServiceId, + "type": formatServiceType(service.ServiceType), + "healthy": service.IsHealthy, + } + if service.LastHeartbeat != nil { + row["last_heartbeat"] = service.LastHeartbeat.AsTime() } - w.Flush() - } else { - // Detailed format output - fmt.Printf("Registered Services (%d):\n", len(resp.Services)) - fmt.Println("========================") - for _, service := range resp.Services { - status := "🔴 UNHEALTHY" - if service.IsHealthy { - status = "🟢 HEALTHY" - } - - fmt.Printf("ID: %s\n", service.ServiceId) - fmt.Printf("Type: %s\n", formatServiceType(service.ServiceType)) - fmt.Printf("Status: %s\n", status) - if service.LastHeartbeat != nil { - fmt.Printf("Last Heartbeat: %s\n", service.LastHeartbeat.AsTime().Format(time.RFC3339)) - } - - if len(service.Metrics) > 0 { - fmt.Println("Metrics:") - for k, v := range service.Metrics { - fmt.Printf(" %s: %.2f\n", k, v) - } - } - fmt.Println("------------------------") + if len(service.Metrics) > 0 { + row["metrics"] = service.Metrics } + rows = append(rows, row) } - return nil + return p.List(cliout.List{ + Schema: "flowctl.service_list.v1", + ItemSchema: "flowctl.service.v1", + Columns: []cliout.Column{ + {Header: "ID", Key: "id"}, + {Header: "Type", Key: "type"}, + {Header: "Status", Key: "healthy", Format: health}, + {Header: "Last Heartbeat", Key: "last_heartbeat"}, + }, + Rows: rows, + Empty: "No services registered with the control plane.", + }) }, } - // Add flags cmd.Flags().StringVarP(&endpoint, "endpoint", "e", "localhost:8080", "Control plane endpoint") - cmd.Flags().BoolVar(&waitForServer, "wait", true, "Wait briefly for embedded control plane") - cmd.Flags().BoolVarP(&tableFormat, "table", "t", false, "Display output in table format") + cmd.Flags().BoolVar(&waitForServer, "wait", true, "Wait briefly for an embedded control plane") + cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Second, "Timeout for the control plane call") return cmd } + +// health renders the boolean health field for human output. +func health(v any) string { + if healthy, ok := v.(bool); ok && healthy { + return "HEALTHY" + } + return "UNHEALTHY" +} diff --git a/cmd/pipelines.go b/cmd/pipelines.go index 607e039..73d87a8 100644 --- a/cmd/pipelines.go +++ b/cmd/pipelines.go @@ -3,17 +3,86 @@ package cmd import ( "context" "fmt" - "os" + "sort" "strings" - "text/tabwriter" "time" "github.com/spf13/cobra" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" + "github.com/withobsrvr/flowctl/internal/sanitize" flowctlpb "github.com/withobsrvr/flowctl/proto" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) +// Schema identifiers for this command group's machine-readable output. +const ( + pipelineListSchema = "flowctl.pipeline_list.v1" + pipelineSchema = "flowctl.pipeline.v1" + pipelineRunListSchema = "flowctl.pipeline_run_list.v1" + pipelineRunSchema = "flowctl.pipeline_run.v1" +) + +// runRow converts a pipeline run into a record. Machine output carries full +// ids, raw counts and real timestamps; the table shortens and humanises them. +func runRow(run *flowctlpb.PipelineRun) cliout.Row { + row := cliout.Row{ + "run_id": run.GetRunId(), + "pipeline_name": run.GetPipelineName(), + "status": formatRunStatus(run.GetStatus()), + } + if ts := run.GetStartTime(); ts != nil { + row["started_at"] = ts.AsTime() + row["_started"] = formatRunTime(ts.AsTime()) + } + if ts := run.GetEndTime(); ts != nil { + row["ended_at"] = ts.AsTime() + } + row["_duration"] = formatRunDuration(run) + if m := run.GetMetrics(); m != nil { + row["events_processed"] = m.GetEventsProcessed() + row["_events"] = formatEventCount(m.GetEventsProcessed()) + if m.GetEventsPerSecond() > 0 { + row["events_per_second"] = m.GetEventsPerSecond() + } + } + if ids := run.GetComponentIds(); len(ids) > 0 { + row["component_ids"] = ids + } + if v := run.GetError(); v != "" { + row["error"] = v + } + row["_short_id"] = shortID(run.GetRunId()) + return row +} + +// shortID is the abbreviated run id shown in tables. It slices by rune count +// and tolerates a short id: the control plane is a trust boundary, and +// run.RunId[:8] panics on anything shorter than eight bytes. +func shortID(id string) string { + r := []rune(id) + if len(r) <= 8 { + return id + } + return string(r[:8]) +} + +// runListColumns is the table layout for a list of runs. +func runListColumns(withPipeline bool) []cliout.Column { + cols := []cliout.Column{{Header: "Run ID", Key: "_short_id"}} + if withPipeline { + cols = append(cols, cliout.Column{Header: "Pipeline", Key: "pipeline_name"}) + } + cols = append(cols, + cliout.Column{Header: "Status", Key: "status"}, + cliout.Column{Header: "Started", Key: "_started"}, + cliout.Column{Header: "Duration", Key: "_duration"}, + cliout.Column{Header: "Events", Key: "_events"}, + ) + return cols +} + var ( pipelinesControlPlaneAddr string pipelinesControlPlanePort int @@ -54,46 +123,67 @@ var pipelinesListCmd = &cobra.Command{ } defer conn.Close() - // Get all runs - resp, err := client.ListPipelineRuns(context.Background(), &flowctlpb.ListPipelineRunsRequest{ - Limit: 1000, // Get all runs to group by pipeline + resp, err := client.ListPipelineRuns(cmd.Context(), &flowctlpb.ListPipelineRunsRequest{ + Limit: 1000, // all runs, so they can be grouped by pipeline }) if err != nil { - return fmt.Errorf("failed to list pipeline runs: %w", err) + return err } - // Group by pipeline name - pipelineMap := make(map[string][]*flowctlpb.PipelineRun) + // Group by pipeline name, keeping the most recent run of each. + latest := make(map[string]*flowctlpb.PipelineRun) + counts := make(map[string]int) for _, run := range resp.Runs { - pipelineMap[run.PipelineName] = append(pipelineMap[run.PipelineName], run) + name := run.GetPipelineName() + counts[name]++ + prev, ok := latest[name] + if !ok || run.GetStartTime().AsTime().After(prev.GetStartTime().AsTime()) { + latest[name] = run + } } - if len(pipelineMap) == 0 { - fmt.Println("No pipelines found") - return nil + // Sorted so the output is stable between runs; map iteration is not. + names := make([]string, 0, len(latest)) + for name := range latest { + names = append(names, name) } - - // Print table - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "NAME\tLAST RUN\tSTATUS\tRUNS") - for name, runs := range pipelineMap { - // Find most recent run - var lastRun *flowctlpb.PipelineRun - for _, run := range runs { - if lastRun == nil || run.StartTime.AsTime().After(lastRun.StartTime.AsTime()) { - lastRun = run - } + sort.Strings(names) + + rows := make([]cliout.Row, 0, len(names)) + for _, name := range names { + last := latest[name] + row := cliout.Row{ + "pipeline_name": name, + "run_count": counts[name], + "last_run_id": last.GetRunId(), + "last_status": formatRunStatus(last.GetStatus()), + } + if ts := last.GetStartTime(); ts != nil { + row["last_run_at"] = ts.AsTime() + row["_last_run"] = formatRunTime(ts.AsTime()) } + rows = append(rows, row) + } - fmt.Fprintf(w, "%s\t%s\t%s\t%d\n", - name, - formatRunTime(lastRun.StartTime.AsTime()), - formatRunStatus(lastRun.Status), - len(runs)) + p := cliout.FromContext(cmd.Context()) + if err := p.List(cliout.List{ + Schema: pipelineListSchema, + ItemSchema: pipelineSchema, + Columns: []cliout.Column{ + {Header: "Name", Key: "pipeline_name"}, + {Header: "Last Run", Key: "_last_run"}, + {Header: "Status", Key: "last_status"}, + {Header: "Runs", Key: "run_count"}, + }, + Rows: rows, + Empty: "No pipelines found.", + }); err != nil { + return err } - w.Flush() - fmt.Printf("\n%d pipeline(s) found\n", len(pipelineMap)) + if !p.Machine() && len(rows) > 0 { + p.Info("\n%d pipeline(s) found", len(rows)) + } return nil }, } @@ -112,33 +202,33 @@ var pipelinesRunsCmd = &cobra.Command{ } defer conn.Close() - resp, err := client.ListPipelineRuns(context.Background(), &flowctlpb.ListPipelineRunsRequest{ + resp, err := client.ListPipelineRuns(cmd.Context(), &flowctlpb.ListPipelineRunsRequest{ PipelineName: pipelineName, Limit: pipelinesLimit, }) if err != nil { - return fmt.Errorf("failed to list pipeline runs: %w", err) + return err } - if len(resp.Runs) == 0 { - fmt.Printf("No runs found for pipeline: %s\n", pipelineName) - return nil + rows := make([]cliout.Row, 0, len(resp.Runs)) + for _, run := range resp.Runs { + rows = append(rows, runRow(run)) } - // Print table - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "RUN ID\tSTATUS\tSTARTED\tDURATION\tEVENTS") - for _, run := range resp.Runs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", - run.RunId[:8], - formatRunStatus(run.Status), - formatRunTime(run.StartTime.AsTime()), - formatRunDuration(run), - formatEventCount(run.Metrics.EventsProcessed)) + p := cliout.FromContext(cmd.Context()) + if err := p.List(cliout.List{ + Schema: pipelineRunListSchema, + ItemSchema: pipelineRunSchema, + Columns: runListColumns(false), + Rows: rows, + Empty: fmt.Sprintf("No runs found for pipeline %s.", sanitize.Line(pipelineName)), + }); err != nil { + return err } - w.Flush() - fmt.Printf("\nShowing last %d run(s). Use --limit to see more.\n", len(resp.Runs)) + if !p.Machine() && len(rows) > 0 { + p.Info("\nShowing last %d run(s). Use --limit to see more.", len(rows)) + } return nil }, } @@ -167,36 +257,45 @@ var pipelinesRunInfoCmd = &cobra.Command{ RunId: runID, }) if err != nil { - return fmt.Errorf("failed to get pipeline run: %w", err) + return exitcode.Errorf(exitcode.NotFound, "pipeline run %q not found: %v", runID, err) } - // Print run details - fmt.Printf("Run ID: %s\n", run.RunId) - fmt.Printf("Pipeline: %s\n", run.PipelineName) - fmt.Printf("Status: %s\n", formatRunStatus(run.Status)) - fmt.Printf("Started: %s\n", run.StartTime.AsTime().Format("2006-01-02 15:04:05 MST")) - fmt.Printf("Duration: %s\n", formatRunDuration(run)) - fmt.Printf("Events: %s\n", formatEventCount(run.Metrics.EventsProcessed)) - - if run.Metrics.EventsPerSecond > 0 { - fmt.Printf("Throughput: %.1f events/sec\n", run.Metrics.EventsPerSecond) + row := runRow(run) + if m := run.GetMetrics(); m != nil && m.GetEventsPerSecond() > 0 { + row["_throughput"] = fmt.Sprintf("%.1f events/sec", m.GetEventsPerSecond()) } - if len(run.ComponentIds) > 0 { - fmt.Printf("\nComponents:\n") - for _, compID := range run.ComponentIds { - fmt.Printf(" - %s\n", compID) - } - } - - if run.Error != "" { - fmt.Printf("\nError: %s\n", run.Error) - } - - return nil + return cliout.FromContext(cmd.Context()).Object(cliout.Object{ + Schema: pipelineRunSchema, + Fields: []cliout.Field{ + {Label: "Run ID", Key: "run_id"}, + {Label: "Pipeline", Key: "pipeline_name"}, + {Label: "Status", Key: "status"}, + {Label: "Started", Key: "started_at"}, + {Label: "Duration", Key: "_duration"}, + {Label: "Events", Key: "_events"}, + {Label: "Throughput", Key: "_throughput"}, + {Label: "Components", Key: "component_ids", Format: joinList}, + {Label: "Error", Key: "error"}, + }, + Data: row, + }) }, } +// joinList renders a string slice for a detail view. The record keeps the slice. +func joinList(v any) string { + items, ok := v.([]string) + if !ok || len(items) == 0 { + return "-" + } + out := make([]string, len(items)) + for i, s := range items { + out[i] = cliout.Value(s) + } + return strings.Join(out, ", ") +} + // pipelinesStopCmd stops a running pipeline var pipelinesStopCmd = &cobra.Command{ Use: "stop ", @@ -217,17 +316,25 @@ var pipelinesStopCmd = &cobra.Command{ return err } - fmt.Printf("Stopping pipeline run %s...\n", runID) + p := cliout.FromContext(cmd.Context()) + p.Info("Stopping pipeline run %s...", runID) run, err := client.StopPipelineRun(ctx, &flowctlpb.StopPipelineRunRequest{ RunId: runID, }) if err != nil { - return fmt.Errorf("failed to stop pipeline run: %w", err) + return err } - fmt.Printf("✓ Pipeline stopped (status: %s)\n", formatRunStatus(run.Status)) - return nil + return p.Object(cliout.Object{ + Schema: pipelineRunSchema, + Fields: []cliout.Field{ + {Label: "Run ID", Key: "run_id"}, + {Label: "Pipeline", Key: "pipeline_name"}, + {Label: "Status", Key: "status"}, + }, + Data: runRow(run), + }) }, } @@ -242,35 +349,26 @@ var pipelinesActiveCmd = &cobra.Command{ } defer conn.Close() - resp, err := client.ListPipelineRuns(context.Background(), &flowctlpb.ListPipelineRunsRequest{ + resp, err := client.ListPipelineRuns(cmd.Context(), &flowctlpb.ListPipelineRunsRequest{ Status: flowctlpb.RunStatus_RUN_STATUS_RUNNING, Limit: 100, }) if err != nil { - return fmt.Errorf("failed to list active pipeline runs: %w", err) - } - - if len(resp.Runs) == 0 { - fmt.Println("No active pipeline runs") - return nil + return err } - // Print table - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "RUN ID\tPIPELINE\tSTARTED\tDURATION\tEVENTS\tSTATUS") + rows := make([]cliout.Row, 0, len(resp.Runs)) for _, run := range resp.Runs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", - run.RunId[:8], - run.PipelineName, - formatRunTime(run.StartTime.AsTime()), - formatRunDuration(run), - formatEventCount(run.Metrics.EventsProcessed), - formatRunStatus(run.Status)) + rows = append(rows, runRow(run)) } - w.Flush() - fmt.Printf("\n%d active pipeline run(s)\n", len(resp.Runs)) - return nil + return cliout.FromContext(cmd.Context()).List(cliout.List{ + Schema: pipelineRunListSchema, + ItemSchema: pipelineRunSchema, + Columns: runListColumns(true), + Rows: rows, + Empty: "No active pipeline runs.", + }) }, } @@ -279,12 +377,12 @@ var pipelinesActiveCmd = &cobra.Command{ func connectToControlPlane() (flowctlpb.ControlPlaneClient, *grpc.ClientConn, error) { endpoint := fmt.Sprintf("%s:%d", pipelinesControlPlaneAddr, pipelinesControlPlanePort) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - - conn, err := grpc.DialContext(ctx, endpoint, + conn, err := grpc.NewClient(endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), + // grpc.NewClient connects lazily, so an unreachable control plane + // surfaces on the first RPC rather than here. The interceptor restores + // the actionable message that WithBlock used to produce at dial time. + cliout.UnreachableInterceptor(endpoint), ) if err != nil { return nil, nil, formatPipelinesControlPlaneError(endpoint, err) @@ -311,7 +409,7 @@ func formatRunTime(t time.Time) string { } func formatRunDuration(run *flowctlpb.PipelineRun) string { - start := run.StartTime.AsTime() + start := run.GetStartTime().AsTime() var end time.Time if run.EndTime != nil { @@ -367,7 +465,9 @@ func resolveRunID(ctx context.Context, client flowctlpb.ControlPlaneClient, runI // Fast path: exact lookup succeeds. run, err := client.GetPipelineRun(ctx, &flowctlpb.GetPipelineRunRequest{RunId: runID}) if err == nil { - return run.RunId, nil + // The id comes back over the wire and is echoed into later output, so + // it is sanitised like any other control-plane string (CLI-22). + return sanitize.Line(run.RunId), nil } // Fallback: allow a unique prefix so operators can use the short IDs shown in tables. @@ -376,6 +476,12 @@ func resolveRunID(ctx context.Context, client flowctlpb.ControlPlaneClient, runI return "", fmt.Errorf("failed to resolve run id %q: %w", runID, listErr) } + // Everything below is written by components flowctl did not start and is + // printed to a terminal, so strip control characters first (CLI-22). + // Remove this when the command is converted to internal/cliout, which + // sanitises at the point it formats a cell. + resp = sanitize.Message(resp) + matches := make([]string, 0) for _, candidate := range resp.Runs { if strings.HasPrefix(candidate.RunId, runID) { diff --git a/cmd/processors/find.go b/cmd/processors/find.go index 67d3056..afbd83a 100644 --- a/cmd/processors/find.go +++ b/cmd/processors/find.go @@ -2,9 +2,7 @@ package processors import ( "context" - "fmt" - "os" - "text/tabwriter" + "strings" "time" "github.com/spf13/cobra" @@ -12,6 +10,8 @@ import ( "google.golang.org/grpc/credentials/insecure" flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" ) var ( @@ -36,7 +36,7 @@ This is the foundation for automatic topology discovery.`, flowctl processors find --input stellar.ledger.v1 # Find processors that produce token.transfer events - flowctl processors find --output stellar.token.transfer.v1 + flowctl processors find --output-type stellar.token.transfer.v1 # Find processors for a specific network flowctl processors find --metadata network=testnet @@ -47,25 +47,33 @@ This is the foundation for automatic topology discovery.`, } cmd.Flags().StringVar(&inputType, "input", "", "Filter by input event type") - cmd.Flags().StringVar(&outputType, "output", "", "Filter by output event type") + // Renamed from --output, which CLI-3 reserves for selecting an output format. + // No deprecated alias: a local --output would shadow the global one, making + // `--output json` filter by the event type "json" instead of emitting JSON. + cmd.Flags().StringVar(&outputType, "output-type", "", "Filter by output event type") cmd.Flags().StringToStringVar(&metadata, "metadata", nil, "Filter by metadata (key=value)") return cmd } func runFind(cmd *cobra.Command, args []string) error { - // Validate that at least one filter is provided + p := cliout.FromContext(cmd.Context()) + if inputType == "" && outputType == "" && len(metadata) == 0 { - return fmt.Errorf("at least one filter must be specified (--input, --output, or --metadata)") + return exitcode.Errorf(exitcode.Usage, + "at least one filter must be specified (--input, --output-type, or --metadata)") } - // Connect to control plane - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second) defer cancel() - conn, err := grpc.DialContext(ctx, endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + cliout.UnreachableInterceptor(endpoint), + ) if err != nil { - return fmt.Errorf("failed to connect to control plane at %s: %w", endpoint, err) + return exitcode.Errorf(exitcode.Unavailable, + "cannot reach the control plane at %s: %v", endpoint, err) } defer conn.Close() @@ -88,69 +96,55 @@ func runFind(cmd *cobra.Command, args []string) error { } req.EventTypes = eventTypes - // Execute discovery resp, err := client.DiscoverComponents(ctx, req) if err != nil { - return fmt.Errorf("discovery failed: %w", err) + return err } - // Filter results based on input/output specificity - // (DiscoverComponents does OR logic on event types, we need to be more specific) - var matches []*flowctlv1.ComponentInfo + // No sanitize.Message here: cliout strips control characters as it formats + // each cell, and machine output must keep the original bytes (CLI-22). + + // DiscoverComponents ORs the event-type filters, so narrow to components + // matching every filter the user gave. + rows := make([]cliout.Row, 0, len(resp.Components)) for _, comp := range resp.Components { if matchesFilters(comp, inputType, outputType) { - matches = append(matches, comp) + rows = append(rows, componentInfoRow(comp)) } } - // Print results - if len(matches) == 0 { - fmt.Println("No processors found matching the specified criteria.") - fmt.Println() - fmt.Println("Search criteria:") - if inputType != "" { - fmt.Printf(" Input type: %s\n", inputType) - } - if outputType != "" { - fmt.Printf(" Output type: %s\n", outputType) - } - if len(metadata) > 0 { - fmt.Println(" Metadata:") - for k, v := range metadata { - fmt.Printf(" %s: %s\n", k, v) - } - } - return nil + if err := p.List(cliout.List{ + Schema: processorListSchema, + ItemSchema: processorSchema, + Columns: discoveryColumns(), + Rows: rows, + Empty: "No processors found matching " + describeFilters() + ".", + }); err != nil { + return err } - // Print table header - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ID\tINPUT TYPES\tOUTPUT TYPES\tENDPOINT") - fmt.Fprintln(w, "──\t───────────\t────────────\t────────") - - for _, comp := range matches { - inputTypes := formatEventTypes(comp.InputEventTypes) - outputTypes := formatEventTypes(comp.OutputEventTypes) - - endpoint := comp.Endpoint - if endpoint == "" { - endpoint = "-" - } - - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", - comp.Id, - inputTypes, - outputTypes, - endpoint, - ) + if !p.Machine() && len(rows) > 0 { + p.Info("\nFound: %d processor(s)", len(rows)) } - - w.Flush() - fmt.Printf("\nFound: %d processor(s)\n", len(matches)) - return nil } +// describeFilters renders the active search criteria for the empty-result +// message, so a person can see what was actually asked for. +func describeFilters() string { + var parts []string + if inputType != "" { + parts = append(parts, "input type "+inputType) + } + if outputType != "" { + parts = append(parts, "output type "+outputType) + } + for k, v := range metadata { + parts = append(parts, "metadata "+k+"="+v) + } + return strings.Join(parts, ", ") +} + // matchesFilters checks if a component matches the input/output type filters func matchesFilters(comp *flowctlv1.ComponentInfo, inputType, outputType string) bool { // Check input type diff --git a/cmd/processors/list.go b/cmd/processors/list.go index 09b2502..766409e 100644 --- a/cmd/processors/list.go +++ b/cmd/processors/list.go @@ -2,10 +2,6 @@ package processors import ( "context" - "fmt" - "os" - "strings" - "text/tabwriter" "time" "github.com/spf13/cobra" @@ -13,6 +9,8 @@ import ( "google.golang.org/grpc/credentials/insecure" flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" ) func newListCommand() *cobra.Command { @@ -30,123 +28,72 @@ This command shows their input types, output types, health status, and endpoints flowctl processors list --include-unhealthy # List processors on a different control plane - flowctl processors list --endpoint remote-host:8080`, + flowctl processors list --endpoint remote-host:8080 + + # Every processor that accepts ledgers, as JSON + flowctl processors list --output json | + jq '.items[] | select(.input_event_types[] == "stellar.ledger.v1")'`, + Args: cobra.NoArgs, RunE: runList, } } func runList(cmd *cobra.Command, args []string) error { - // Connect to control plane - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + p := cliout.FromContext(cmd.Context()) + + ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second) defer cancel() - conn, err := grpc.DialContext(ctx, endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + cliout.UnreachableInterceptor(endpoint), + ) if err != nil { - return fmt.Errorf("failed to connect to control plane at %s: %w\nHint: Make sure 'flowctl run' is running or the control plane is available", endpoint, err) + return exitcode.Errorf(exitcode.Unavailable, + "cannot reach the control plane at %s: %v\nHint: start 'flowctl run' or 'flowctl server' first", endpoint, err) } defer conn.Close() - // Create client client := flowctlv1.NewControlPlaneServiceClient(conn) - // List components, filtering for processors only resp, err := client.ListComponents(ctx, &flowctlv1.ListComponentsRequest{ TypeFilter: flowctlv1.ComponentType_COMPONENT_TYPE_PROCESSOR, IncludeUnhealthy: includeUnhealthy, }) if err != nil { - return fmt.Errorf("failed to list processors: %w", err) - } - - // Check if any processors found - if len(resp.Components) == 0 { - if includeUnhealthy { - fmt.Println("No processors registered with the control plane.") - } else { - fmt.Println("No healthy processors registered with the control plane.") - fmt.Println("Hint: Use --include-unhealthy to see unhealthy processors") - } - return nil + return err } - // Print table - w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) - fmt.Fprintln(w, "ID\tINPUT TYPES\tOUTPUT TYPES\tSTATUS\tENDPOINT") - fmt.Fprintln(w, "──\t───────────\t────────────\t──────\t────────") + // No sanitize.Message here: cliout strips control characters as it formats + // each cell, and machine output must keep the original bytes (CLI-22). + rows := make([]cliout.Row, 0, len(resp.Components)) for _, comp := range resp.Components { - if comp.Component == nil { + if comp.GetComponent() == nil { continue } - - // Format input types - inputTypes := formatEventTypes(comp.Component.InputEventTypes) - outputTypes := formatEventTypes(comp.Component.OutputEventTypes) - - // Format status - status := formatHealthStatus(comp.Status) - - // Get endpoint - endpoint := comp.Component.Endpoint - if endpoint == "" { - endpoint = "-" - } - - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", - comp.Component.Id, - inputTypes, - outputTypes, - status, - endpoint, - ) + rows = append(rows, componentRow(comp)) } - w.Flush() - fmt.Printf("\nTotal: %d processor(s)\n", len(resp.Components)) - - return nil -} - -// formatEventTypes formats a list of event types for display -func formatEventTypes(types []string) string { - if len(types) == 0 { - return "-" - } - if len(types) == 1 { - return types[0] + empty := "No healthy processors registered with the control plane.\nHint: use --include-unhealthy to see unhealthy processors." + if includeUnhealthy { + empty = "No processors registered with the control plane." } - // Show first type + count if multiple - return fmt.Sprintf("%s (+%d)", types[0], len(types)-1) -} -// formatHealthStatus formats health status with color indicators -func formatHealthStatus(status flowctlv1.HealthStatus) string { - switch status { - case flowctlv1.HealthStatus_HEALTH_STATUS_HEALTHY: - return "✓ healthy" - case flowctlv1.HealthStatus_HEALTH_STATUS_DEGRADED: - return "⚠ degraded" - case flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY: - return "✗ unhealthy" - case flowctlv1.HealthStatus_HEALTH_STATUS_STARTING: - return "⟳ starting" - case flowctlv1.HealthStatus_HEALTH_STATUS_STOPPING: - return "⊗ stopping" - default: - return "? unknown" + if err := p.List(cliout.List{ + Schema: processorListSchema, + ItemSchema: processorSchema, + Columns: processorColumns(), + Rows: rows, + Empty: empty, + }); err != nil { + return err } -} -// truncateString truncates a string to maxLen with ellipsis -func truncateString(s string, maxLen int) string { - if len(s) <= maxLen { - return s + // A count is a convenience for a person reading a terminal, not a result, + // so it goes to stderr and is omitted when a program is reading (CLI-5). + if !p.Machine() && len(rows) > 0 { + p.Info("\nTotal: %d processor(s)", len(rows)) } - return s[:maxLen-3] + "..." -} - -// joinWithLimit joins strings with separator, truncating if needed -func joinWithLimit(items []string, sep string, maxLen int) string { - result := strings.Join(items, sep) - return truncateString(result, maxLen) + return nil } diff --git a/cmd/processors/processors.go b/cmd/processors/processors.go index 357773e..f2b89d9 100644 --- a/cmd/processors/processors.go +++ b/cmd/processors/processors.go @@ -5,7 +5,7 @@ import ( ) var ( - endpoint string + endpoint string includeUnhealthy bool ) diff --git a/cmd/processors/processors_test.go b/cmd/processors/processors_test.go index 95be637..5337857 100644 --- a/cmd/processors/processors_test.go +++ b/cmd/processors/processors_test.go @@ -6,82 +6,57 @@ import ( flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" ) -func TestFormatEventTypes(t *testing.T) { +func TestSummariseTypes(t *testing.T) { tests := []struct { name string - types []string + types any expected string }{ + {"nil", nil, "-"}, + {"empty", []string{}, "-"}, + {"single type", []string{"stellar.ledger.v1"}, "stellar.ledger.v1"}, { - name: "empty", - types: []string{}, - expected: "-", - }, - { - name: "single type", - types: []string{"stellar.ledger.v1"}, - expected: "stellar.ledger.v1", - }, - { - name: "multiple types", + name: "multiple types are condensed for the table", types: []string{"stellar.ledger.v1", "stellar.transaction.v1", "stellar.event.v1"}, expected: "stellar.ledger.v1 (+2)", }, + { + // CLI-22: a type string comes from a registered component. + name: "control characters are stripped", + types: []string{"evil\x1b[2K.v1"}, + expected: "evil[2K.v1", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := formatEventTypes(tt.types) - if result != tt.expected { - t.Errorf("formatEventTypes() = %v, want %v", result, tt.expected) + if got := summariseTypes(tt.types); got != tt.expected { + t.Errorf("summariseTypes() = %v, want %v", got, tt.expected) } }) } } -func TestFormatHealthStatus(t *testing.T) { +func TestHealthName(t *testing.T) { + // These strings appear in --output json and are therefore a contract: + // changing one is a breaking change under VER-3. tests := []struct { name string status flowctlv1.HealthStatus expected string }{ - { - name: "healthy", - status: flowctlv1.HealthStatus_HEALTH_STATUS_HEALTHY, - expected: "✓ healthy", - }, - { - name: "unhealthy", - status: flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY, - expected: "✗ unhealthy", - }, - { - name: "degraded", - status: flowctlv1.HealthStatus_HEALTH_STATUS_DEGRADED, - expected: "⚠ degraded", - }, - { - name: "starting", - status: flowctlv1.HealthStatus_HEALTH_STATUS_STARTING, - expected: "⟳ starting", - }, - { - name: "stopping", - status: flowctlv1.HealthStatus_HEALTH_STATUS_STOPPING, - expected: "⊗ stopping", - }, - { - name: "unknown", - status: flowctlv1.HealthStatus_HEALTH_STATUS_UNKNOWN, - expected: "? unknown", - }, + {"healthy", flowctlv1.HealthStatus_HEALTH_STATUS_HEALTHY, "healthy"}, + {"unhealthy", flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY, "unhealthy"}, + {"degraded", flowctlv1.HealthStatus_HEALTH_STATUS_DEGRADED, "degraded"}, + {"starting", flowctlv1.HealthStatus_HEALTH_STATUS_STARTING, "starting"}, + {"stopping", flowctlv1.HealthStatus_HEALTH_STATUS_STOPPING, "stopping"}, + {"unknown", flowctlv1.HealthStatus_HEALTH_STATUS_UNKNOWN, "unknown"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := formatHealthStatus(tt.status) - if result != tt.expected { - t.Errorf("formatHealthStatus() = %v, want %v", result, tt.expected) + if got := healthName(tt.status); got != tt.expected { + t.Errorf("healthName() = %v, want %v", got, tt.expected) } }) } @@ -166,40 +141,3 @@ func TestMatchesFilters(t *testing.T) { }) } } - -func TestTruncateString(t *testing.T) { - tests := []struct { - name string - input string - maxLen int - expected string - }{ - { - name: "no truncation needed", - input: "short", - maxLen: 10, - expected: "short", - }, - { - name: "truncation needed", - input: "this is a very long string", - maxLen: 10, - expected: "this is...", - }, - { - name: "exact length", - input: "exactly10!", - maxLen: 10, - expected: "exactly10!", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := truncateString(tt.input, tt.maxLen) - if result != tt.expected { - t.Errorf("truncateString() = %v, want %v", result, tt.expected) - } - }) - } -} diff --git a/cmd/processors/render.go b/cmd/processors/render.go new file mode 100644 index 0000000..acb0bf8 --- /dev/null +++ b/cmd/processors/render.go @@ -0,0 +1,135 @@ +package processors + +import ( + "fmt" + "strings" + + flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" + "github.com/withobsrvr/flowctl/internal/cliout" +) + +// Schema identifiers for this command group's machine-readable output. These +// are contracts: changing a shape is a breaking change under VER-3. +const ( + processorListSchema = "flowctl.processor_list.v1" + processorSchema = "flowctl.processor.v1" +) + +// componentRow converts a component into a record. +// +// Machine output carries the full values — every event type, the status as a +// stable lowercase string — while the table columns below condense them. The +// table is not a contract and may be shortened freely; the record may not. +func componentRow(comp *flowctlv1.ComponentStatusResponse) cliout.Row { + info := comp.GetComponent() + row := cliout.Row{ + "id": info.GetId(), + "type": componentTypeName(info.GetType()), + "status": healthName(comp.GetStatus()), + "healthy": comp.GetStatus() == flowctlv1.HealthStatus_HEALTH_STATUS_HEALTHY, + "input_event_types": info.GetInputEventTypes(), + "output_event_types": info.GetOutputEventTypes(), + } + if v := info.GetName(); v != "" { + row["name"] = v + } + if v := info.GetVersion(); v != "" { + row["version"] = v + } + if v := info.GetDescription(); v != "" { + row["description"] = v + } + if v := info.GetEndpoint(); v != "" { + row["endpoint"] = v + } + if len(info.GetMetadata()) > 0 { + row["metadata"] = info.GetMetadata() + } + if ts := comp.GetLastHeartbeat(); ts != nil { + row["last_heartbeat"] = ts.AsTime() + } + if ts := comp.GetRegisteredAt(); ts != nil { + row["registered_at"] = ts.AsTime() + } + return row +} + +// processorColumns is the table layout shared by `list` and `find`. +func processorColumns() []cliout.Column { + return []cliout.Column{ + {Header: "ID", Key: "id"}, + {Header: "Input Types", Key: "input_event_types", Format: summariseTypes}, + {Header: "Output Types", Key: "output_event_types", Format: summariseTypes}, + {Header: "Status", Key: "status"}, + {Header: "Endpoint", Key: "endpoint"}, + } +} + +// summariseTypes condenses an event-type list for a table cell. The record +// keeps the full list. +func summariseTypes(v any) string { + types, ok := v.([]string) + if !ok || len(types) == 0 { + return "-" + } + if len(types) == 1 { + return cliout.Value(types[0]) + } + return fmt.Sprintf("%s (+%d)", cliout.Value(types[0]), len(types)-1) +} + +// healthName renders a health status as a stable lowercase string. Machine +// consumers match on this, so the values are part of the schema. +func healthName(status flowctlv1.HealthStatus) string { + name := strings.TrimPrefix(status.String(), "HEALTH_STATUS_") + if name == "" || name == "UNSPECIFIED" { + return "unknown" + } + return strings.ToLower(name) +} + +// componentTypeName renders a component type as a stable lowercase string. +func componentTypeName(t flowctlv1.ComponentType) string { + name := strings.TrimPrefix(t.String(), "COMPONENT_TYPE_") + if name == "" || name == "UNSPECIFIED" { + return "unknown" + } + return strings.ToLower(name) +} + +// componentInfoRow is componentRow for a bare ComponentInfo, which is what +// DiscoverComponents returns — it carries no health status. +func componentInfoRow(info *flowctlv1.ComponentInfo) cliout.Row { + row := cliout.Row{ + "id": info.GetId(), + "type": componentTypeName(info.GetType()), + "input_event_types": info.GetInputEventTypes(), + "output_event_types": info.GetOutputEventTypes(), + } + if v := info.GetName(); v != "" { + row["name"] = v + } + if v := info.GetVersion(); v != "" { + row["version"] = v + } + if v := info.GetDescription(); v != "" { + row["description"] = v + } + if v := info.GetEndpoint(); v != "" { + row["endpoint"] = v + } + if len(info.GetMetadata()) > 0 { + row["metadata"] = info.GetMetadata() + } + return row +} + +// discoveryColumns is the table layout for `find`, which has no health column. +func discoveryColumns() []cliout.Column { + return []cliout.Column{ + {Header: "ID", Key: "id"}, + {Header: "Input Types", Key: "input_event_types", Format: summariseTypes}, + {Header: "Output Types", Key: "output_event_types", Format: summariseTypes}, + {Header: "Endpoint", Key: "endpoint"}, + } +} diff --git a/cmd/processors/show.go b/cmd/processors/show.go index 1f0a5fb..2c451f9 100644 --- a/cmd/processors/show.go +++ b/cmd/processors/show.go @@ -2,7 +2,8 @@ package processors import ( "context" - "fmt" + "sort" + "strings" "time" "github.com/spf13/cobra" @@ -10,6 +11,8 @@ import ( "google.golang.org/grpc/credentials/insecure" flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" ) func newShowCommand() *cobra.Command { @@ -35,125 +38,101 @@ This command displays full details including: } func runShow(cmd *cobra.Command, args []string) error { + p := cliout.FromContext(cmd.Context()) processorID := args[0] - // Connect to control plane - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(cmd.Context(), 5*time.Second) defer cancel() - conn, err := grpc.DialContext(ctx, endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + cliout.UnreachableInterceptor(endpoint), + ) if err != nil { - return fmt.Errorf("failed to connect to control plane at %s: %w", endpoint, err) + return exitcode.Errorf(exitcode.Unavailable, + "cannot reach the control plane at %s: %v", endpoint, err) } defer conn.Close() - // Create client client := flowctlv1.NewControlPlaneServiceClient(conn) - // Get component status resp, err := client.GetComponentStatus(ctx, &flowctlv1.ComponentStatusRequest{ ServiceId: processorID, }) if err != nil { - return fmt.Errorf("processor '%s' not found: %w\nHint: Use 'flowctl processors list' to see available processors", processorID, err) + return exitcode.Errorf(exitcode.NotFound, + "processor %q not found: %v\nHint: run 'flowctl processors list' to see what is registered", processorID, err) } - // Verify it's a processor - if resp.Component == nil { - return fmt.Errorf("component '%s' has no information available", processorID) - } + // No sanitize.Message here: cliout strips control characters as it formats + // each cell, and machine output must keep the original bytes (CLI-22). + if resp.GetComponent() == nil { + return exitcode.Errorf(exitcode.NotFound, + "component %q has no information available", processorID) + } if resp.Component.Type != flowctlv1.ComponentType_COMPONENT_TYPE_PROCESSOR { - return fmt.Errorf("component '%s' is not a processor (type: %s)", processorID, resp.Component.Type) + return exitcode.Errorf(exitcode.Conflict, + "component %q is not a processor (type: %s)", processorID, componentTypeName(resp.Component.Type)) } - // Print detailed information - printProcessorDetails(resp) + row := componentRow(resp) + if len(resp.GetMetrics()) > 0 { + row["metrics"] = resp.GetMetrics() + } + if ts := resp.GetLastHeartbeat(); ts != nil { + row["last_heartbeat_age"] = time.Since(ts.AsTime()).Round(time.Second).String() + } - return nil + return p.Object(cliout.Object{ + Schema: processorSchema, + Fields: []cliout.Field{ + {Label: "Processor", Key: "id"}, + {Label: "Name", Key: "name"}, + {Label: "Version", Key: "version"}, + {Label: "Description", Key: "description"}, + {Label: "Input types", Key: "input_event_types", Format: listLines}, + {Label: "Output types", Key: "output_event_types", Format: listLines}, + {Label: "Endpoint", Key: "endpoint"}, + {Label: "Status", Key: "status"}, + {Label: "Last heartbeat", Key: "last_heartbeat"}, + {Label: "Registered at", Key: "registered_at"}, + {Label: "Metrics", Key: "metrics", Format: mapLines}, + {Label: "Metadata", Key: "metadata", Format: mapLines}, + }, + Data: row, + }) } -func printProcessorDetails(status *flowctlv1.ComponentStatusResponse) { - comp := status.Component - - fmt.Printf("Processor: %s\n", comp.Id) - fmt.Println("═══════════════════════════════════════════════════════") - fmt.Println() - - // Basic info - fmt.Println("Basic Information:") - if comp.Name != "" { - fmt.Printf(" Name: %s\n", comp.Name) +// listLines renders a string slice for a detail view: all values, comma +// separated. The record keeps the slice itself. +func listLines(v any) string { + items, ok := v.([]string) + if !ok || len(items) == 0 { + return "-" } - if comp.Version != "" { - fmt.Printf(" Version: %s\n", comp.Version) - } - if comp.Description != "" { - fmt.Printf(" Description: %s\n", comp.Description) - } - fmt.Println() - - // Event types - fmt.Println("Event Types:") - fmt.Printf(" Input Types: ") - if len(comp.InputEventTypes) == 0 { - fmt.Println("(none)") - } else { - fmt.Println() - for _, t := range comp.InputEventTypes { - fmt.Printf(" - %s\n", t) - } + out := make([]string, len(items)) + for i, s := range items { + out[i] = cliout.Value(s) } + return strings.Join(out, ", ") +} - fmt.Printf(" Output Types: ") - if len(comp.OutputEventTypes) == 0 { - fmt.Println("(none)") - } else { - fmt.Println() - for _, t := range comp.OutputEventTypes { - fmt.Printf(" - %s\n", t) - } - } - fmt.Println() - - // Connection info - fmt.Println("Connection:") - if comp.Endpoint != "" { - fmt.Printf(" Endpoint: %s\n", comp.Endpoint) - } else { - fmt.Printf(" Endpoint: (not specified)\n") +// mapLines renders a string map for a detail view, with keys sorted so the +// output is stable between runs. +func mapLines(v any) string { + m, ok := v.(map[string]string) + if !ok || len(m) == 0 { + return "-" } - fmt.Println() - - // Health status - fmt.Println("Health:") - fmt.Printf(" Status: %s\n", formatHealthStatus(status.Status)) - if status.LastHeartbeat != nil { - lastSeen := status.LastHeartbeat.AsTime() - fmt.Printf(" Last Heartbeat: %s (%s ago)\n", - lastSeen.Format("2006-01-02 15:04:05"), - time.Since(lastSeen).Round(time.Second)) + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) } - if status.RegisteredAt != nil { - fmt.Printf(" Registered At: %s\n", status.RegisteredAt.AsTime().Format("2006-01-02 15:04:05")) - } - fmt.Println() - - // Metrics - if len(status.Metrics) > 0 { - fmt.Println("Metrics:") - for key, value := range status.Metrics { - fmt.Printf(" %s: %s\n", key, value) - } - fmt.Println() - } - - // Metadata - if len(comp.Metadata) > 0 { - fmt.Println("Metadata:") - for key, value := range comp.Metadata { - fmt.Printf(" %s: %s\n", key, value) - } - fmt.Println() + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, cliout.Value(k)+"="+cliout.Value(m[k])) } + return strings.Join(parts, ", ") } diff --git a/cmd/root.go b/cmd/root.go index 8677563..86e1df3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,6 +3,8 @@ package cmd import ( "fmt" "os" + "path/filepath" + "strings" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -12,10 +14,17 @@ import ( "github.com/withobsrvr/flowctl/cmd/server" "github.com/withobsrvr/flowctl/cmd/translate" "github.com/withobsrvr/flowctl/cmd/version" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" "github.com/withobsrvr/flowctl/internal/utils/logger" "go.uber.org/zap" ) +// envPrefix namespaces flowctl's environment variables. Without it viper's +// AutomaticEnv binds bare names like OUTPUT and CONTEXT, which collide with unrelated +// variables in any normal shell (CLI-9). +const envPrefix = "FLOWCTL" + var ( cfgFile string ctxName string @@ -27,41 +36,81 @@ var ( // rootCmd represents the base command when called without any subcommands var rootCmd = &cobra.Command{ - Use: "flowctl", - Short: "Control plane for data pipelines for the Stellar Blockchain", + Use: "flowctl", + Short: "Control plane for data pipelines for the Stellar Blockchain", Long: `flowctl is a command-line tool for managing data pipelines for the Stellar Blockchain. It provides a unified interface for developers, operators, and CI/CD jobs to interact with a Flow-powered stack.`, SilenceErrors: true, SilenceUsage: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return setup(cmd) + }, } -// Execute adds all child commands to the root command and sets flags appropriately. +// Execute runs the root command and exits with the code its error maps to. +// +// Exit codes are a public contract (CLI-15); see internal/exitcode. Errors are +// reported on stderr so that stdout carries results only (CLI-5). func Execute() { defer logger.Sync() if err := rootCmd.Execute(); err != nil { - fmt.Fprintln(os.Stderr, "Error:", err) - os.Exit(1) + // A clean shutdown on SIGINT or SIGTERM exits non-zero so a supervisor + // can tell it apart from completion, but it is not a failure and gets + // no error message (CLI-11). + if !exitcode.IsSignal(err) { + fmt.Fprintln(os.Stderr, "Error:", err) + } + os.Exit(exitcode.Of(err)) + } +} + +// setup resolves configuration and installs the output printer before any command +// runs. It replaces cobra.OnInitialize, which cannot report an error and so had to +// call os.Exit directly, bypassing the exit-code contract. +func setup(cmd *cobra.Command) error { + if err := initConfig(); err != nil { + return err + } + + format, err := cliout.ParseFormat(output) + if err != nil { + return err + } + cmd.SetContext(cliout.NewContext(cmd.Context(), + cliout.New(format, cmd.OutOrStdout(), cmd.ErrOrStderr()))) + + // Structured logs are diagnostics, not results, so they must not interleave with + // machine-readable output on a pipe. They already go to stderr; in a machine + // format we additionally drop below WARN unless the user asked for more, so that + // `flowctl … --output json 2>/dev/null` is quiet without losing real problems. + level := logLevel + if format != cliout.FormatTable && !cmd.Flags().Changed("log-level") { + level = "warn" } + return logger.Init(level, logger.WithColor(cliout.ShouldColor(os.Stderr))) } func init() { - cobra.OnInitialize(initConfig) + // Flag parsing failures are usage errors, not generic failures (CLI-15). + rootCmd.SetFlagErrorFunc(func(c *cobra.Command, err error) error { + return exitcode.Wrap(exitcode.Usage, err) + }) // Global flags - rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.config/flowctl/flowctl.yaml)") + rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $XDG_CONFIG_HOME/flowctl/flowctl.yaml)") rootCmd.PersistentFlags().StringVar(&ctxName, "context", "", "override current context") rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", "", "target Kubernetes namespace / logical tenant") - rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "table", "output format (table|yaml|json|wide)") + rootCmd.PersistentFlags().StringVarP(&output, "output", "o", "table", "output format (table|json|ndjson)") rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "log level (debug|info|warn|error)") - rootCmd.PersistentFlags().BoolVarP(&nonInteractive, "yes", "y", false, "non-interactive confirmations") + rootCmd.PersistentFlags().BoolVarP(&nonInteractive, "yes", "y", false, "assume yes for confirmations; never prompt") // Bind flags to viper - viper.BindPFlag("context", rootCmd.PersistentFlags().Lookup("context")) - viper.BindPFlag("namespace", rootCmd.PersistentFlags().Lookup("namespace")) - viper.BindPFlag("output", rootCmd.PersistentFlags().Lookup("output")) - viper.BindPFlag("log-level", rootCmd.PersistentFlags().Lookup("log-level")) + _ = viper.BindPFlag("context", rootCmd.PersistentFlags().Lookup("context")) + _ = viper.BindPFlag("namespace", rootCmd.PersistentFlags().Lookup("namespace")) + _ = viper.BindPFlag("output", rootCmd.PersistentFlags().Lookup("output")) + _ = viper.BindPFlag("log-level", rootCmd.PersistentFlags().Lookup("log-level")) // Add commands rootCmd.AddCommand(list.NewCommand()) @@ -69,6 +118,7 @@ func init() { rootCmd.AddCommand(sandbox.NewCommand()) rootCmd.AddCommand(translate.NewCommand()) rootCmd.AddCommand(version.NewCommand()) + rootCmd.AddCommand(newConfigCommand()) // Add hidden server command (for backward compatibility) serverCmd := server.NewCommand() @@ -76,35 +126,75 @@ func init() { rootCmd.AddCommand(serverCmd) } -// initConfig reads in config file and ENV variables if set. -func initConfig() { +// configDir returns the directory flowctl reads its user config from, honouring +// XDG_CONFIG_HOME as CLI-9 requires and falling back to the spec's default. +func configDir() (string, error) { + if dir := os.Getenv("XDG_CONFIG_HOME"); dir != "" { + return filepath.Join(dir, "flowctl"), nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", exitcode.Wrap(exitcode.Usage, + fmt.Errorf("cannot locate a config directory: %w; set XDG_CONFIG_HOME or pass --config", err)) + } + return filepath.Join(home, ".config", "flowctl"), nil +} + +// initConfig reads configuration in the precedence CLI-9 mandates: flags, then +// FLOWCTL_* environment variables, then the project file, then the user file, then +// defaults. +func initConfig() error { + viper.SetEnvPrefix(envPrefix) + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_")) + viper.AutomaticEnv() + if cfgFile != "" { - // Use config file from the flag. viper.SetConfigFile(cfgFile) - } else { - // Find home directory. - home, err := os.UserHomeDir() - if err != nil { - fmt.Println(err) - os.Exit(1) + // An explicitly named config file that cannot be read is a usage error. + // Only a discovered one may be absent. + if err := viper.ReadInConfig(); err != nil { + return exitcode.Wrap(exitcode.Usage, fmt.Errorf("reading config %s: %w", cfgFile, err)) } - - // Search config in home directory with name ".flowctl" (without extension). - viper.AddConfigPath(home + "/.config/flowctl") - viper.SetConfigType("yaml") - viper.SetConfigName("flowctl") + return nil } - viper.AutomaticEnv() // read in environment variables that match - - // Initialize the logger - if err := logger.Init(logLevel); err != nil { - fmt.Fprintln(os.Stderr, "Failed to initialize logger:", err) - os.Exit(1) + // Lowest precedence first: the user file, then the project file merged over it. + // + // Config files are located by exact name rather than viper's SetConfigName + // search. With a config type set, that search also matches an extension-less + // file — and a developer working in the repo root has a `flowctl` *binary* + // sitting there, which viper would happily try to parse as YAML. + if dir, err := configDir(); err == nil { + if err := mergeConfigFile(filepath.Join(dir, "flowctl.yaml"), false); err != nil { + return err + } + } + for _, name := range []string{"flowctl.yaml", "flowctl.yml"} { + if err := mergeConfigFile(name, true); err != nil { + return err + } } + if file := viper.ConfigFileUsed(); file != "" { + logger.Info("Using config file", zap.String("file", file)) + } + return nil +} - // If a config file is found, read it in. - if err := viper.ReadInConfig(); err == nil { - logger.Info("Using config file", zap.String("file", viper.ConfigFileUsed())) +// mergeConfigFile layers path over whatever is already loaded. A file that is not +// there is normal and ignored; a file that is there and malformed is a usage error, +// because silently ignoring it would run the command with configuration the operator +// believes is in effect. +func mergeConfigFile(path string, merge bool) error { + if _, err := os.Stat(path); err != nil { + return nil + } + viper.SetConfigFile(path) + read := viper.ReadInConfig + if merge { + read = viper.MergeInConfig } -} \ No newline at end of file + if err := read(); err != nil { + return exitcode.Wrap(exitcode.Usage, fmt.Errorf("reading config %s: %w", path, err)) + } + return nil +} diff --git a/cmd/run.go b/cmd/run.go index d54ce30..b8b0a22 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -121,26 +121,26 @@ func runV1Pipeline(cmd *cobra.Command, pipelineFile string) error { } func printRunPreflight(pipelineFile, pipelineName string) { - fmt.Println("Starting pipeline") - fmt.Println("-----------------") - fmt.Printf("Pipeline: %s\n", pipelineName) - fmt.Printf("Config file: %s\n", pipelineFile) - fmt.Printf("Orchestrator: %s\n", orchestratorType) + fmt.Fprintln(os.Stderr, "Starting pipeline") + fmt.Fprintln(os.Stderr, "-----------------") + fmt.Fprintf(os.Stderr, "Pipeline: %s\n", pipelineName) + fmt.Fprintf(os.Stderr, "Config file: %s\n", pipelineFile) + fmt.Fprintf(os.Stderr, "Orchestrator: %s\n", orchestratorType) if useExternalCP { - fmt.Printf("Control plane: external (%s:%d)\n", controlPlaneAddress, controlPlanePort) + fmt.Fprintf(os.Stderr, "Control plane: external (%s:%d)\n", controlPlaneAddress, controlPlanePort) } else if controlPlanePort == 0 { - fmt.Printf("Control plane: embedded (%s:auto)\n", controlPlaneAddress) + fmt.Fprintf(os.Stderr, "Control plane: embedded (%s:auto)\n", controlPlaneAddress) } else { - fmt.Printf("Control plane: embedded (%s:%d)\n", controlPlaneAddress, controlPlanePort) + fmt.Fprintf(os.Stderr, "Control plane: embedded (%s:%d)\n", controlPlaneAddress, controlPlanePort) } - fmt.Printf("Logs: %s\n", logDir) - fmt.Println() - fmt.Println("What happens next:") - fmt.Println(" 1. flowctl resolves and downloads any referenced components") - fmt.Println(" 2. the control plane starts") - fmt.Println(" 3. components launch and register") - fmt.Println(" 4. the pipeline is wired and starts running") - fmt.Println() + fmt.Fprintf(os.Stderr, "Logs: %s\n", logDir) + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "What happens next:") + fmt.Fprintln(os.Stderr, " 1. flowctl resolves and downloads any referenced components") + fmt.Fprintln(os.Stderr, " 2. the control plane starts") + fmt.Fprintln(os.Stderr, " 3. components launch and register") + fmt.Fprintln(os.Stderr, " 4. the pipeline is wired and starts running") + fmt.Fprintln(os.Stderr) } func init() { diff --git a/cmd/sandbox/logs.go b/cmd/sandbox/logs.go index 8dce6b5..2ece179 100644 --- a/cmd/sandbox/logs.go +++ b/cmd/sandbox/logs.go @@ -66,4 +66,4 @@ func runLogs(serviceName string, opts *logsOptions) error { } return rt.StreamAllLogs(logOptions) -} \ No newline at end of file +} diff --git a/cmd/sandbox/sandbox.go b/cmd/sandbox/sandbox.go index 59106d0..0b6120b 100644 --- a/cmd/sandbox/sandbox.go +++ b/cmd/sandbox/sandbox.go @@ -24,4 +24,4 @@ eliminating manual service startup and configuration.`, cmd.AddCommand(newUpgradeCommand()) return cmd -} \ No newline at end of file +} diff --git a/cmd/sandbox/start.go b/cmd/sandbox/start.go index 3528098..500a7d5 100644 --- a/cmd/sandbox/start.go +++ b/cmd/sandbox/start.go @@ -59,15 +59,15 @@ Use --services-only for a better development experience.`, func runStart(opts *startOptions) error { if opts.servicesOnly { - logger.Info("Starting Flowctl sandbox (services-only mode)", + logger.Info("Starting Flowctl sandbox (services-only mode)", zap.String("services", opts.services), zap.String("backend", opts.backend)) } else { - logger.Info("Starting Flowctl sandbox", + logger.Info("Starting Flowctl sandbox", zap.String("pipeline", opts.pipeline), zap.String("services", opts.services), zap.String("backend", opts.backend)) - + // Validate pipeline file exists (only when not in services-only mode) if _, err := os.Stat(opts.pipeline); os.IsNotExist(err) { return fmt.Errorf("pipeline file not found: %s", opts.pipeline) @@ -138,4 +138,4 @@ func runStart(opts *startOptions) error { } return nil -} \ No newline at end of file +} diff --git a/cmd/sandbox/status.go b/cmd/sandbox/status.go index a17f842..fad4bbe 100644 --- a/cmd/sandbox/status.go +++ b/cmd/sandbox/status.go @@ -2,10 +2,10 @@ package sandbox import ( "fmt" - "os" - "text/tabwriter" "github.com/spf13/cobra" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" "github.com/withobsrvr/flowctl/internal/sandbox/runtime" "github.com/withobsrvr/flowctl/internal/utils/logger" "go.uber.org/zap" @@ -20,60 +20,74 @@ func newStatusCommand() *cobra.Command { - Port mappings - Service endpoints - Resource usage`, + Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - return runStatus() + return runStatus(cliout.FromContext(cmd.Context())) }, } return cmd } -func runStatus() error { +func runStatus(p *cliout.Printer) error { logger.Info("Checking sandbox status") - // Get runtime instance rt, err := runtime.GetExistingRuntime() if err != nil { - return fmt.Errorf("failed to connect to runtime: %w", err) + return exitcode.Errorf(exitcode.Unavailable, + "cannot reach the container runtime: %v", err) } - // Get container status containers, err := rt.ListContainers() if err != nil { - return fmt.Errorf("failed to list containers: %w", err) + return exitcode.Errorf(exitcode.Unavailable, "failed to list containers: %v", err) } - if len(containers) == 0 { - fmt.Println("No sandbox containers running.") - fmt.Println("Use 'flowctl sandbox start' to start the sandbox.") - return nil - } - - // Display status in table format - w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) - fmt.Fprintln(w, "NAME\tSTATUS\tPORTS\tIMAGE\tUPTIME") - fmt.Fprintln(w, "----\t------\t-----\t-----\t------") - + rows := make([]cliout.Row, 0, len(containers)) for _, container := range containers { - ports := formatPorts(container.Ports) - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", - container.Name, - container.Status, - ports, - container.Image, - container.Uptime, - ) + ports := make([]map[string]any, 0, len(container.Ports)) + for _, port := range container.Ports { + ports = append(ports, map[string]any{ + "host": port.Host, + "container": port.Container, + }) + } + rows = append(rows, cliout.Row{ + "name": container.Name, + "status": container.Status, + "image": container.Image, + "uptime": container.Uptime, + "ports": ports, + "_ports": formatPorts(container.Ports), + }) } - w.Flush() + if err := p.List(cliout.List{ + Schema: "flowctl.sandbox_container_list.v1", + ItemSchema: "flowctl.sandbox_container.v1", + Columns: []cliout.Column{ + {Header: "Name", Key: "name"}, + {Header: "Status", Key: "status"}, + {Header: "Ports", Key: "_ports"}, + {Header: "Image", Key: "image"}, + {Header: "Uptime", Key: "uptime"}, + }, + Rows: rows, + Empty: "No sandbox containers running.\nUse 'flowctl sandbox start' to start the sandbox.", + }); err != nil { + return err + } - // Show network information - network, err := rt.GetNetworkInfo() - if err != nil { - logger.Warn("Failed to get network info", zap.Error(err)) - } else { - fmt.Printf("\nNetwork: %s\n", network.Name) - fmt.Printf("Subnet: %s\n", network.Subnet) + // Network details are context for a person, not part of the container + // list, so they go to stderr and are omitted for a program (CLI-5). + if !p.Machine() && len(rows) > 0 { + network, err := rt.GetNetworkInfo() + if err != nil { + logger.Warn("Failed to get network info", zap.Error(err)) + } else { + p.Info("\nNetwork: %s", network.Name) + p.Info("Subnet: %s", network.Subnet) + } } return nil @@ -92,4 +106,4 @@ func formatPorts(ports []runtime.PortMapping) string { result += fmt.Sprintf("%d:%d", port.Host, port.Container) } return result -} \ No newline at end of file +} diff --git a/cmd/sandbox/stop.go b/cmd/sandbox/stop.go index fc6603d..91fdaf2 100644 --- a/cmd/sandbox/stop.go +++ b/cmd/sandbox/stop.go @@ -36,7 +36,7 @@ and clean up any temporary resources.`, } func runStop(opts *stopOptions) error { - logger.Info("Stopping sandbox environment", + logger.Info("Stopping sandbox environment", zap.Int("timeout", opts.timeout), zap.Bool("force", opts.force)) @@ -58,4 +58,4 @@ func runStop(opts *stopOptions) error { logger.Info("Sandbox stopped successfully") return nil -} \ No newline at end of file +} diff --git a/cmd/sandbox/upgrade.go b/cmd/sandbox/upgrade.go index eeb4976..aa92ef7 100644 --- a/cmd/sandbox/upgrade.go +++ b/cmd/sandbox/upgrade.go @@ -52,4 +52,4 @@ func runUpgrade(opts *upgradeOptions) error { logger.Info("Runtime upgrade completed successfully") return nil -} \ No newline at end of file +} diff --git a/cmd/server/server.go b/cmd/server/server.go index 83f87d2..a64688d 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -13,10 +13,10 @@ import ( "github.com/spf13/cobra" flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" "github.com/withobsrvr/flowctl/internal/api" - flowctlpb "github.com/withobsrvr/flowctl/proto" "github.com/withobsrvr/flowctl/internal/config" "github.com/withobsrvr/flowctl/internal/storage" "github.com/withobsrvr/flowctl/internal/utils/logger" + flowctlpb "github.com/withobsrvr/flowctl/proto" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -59,7 +59,7 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. // Create gRPC server options var serverOptions []grpc.ServerOption - + // Configure TLS based on CLI flags tlsConfig := &config.TLSConfig{ Mode: config.TLSModeDisabled, @@ -67,28 +67,28 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. KeyFile: opts.TLSKey, CAFile: opts.TLSCACert, } - + // Enable TLS if cert and key are provided if opts.TLSCert != "" && opts.TLSKey != "" { tlsConfig.Mode = config.TLSModeEnabled - + // Enable mutual TLS if CA certificate is provided if opts.TLSCACert != "" { tlsConfig.Mode = config.TLSModeMutual } - + logger.Info("TLS configuration", zap.String("mode", string(tlsConfig.Mode)), zap.String("cert", tlsConfig.CertFile), zap.String("key", tlsConfig.KeyFile), zap.String("ca_cert", tlsConfig.CAFile)) - + // Load server credentials from TLS config serverCreds, err := tlsConfig.LoadServerCredentials() if err != nil { return fmt.Errorf("failed to load TLS credentials: %w", err) } - + // Add TLS credentials to server options if enabled if serverCreds != nil { serverOptions = append(serverOptions, serverCreds) @@ -97,7 +97,7 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. // Create gRPC server grpcServer := grpc.NewServer(serverOptions...) - + // Create and configure storage if persistence is enabled var serviceStorage storage.ServiceStorage if !opts.NoPersistence { @@ -110,13 +110,13 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. } dbPath = filepath.Join(homeDir, ".flowctl", storage.DefaultBoltFilePath) } - + // Ensure directory exists dbDir := filepath.Dir(dbPath) if err := os.MkdirAll(dbDir, 0755); err != nil { return fmt.Errorf("failed to create directory for database: %w", err) } - + logger.Info("Using persistent storage for service registry", zap.String("path", dbPath)) serviceStorage = storage.NewBoltDBStorage(&storage.BoltOptions{ Path: dbPath, @@ -124,10 +124,10 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. } else { logger.Info("Persistence disabled, using in-memory service registry") } - + // Create control plane server controlPlane := api.NewControlPlaneServer(serviceStorage) - + // Configure heartbeat TTL and janitor interval if opts.HeartbeatTTL > 0 { controlPlane.SetHeartbeatTTL(opts.HeartbeatTTL) @@ -135,12 +135,12 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. if opts.JanitorInterval > 0 { controlPlane.SetJanitorInterval(opts.JanitorInterval) } - + // Start the health check janitor and initialize storage if err := controlPlane.Start(); err != nil { return fmt.Errorf("failed to start control plane server: %w", err) } - + // Register both control plane services flowctlv1.RegisterControlPlaneServiceServer(grpcServer, controlPlane) @@ -151,7 +151,7 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. // Set up signal handling for graceful shutdown ctx, cancel := context.WithCancel(context.Background()) defer cancel() - + // Handle signals for graceful shutdown shutdown := make(chan os.Signal, 1) signal.Notify(shutdown, syscall.SIGINT, syscall.SIGTERM) @@ -197,4 +197,4 @@ monitoring, and scaling. This server exposes a gRPC API for pipeline components. cmd.Flags().BoolVar(&opts.NoPersistence, "no-persistence", false, "disable persistence for service registry") return cmd -} \ No newline at end of file +} diff --git a/cmd/status.go b/cmd/status.go index 630eb50..64fa4be 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -3,8 +3,14 @@ package cmd import ( "context" "fmt" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" + "github.com/withobsrvr/flowctl/internal/sanitize" + "io" + "os" "sort" "strings" + "sync" "time" "github.com/spf13/cobra" @@ -79,31 +85,42 @@ Examples: client := flowctlv1.NewControlPlaneServiceClient(conn) + p := cliout.FromContext(cmd.Context()) + + // Watch mode redraws a terminal; it has no meaning for a program + // consuming one JSON document (CLI-6). + if statusWatch && p.Machine() { + return exitcode.Errorf(exitcode.Usage, + "--watch cannot be combined with --output %s; poll the command instead", p.Format()) + } + // If watch mode, loop if statusWatch { for { rpcCtx, rpcCancel := context.WithTimeout(context.Background(), 5*time.Second) - err := displayStatus(rpcCtx, client, pipelineName) + err := displayStatus(rpcCtx, p, client, pipelineName) rpcCancel() if err != nil { return formatControlPlaneError(err, endpoint) } time.Sleep(time.Duration(statusWatchInterval) * time.Second) - fmt.Print("\033[H\033[2J") // Clear screen + if colorEnabled() { + fmt.Print("\033[H\033[2J") // Clear screen + } } } // Single status display rpcCtx, rpcCancel := context.WithTimeout(context.Background(), 5*time.Second) defer rpcCancel() - if err := displayStatus(rpcCtx, client, pipelineName); err != nil { + if err := displayStatus(rpcCtx, p, client, pipelineName); err != nil { return formatControlPlaneError(err, endpoint) } return nil }, } -func displayStatus(ctx context.Context, client flowctlv1.ControlPlaneServiceClient, pipelineName string) error { +func displayStatus(ctx context.Context, p *cliout.Printer, client flowctlv1.ControlPlaneServiceClient, pipelineName string) error { // Query all components resp, err := client.ListComponents(ctx, &flowctlv1.ListComponentsRequest{ IncludeUnhealthy: statusIncludeUnhealthy, @@ -112,19 +129,59 @@ func displayStatus(ctx context.Context, client flowctlv1.ControlPlaneServiceClie return fmt.Errorf("failed to list components: %w", err) } + // Both renderings need the health counts; only the human one groups by type. + counts := map[string]int{} + for _, comp := range resp.Components { + counts[statusHealthName(comp.GetStatus())]++ + } + + // The worst component health decides the pipeline's overall status. + overall := "running" + switch { + case counts["unhealthy"] > 0: + overall = "unhealthy" + case counts["degraded"] > 0: + overall = "degraded" + case counts["starting"] > 0: + overall = "starting" + } + + if p.Machine() { + // One document carrying the summary and every component, so a caller + // does not have to make a second call to get the detail. + components := make([]map[string]any, 0, len(resp.Components)) + for _, comp := range resp.Components { + components = append(components, statusComponentRecord(comp)) + } + data := cliout.Row{ + "status": overall, + "component_count": len(resp.Components), + "healthy": counts["healthy"], + "degraded": counts["degraded"], + "unhealthy": counts["unhealthy"], + "starting": counts["starting"], + "components": components, + } + if pipelineName != "" { + data["pipeline"] = pipelineName + } + return p.Object(cliout.Object{Schema: "flowctl.pipeline_status.v1", Data: data}) + } + if len(resp.Components) == 0 { - fmt.Println("No components registered with control plane") - fmt.Println("Start a pipeline with: flowctl run pipeline.yaml") + p.Info("No components registered with control plane") + p.Info("Start a pipeline with: flowctl run pipeline.yaml") return nil } - // Group components by type - sources := []*flowctlv1.ComponentStatusResponse{} - processors := []*flowctlv1.ComponentStatusResponse{} - sinks := []*flowctlv1.ComponentStatusResponse{} + // Human rendering below. This grouped layout is not a contract and may + // change freely; the record above is the contract. Untrusted strings are + // stripped here because they do not pass through a cliout cell formatter. + resp = sanitize.Message(resp) + var sources, processors, sinks []*flowctlv1.ComponentStatusResponse for _, comp := range resp.Components { - switch comp.Component.Type { + switch comp.GetComponent().GetType() { case flowctlv1.ComponentType_COMPONENT_TYPE_SOURCE: sources = append(sources, comp) case flowctlv1.ComponentType_COMPONENT_TYPE_PROCESSOR: @@ -134,84 +191,84 @@ func displayStatus(ctx context.Context, client flowctlv1.ControlPlaneServiceClie } } - // Count health statuses - healthy := 0 - degraded := 0 - unhealthy := 0 - starting := 0 - for _, comp := range resp.Components { - switch comp.Status { - case flowctlv1.HealthStatus_HEALTH_STATUS_HEALTHY: - healthy++ - case flowctlv1.HealthStatus_HEALTH_STATUS_DEGRADED: - degraded++ - case flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY: - unhealthy++ - case flowctlv1.HealthStatus_HEALTH_STATUS_STARTING: - starting++ - } - } - - // Print header + out := p.Out() if pipelineName != "" { - fmt.Printf("Pipeline: %s\n", pipelineName) + fmt.Fprintf(out, "Pipeline: %s\n", sanitize.Line(pipelineName)) } else { - fmt.Println("Pipeline Status") + fmt.Fprintln(out, "Pipeline Status") } - status := "RUNNING" - if unhealthy > 0 { - status = "UNHEALTHY" - } else if degraded > 0 { - status = "DEGRADED" - } else if starting > 0 { - status = "STARTING" + fmt.Fprintf(out, "Status: %s\n", colorizeStatus(strings.ToUpper(overall))) + fmt.Fprintf(out, "Components: %d total (%d healthy", len(resp.Components), counts["healthy"]) + for _, kind := range []string{"starting", "degraded", "unhealthy"} { + if counts[kind] > 0 { + fmt.Fprintf(out, ", %d %s", counts[kind], kind) + } } + fmt.Fprintln(out, ")") + fmt.Fprintln(out) + + for _, group := range []struct { + title string + comps []*flowctlv1.ComponentStatusResponse + }{ + {"SOURCES", sources}, + {"PROCESSORS", processors}, + {"CONSUMERS", sinks}, + } { + if len(group.comps) == 0 { + continue + } + fmt.Fprintf(out, "%s (%d)\n", group.title, len(group.comps)) + for _, comp := range group.comps { + printComponentStatus(out, comp) + } + fmt.Fprintln(out) + } + + return nil +} - fmt.Printf("Status: %s\n", colorizeStatus(status)) - fmt.Printf("Components: %d total (%d healthy", len(resp.Components), healthy) - if starting > 0 { - fmt.Printf(", %d starting", starting) +// statusHealthName renders a health status as a stable lowercase string. These +// values appear in --output json and are part of the schema. +func statusHealthName(status flowctlv1.HealthStatus) string { + name := strings.TrimPrefix(status.String(), "HEALTH_STATUS_") + if name == "" || name == "UNSPECIFIED" { + return "unknown" } - if degraded > 0 { - fmt.Printf(", %d degraded", degraded) + return strings.ToLower(name) +} + +// statusComponentRecord is one component inside the status document. +func statusComponentRecord(comp *flowctlv1.ComponentStatusResponse) map[string]any { + info := comp.GetComponent() + rec := map[string]any{ + "id": info.GetId(), + "type": strings.ToLower(strings.TrimPrefix(info.GetType().String(), "COMPONENT_TYPE_")), + "status": statusHealthName(comp.GetStatus()), } - if unhealthy > 0 { - fmt.Printf(", %d unhealthy", unhealthy) + if v := info.GetEndpoint(); v != "" { + rec["endpoint"] = v } - fmt.Println(")") - fmt.Println() - - // Print sources - if len(sources) > 0 { - fmt.Printf("SOURCES (%d)\n", len(sources)) - for _, comp := range sources { - printComponentStatus(comp) - } - fmt.Println() + if len(info.GetInputEventTypes()) > 0 { + rec["input_event_types"] = info.GetInputEventTypes() } - - // Print processors - if len(processors) > 0 { - fmt.Printf("PROCESSORS (%d)\n", len(processors)) - for _, comp := range processors { - printComponentStatus(comp) - } - fmt.Println() + if len(info.GetOutputEventTypes()) > 0 { + rec["output_event_types"] = info.GetOutputEventTypes() } - - // Print sinks - if len(sinks) > 0 { - fmt.Printf("CONSUMERS (%d)\n", len(sinks)) - for _, comp := range sinks { - printComponentStatus(comp) - } + if ts := comp.GetRegisteredAt(); ts != nil { + rec["registered_at"] = ts.AsTime() } - - return nil + if ts := comp.GetLastHeartbeat(); ts != nil { + rec["last_heartbeat"] = ts.AsTime() + } + if len(comp.GetMetrics()) > 0 { + rec["metrics"] = comp.GetMetrics() + } + return rec } -func printComponentStatus(comp *flowctlv1.ComponentStatusResponse) { +func printComponentStatus(out io.Writer, comp *flowctlv1.ComponentStatusResponse) { statusStr := colorizeHealth(comp.Status.String()) // Calculate uptime @@ -234,24 +291,24 @@ func printComponentStatus(comp *flowctlv1.ComponentStatusResponse) { lastHeartbeat = "never" } - fmt.Printf(" %s (%s)\n", comp.Component.Id, statusStr) - fmt.Printf(" Endpoint: %s\n", comp.Component.Endpoint) + fmt.Fprintf(out, " %s (%s)\n", comp.Component.Id, statusStr) + fmt.Fprintf(out, " Endpoint: %s\n", comp.Component.Endpoint) if uptime != "" { - fmt.Printf(" Uptime: %s\n", uptime) + fmt.Fprintf(out, " Uptime: %s\n", uptime) } - fmt.Printf(" Last heartbeat: %s\n", lastHeartbeat) + fmt.Fprintf(out, " Last heartbeat: %s\n", lastHeartbeat) // Print event types if len(comp.Component.InputEventTypes) > 0 { - fmt.Printf(" Input types: %s\n", strings.Join(comp.Component.InputEventTypes, ", ")) + fmt.Fprintf(out, " Input types: %s\n", strings.Join(comp.Component.InputEventTypes, ", ")) } if len(comp.Component.OutputEventTypes) > 0 { - fmt.Printf(" Output types: %s\n", strings.Join(comp.Component.OutputEventTypes, ", ")) + fmt.Fprintf(out, " Output types: %s\n", strings.Join(comp.Component.OutputEventTypes, ", ")) } // Print metrics if available if len(comp.Metrics) > 0 { - fmt.Println(" Metrics:") + fmt.Fprintln(out, " Metrics:") // Sort metric keys for consistent output keys := make([]string, 0, len(comp.Metrics)) @@ -262,21 +319,39 @@ func printComponentStatus(comp *flowctlv1.ComponentStatusResponse) { for _, key := range keys { value := comp.Metrics[key] - fmt.Printf(" %s: %s\n", key, value) + fmt.Fprintf(out, " %s: %s\n", key, value) } } } +// colorEnabled reports whether ANSI may be written to stdout. CLI-7 requires +// colour off when stdout is not a terminal, when NO_COLOR is set, or when +// TERM=dumb — otherwise `flowctl status > file` embeds escape codes in the file +// and `flowctl status | grep RUNNING` fails to match. +// +// Resolved once: stdout cannot change underneath a single invocation, and +// re-checking per cell would cost a syscall per field in watch mode. +var colorEnabled = sync.OnceValue(func() bool { + return cliout.ShouldColor(os.Stdout) +}) + +func colorize(code, text string) string { + if !colorEnabled() { + return text + } + return "\033[" + code + "m" + text + "\033[0m" +} + func colorizeStatus(status string) string { switch status { case "RUNNING": - return fmt.Sprintf("\033[32m%s\033[0m", status) // Green + return colorize("32", status) // Green case "STARTING": - return fmt.Sprintf("\033[36m%s\033[0m", status) // Cyan + return colorize("36", status) // Cyan case "DEGRADED": - return fmt.Sprintf("\033[33m%s\033[0m", status) // Yellow + return colorize("33", status) // Yellow case "UNHEALTHY": - return fmt.Sprintf("\033[31m%s\033[0m", status) // Red + return colorize("31", status) // Red default: return status } @@ -284,13 +359,13 @@ func colorizeStatus(status string) string { func colorizeHealth(health string) string { if strings.Contains(health, "HEALTHY") && !strings.Contains(health, "UNHEALTHY") { - return fmt.Sprintf("\033[32m%s\033[0m", health) // Green + return colorize("32", health) // Green } else if strings.Contains(health, "DEGRADED") { - return fmt.Sprintf("\033[33m%s\033[0m", health) // Yellow + return colorize("33", health) // Yellow } else if strings.Contains(health, "UNHEALTHY") { - return fmt.Sprintf("\033[31m%s\033[0m", health) // Red + return colorize("31", health) // Red } else if strings.Contains(health, "STARTING") { - return fmt.Sprintf("\033[36m%s\033[0m", health) // Cyan + return colorize("36", health) // Cyan } return health } diff --git a/cmd/translate/translate.go b/cmd/translate/translate.go index b13876f..bee624c 100644 --- a/cmd/translate/translate.go +++ b/cmd/translate/translate.go @@ -1,11 +1,11 @@ package translate import ( - "context" "fmt" "github.com/spf13/cobra" - "github.com/withobsrvr/flowctl/internal/interfaces" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" "github.com/withobsrvr/flowctl/internal/model" "github.com/withobsrvr/flowctl/internal/translator" "github.com/withobsrvr/flowctl/internal/utils/logger" @@ -15,7 +15,7 @@ import ( // Options contains the translate command options type Options struct { InputFile string - OutputFormat string + Target string OutputFile string ResourcePrefix string RegistryPrefix string @@ -26,27 +26,34 @@ var opts Options // NewCommand creates the translate command func NewCommand() *cobra.Command { cmd := &cobra.Command{ - Use: "translate -f -o ", + Use: "translate -f -t ", Short: "Translate pipeline to different deployment formats", Long: `Translate a pipeline specification into different deployment formats such as Docker Compose or for local execution. Examples: # Translate to Docker Compose and print to stdout - flowctl translate -f pipeline.yaml -o docker-compose + flowctl translate -f pipeline.yaml -t docker-compose # Save to a file - flowctl translate -f pipeline.yaml -o docker-compose --to-file docker-compose.yml + flowctl translate -f pipeline.yaml -t docker-compose --to-file docker-compose.yml # Translate with resource prefix for naming consistency - flowctl translate -f pipeline.yaml -o docker-compose --prefix myapp`, + flowctl translate -f pipeline.yaml -t docker-compose --prefix myapp + + # Wrap the result in a JSON envelope, for a script or an agent + flowctl translate -f pipeline.yaml -t docker-compose --output json | jq -r '.data.content'`, + Args: cobra.NoArgs, RunE: runTranslate, } - // Add flags + // Add flags. + // + // The deployment target used to be --output/-o, which collided with the + // global --output selecting an encoding (CLI-3). It is --target/-t now. cmd.Flags().StringVarP(&opts.InputFile, "file", "f", "", "input pipeline specification file") - cmd.Flags().StringVarP(&opts.OutputFormat, "output", "o", "", "output format (docker-compose, local)") - cmd.Flags().StringVar(&opts.OutputFile, "to-file", "", "write output to file instead of stdout") + cmd.Flags().StringVarP(&opts.Target, "target", "t", "", "deployment target (docker-compose, local)") + cmd.Flags().StringVar(&opts.OutputFile, "to-file", "", "write the result to a file instead of stdout") cmd.Flags().StringVar(&opts.ResourcePrefix, "prefix", "", "prefix for resource names") cmd.Flags().StringVar(&opts.RegistryPrefix, "registry", "", "container registry prefix") @@ -55,26 +62,23 @@ Examples: // runTranslate handles the execution of the translate command func runTranslate(cmd *cobra.Command, args []string) error { + p := cliout.FromContext(cmd.Context()) + if opts.InputFile == "" { - logger.Error("Missing input file", zap.String("command", "translate")) - return fmt.Errorf("input file is required. Use -f to specify a file") + return exitcode.Errorf(exitcode.Usage, "input file is required; use -f to specify one") } - - if opts.OutputFormat == "" { - logger.Error("Missing output format", zap.String("command", "translate")) - return fmt.Errorf("output format is required. Use -o to specify a format") + if opts.Target == "" { + return exitcode.Errorf(exitcode.Usage, "deployment target is required; use -t to specify one") } - // Create translator options := model.TranslationOptions{ ResourcePrefix: opts.ResourcePrefix, RegistryPrefix: opts.RegistryPrefix, OutputPath: opts.OutputFile, } - var t interfaces.Translator = translator.NewTranslator(options) + t := translator.NewTranslator(options) - // Validate output format - format := model.Format(opts.OutputFormat) + format := model.Format(opts.Target) validFormats := t.ValidFormats() isValid := false for _, f := range validFormats { @@ -84,30 +88,55 @@ func runTranslate(cmd *cobra.Command, args []string) error { } } if !isValid { - logger.Error("Invalid output format", - zap.String("format", string(format)), - zap.Any("valid_formats", validFormats)) - return fmt.Errorf("invalid output format: %s. Valid formats: %v", format, validFormats) + return exitcode.Errorf(exitcode.Usage, + "invalid target %q; valid targets: %v", format, validFormats) } - // Translate file - logger.Info("Translating pipeline", - zap.String("input", opts.InputFile), - zap.String("format", string(format))) + logger.Info("Translating pipeline", + zap.String("input", opts.InputFile), + zap.String("target", string(format))) - ctx := context.Background() - result, err := t.TranslateFromFile(ctx, opts.InputFile, format) + result, err := t.TranslateFromFile(cmd.Context(), opts.InputFile, format) if err != nil { - logger.Error("Translation failed", zap.Error(err)) - return fmt.Errorf("translation failed: %w", err) + // Translation fails when the pipeline is incomplete for the target — + // a missing image, say. That is a data validation failure, not an + // unclassified one (CLI-15). + return exitcode.Wrap(exitcode.DataInvalid, fmt.Errorf("translation failed: %w", err)) } - // If no output file was specified, print to stdout - if opts.OutputFile == "" { - fmt.Println(string(result)) - } else { - logger.Info("Output written to file", zap.String("path", opts.OutputFile)) + if opts.OutputFile != "" { + // The translator already wrote the file; report what happened rather + // than echoing the content. + return p.Object(cliout.Object{ + Schema: "flowctl.translation.v1", + Fields: []cliout.Field{ + {Label: "Source", Key: "source"}, + {Label: "Target", Key: "target"}, + {Label: "Written to", Key: "path"}, + }, + Data: cliout.Row{ + "source": opts.InputFile, + "target": string(format), + "path": opts.OutputFile, + }, + }) } - return nil -} \ No newline at end of file + // In a machine format the generated artifact is a field of the envelope, so + // a caller gets the target alongside the content. In table mode stdout is + // the artifact itself, unadorned, so `flowctl translate … > compose.yml` + // keeps working. + if p.Machine() { + return p.Object(cliout.Object{ + Schema: "flowctl.translation.v1", + Data: cliout.Row{ + "source": opts.InputFile, + "target": string(format), + "content": string(result), + }, + }) + } + + _, err = fmt.Fprintln(p.Out(), string(result)) + return err +} diff --git a/cmd/validate.go b/cmd/validate.go index 0fb4960..297af7a 100644 --- a/cmd/validate.go +++ b/cmd/validate.go @@ -1,10 +1,11 @@ package cmd import ( - "fmt" "os" "github.com/spf13/cobra" + "github.com/withobsrvr/flowctl/internal/cliout" + "github.com/withobsrvr/flowctl/internal/exitcode" "github.com/withobsrvr/flowctl/internal/runner" "github.com/withobsrvr/flowctl/internal/validator" ) @@ -22,45 +23,78 @@ This command checks for common configuration issues including: - Component executability (command, image, or type) - Processor chain and sink fan-out warnings +Exits 8 when validation fails, 3 when the file does not exist. + Examples: # Validate a pipeline file flowctl validate my-pipeline.yaml # Validate default pipeline - flowctl validate`, + flowctl validate + + # Gate a deploy on it, reading the reasons from JSON + flowctl validate pipeline.yaml --output json | jq -r '.data.errors[].message'`, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - // Default pipeline file + p := cliout.FromContext(cmd.Context()) + pipelineFile := "flow.yml" if len(args) > 0 { pipelineFile = args[0] } - // Check if file exists if _, err := os.Stat(pipelineFile); err != nil { - return fmt.Errorf("pipeline file not found: %s", pipelineFile) + return exitcode.Errorf(exitcode.NotFound, "pipeline file not found: %s", pipelineFile) } - // Load pipeline configuration - pipeline, err := runner.LoadPipelineFromFile(pipelineFile) - if err != nil { - return fmt.Errorf("failed to load pipeline: %w", err) + pipeline, loadErr := runner.LoadPipelineFromFile(pipelineFile) + if loadErr != nil { + // A file that will not parse is a validation failure, not a + // different kind of error. Reporting it through the same envelope + // means `validate --output json` always emits one document, so a + // caller never has to handle two shapes. + if err := emitResult(p, pipelineFile, validationRecord{ + Valid: false, + Errors: []map[string]any{{"field": "", "message": loadErr.Error()}}, + }, loadErr.Error()); err != nil { + return err + } + return exitcode.Errorf(exitcode.DataInvalid, + "%s could not be loaded: %v", pipelineFile, loadErr) } - // Create validator - v := validator.NewValidator(pipeline) + result := validator.NewValidator(pipeline).Validate() - // Run validation - result := v.Validate() + rec := validationRecord{Valid: result.Valid, Hints: result.Hints} + for _, e := range result.Errors { + item := map[string]any{"field": e.Field, "message": e.Message} + if e.Fix != "" { + item["fix"] = e.Fix + } + if e.Line > 0 { + item["line"] = e.Line + } + rec.Errors = append(rec.Errors, item) + } + for _, w := range result.Warnings { + item := map[string]any{"field": w.Field, "message": w.Message} + if w.Hint != "" { + item["hint"] = w.Hint + } + rec.Warnings = append(rec.Warnings, item) + } - // Print results - fmt.Println(result.Format()) + if err := emitResult(p, pipelineFile, rec, result.Format()); err != nil { + return err + } - // Exit with error code if validation failed if !result.Valid { - os.Exit(1) + // A failed validation is data invalid, not a generic failure + // (CLI-15). This used to be a bare os.Exit(1), which bypassed the + // exit-code contract entirely. + return exitcode.Errorf(exitcode.DataInvalid, + "%s failed validation with %d error(s)", pipelineFile, len(result.Errors)) } - return nil }, } @@ -68,3 +102,39 @@ Examples: func init() { rootCmd.AddCommand(validateCmd) } + +// validationRecord is the machine-readable shape of a validation outcome. +type validationRecord struct { + Valid bool + Errors []map[string]any + Warnings []map[string]any + Hints []string +} + +// emitResult writes a validation outcome in whichever format was asked for. +// humanText is what a person sees; it is ignored in machine mode. +func emitResult(p *cliout.Printer, file string, rec validationRecord, humanText string) error { + if !p.Machine() { + _, err := p.Out().Write([]byte(humanText + "\n")) + return err + } + if rec.Errors == nil { + rec.Errors = []map[string]any{} + } + if rec.Warnings == nil { + rec.Warnings = []map[string]any{} + } + if rec.Hints == nil { + rec.Hints = []string{} + } + return p.Object(cliout.Object{ + Schema: "flowctl.validation_result.v1", + Data: cliout.Row{ + "file": file, + "valid": rec.Valid, + "errors": rec.Errors, + "warnings": rec.Warnings, + "hints": rec.Hints, + }, + }) +} diff --git a/cmd/version/version.go b/cmd/version/version.go index e242702..ce3f83f 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -1,10 +1,10 @@ package version import ( - "fmt" "runtime" "github.com/spf13/cobra" + "github.com/withobsrvr/flowctl/internal/cliout" ) var ( @@ -21,15 +21,38 @@ func NewCommand() *cobra.Command { cmd := &cobra.Command{ Use: "version", Short: "Print the version", - Long: `Print the version, git commit, and build date of the flowctl CLI.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("flowctl version %s\n", Version) - fmt.Printf(" commit: %s\n", Commit) - fmt.Printf(" built: %s\n", BuildDate) - fmt.Printf(" go: %s\n", runtime.Version()) - fmt.Printf(" os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) + Long: `Print the version, git commit, and build date of the flowctl CLI. + +Examples: + # Human-readable + flowctl version + + # Machine-readable, for a release check in CI (CLI-13) + flowctl version --output json | jq -r '.data.version'`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + p := cliout.FromContext(cmd.Context()) + return p.Object(cliout.Object{ + Schema: "flowctl.version.v1", + Fields: []cliout.Field{ + {Label: "flowctl", Key: "version"}, + {Label: "commit", Key: "commit"}, + {Label: "built", Key: "build_date"}, + {Label: "go", Key: "go_version"}, + {Label: "os/arch", Key: "platform"}, + }, + Data: cliout.Row{ + "version": Version, + "commit": Commit, + "build_date": BuildDate, + "go_version": runtime.Version(), + "platform": runtime.GOOS + "/" + runtime.GOARCH, + "os": runtime.GOOS, + "arch": runtime.GOARCH, + }, + }) }, } return cmd -} \ No newline at end of file +} diff --git a/docker/contract-events-processor/Dockerfile b/docker/contract-events-processor/Dockerfile index 1b33b3e..c21566f 100644 --- a/docker/contract-events-processor/Dockerfile +++ b/docker/contract-events-processor/Dockerfile @@ -8,6 +8,14 @@ FROM scratch +# BUILD-4: OCI annotations so an image can be traced back to its source. +ARG VERSION=dev +ARG REVISION=unknown +LABEL org.opencontainers.image.source="https://github.com/withObsrvr/flowctl" \ + org.opencontainers.image.licenses="MPL-2.0" \ + org.opencontainers.image.revision="${REVISION}" \ + org.opencontainers.image.version="${VERSION}" + # Copy CA certificates for HTTPS COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ @@ -22,4 +30,8 @@ ENV PORT=:50054 ENV HEALTH_PORT=8082 ENV NETWORK_PASSPHRASE="Test SDF Network ; September 2015" +# BUILD-4: never run as root. A numeric UID needs no /etc/passwd entry, which +# a scratch image does not have. +USER 65532:65532 + ENTRYPOINT ["/component"] diff --git a/docker/duckdb-consumer/Dockerfile b/docker/duckdb-consumer/Dockerfile index 8f55d2a..207bedc 100644 --- a/docker/duckdb-consumer/Dockerfile +++ b/docker/duckdb-consumer/Dockerfile @@ -1,50 +1,43 @@ # Dockerfile for duckdb-consumer component -# Multi-stage build: builds the Go binary with CGO and packages it +# Packages a pre-built CGO binary. Debian slim rather than scratch, because +# DuckDB needs glibc. # -# Build context should be the flowctl directory: -# docker build -t docker.io/withobsrvr/duckdb-consumer:v1.0.0 \ -# -f docker/duckdb-consumer/Dockerfile \ -# --build-arg COMPONENTS_BASE=/path/to/components . +# Build context should be the duckdb-consumer directory, containing: +# - bin/duckdb-consumer (the compiled binary) +# - metadata.json (component metadata) +# +# Build the binary first, on a glibc-based system (not NixOS): +# CGO_ENABLED=1 go build -o bin/duckdb-consumer . # -# Or with pre-built binary: +# Then: # docker build -t docker.io/withobsrvr/duckdb-consumer:v1.0.0 \ +# --build-arg VERSION=v1.0.0 --build-arg REVISION=$(git rev-parse HEAD) \ # -f docker/duckdb-consumer/Dockerfile /path/to/duckdb-consumer -# Build stage -FROM golang:1.24-bookworm AS builder - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -# Set up workspace -WORKDIR /build - -# Copy all required source directories -# Note: Adjust paths based on your setup -COPY . /build/ - -# Note: This Dockerfile expects the binary to be pre-built using build-in-docker.sh -# The build context should be the duckdb-consumer directory with: -# - bin/duckdb-consumer (the compiled binary) -# - metadata.json (component metadata) - -# Runtime stage - use debian slim for glibc compatibility FROM debian:bookworm-slim +# BUILD-4: OCI annotations so an image can be traced back to its source. +ARG VERSION=dev +ARG REVISION=unknown +LABEL org.opencontainers.image.source="https://github.com/withObsrvr/flowctl" \ + org.opencontainers.image.licenses="MPL-2.0" \ + org.opencontainers.image.revision="${REVISION}" \ + org.opencontainers.image.version="${VERSION}" + # Install runtime dependencies RUN apt-get update && apt-get install -y \ ca-certificates \ && rm -rf /var/lib/apt/lists/* -# Create app directory +# BUILD-4: never run as root. The data directory is owned by this user, since +# the component writes its DuckDB file there. +RUN useradd --system --uid 65532 --user-group --no-create-home --shell /usr/sbin/nologin flowctl \ + && mkdir -p /data \ + && chown flowctl:flowctl /data + WORKDIR /app # Copy the pre-built binary -# Note: Build the binary with: CGO_ENABLED=1 go build -o bin/duckdb-consumer . -# on a glibc-based system (not NixOS) to ensure compatibility COPY bin/duckdb-consumer /component # Copy metadata @@ -55,8 +48,7 @@ ENV DUCKDB_PATH=/data/stellar.duckdb ENV PORT=:50053 ENV HEALTH_PORT=8090 -# Create data directory -RUN mkdir -p /data +USER flowctl:flowctl # The binary is the entrypoint ENTRYPOINT ["/component"] diff --git a/docker/stellar-live-source/Dockerfile b/docker/stellar-live-source/Dockerfile index c344fcc..e29defc 100644 --- a/docker/stellar-live-source/Dockerfile +++ b/docker/stellar-live-source/Dockerfile @@ -11,6 +11,14 @@ FROM scratch +# BUILD-4: OCI annotations so an image can be traced back to its source. +ARG VERSION=dev +ARG REVISION=unknown +LABEL org.opencontainers.image.source="https://github.com/withObsrvr/flowctl" \ + org.opencontainers.image.licenses="MPL-2.0" \ + org.opencontainers.image.revision="${REVISION}" \ + org.opencontainers.image.version="${VERSION}" + # Copy CA certificates for HTTPS connections (needed for Stellar RPC/Horizon) COPY --from=alpine:latest /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ @@ -21,5 +29,9 @@ COPY go/bin/stellar-live-source /component # Copy metadata (must be at /metadata.json for flowctl resolver) COPY metadata.json /metadata.json +# BUILD-4: never run as root. A numeric UID needs no /etc/passwd entry, which +# a scratch image does not have. +USER 65532:65532 + # The binary is the entrypoint ENTRYPOINT ["/component"] diff --git a/docs/CLI_APPLICATION_ROADMAP.md b/docs/CLI_APPLICATION_ROADMAP.md new file mode 100644 index 0000000..10dfea0 --- /dev/null +++ b/docs/CLI_APPLICATION_ROADMAP.md @@ -0,0 +1,1071 @@ +# Flowctl programmable application and CLI roadmap + +**Status:** Proposed implementation roadmap +**Scope:** flowctl CLI, application services, control plane, resource contracts, +storage, process runtime, and operator-facing automation +**Audience:** Maintainers and contributors planning work after `v0.0.12` + +## Purpose + +Flowctl should evolve from a collection of orchestration commands into a +coherent, programmable pipeline application. The CLI, a future web interface, +automation clients, and agents should invoke the same typed operations and +observe the same authoritative state. + +The guiding principle is: + +> Build flowctl as a system of identifiable resources, explicit operations, +> predictable state machines, and resumable events. Make the CLI the first and +> best client of that system. + +This document turns broad CLI and application-design guidance into a roadmap +that fits flowctl's current architecture. It preserves the flagship scope: + +- `flowctl/v1` pipeline definitions; +- process orchestration as the primary runtime; +- embedded control plane as the default local experience; +- external control plane as a secondary advanced mode; +- supervised gRPC delivery through sources, processors, and sinks; +- pipeline run history, health, failure context, and operator control. + +It does not make arbitrary DAG execution, container orchestration, or a hosted +multi-tenant service prerequisites for improving the current product. + +## Product boundary + +Flowctl is responsible for: + +- validating pipeline definitions; +- resolving component artifacts; +- launching and stopping components; +- wiring and supervising data-plane streams; +- recording pipeline run lifecycle and failure context; +- exposing current and historical operational state; +- providing synchronization primitives for operators and automation; +- serving the same capabilities through a typed control-plane API; +- supporting local embedded and external control-plane deployments. + +Flowctl is not currently promising: + +- arbitrary graph execution beyond the documented runtime shape; +- several equally mature runtime drivers; +- a Kubernetes operator; +- a public hosted control plane; +- multi-tenant authentication and billing; +- exactly-once processing without component-specific durable contracts; +- source or sink checkpoint ownership; +- automatic infinite restart; +- dynamic code plugins inside the flowctl process. + +Nebu remains the local Unix-pipe prototyping and processor-development layer. +Flowctl owns production lifecycle and operational state. + +## Existing foundations + +Flowctl is not starting from zero. It already has: + +- pipeline definitions and validation; +- pipeline run IDs and persistent run records; +- explicit run states; +- component registration and heartbeats; +- component health data; +- process and embedded-control-plane operation; +- external control-plane connectivity; +- supervised source, processor, and sink delivery; +- first-error cancellation across stream edges; +- bounded completion and sink acknowledgement checks; +- direct source-to-sink routing; +- structured, credential-free delivery failure records; +- chunk resources for bounded historical work; +- operator commands for active runs, run information, and stop; +- global context, namespace, output, configuration, and noninteractive flags; +- synthetic process-level integration coverage. + +The next step is to make those pieces one consistent application contract. + +## Core design principles + +1. **The CLI is a client of application operations.** Cobra handlers should + parse, call, and render—not implement orchestration or storage behavior. +2. **Embedded and remote modes share semantics.** A local operation and a + control-plane RPC must apply the same validation, authorization boundary, + state transition, and error classification. +3. **Resources have stable identities.** Pipelines, runs, components, chunks, + and future operations use IDs that scripts never derive from display order. +4. **State machines are explicit.** Legal transitions, terminal states, and + idempotent repeat behavior are documented and tested. +5. **Commands, queries, and events are distinct.** Mutations request work, + queries read state, and events describe committed changes. +6. **State and event publication are coordinated.** Clients must not observe an + event for a state change that was not committed, or miss committed changes + because an event was emitted only in process memory. +7. **Machine output is versioned.** JSON, NDJSON, API, and event contracts have + explicit compatibility rules. +8. **Human presentation is replaceable.** Tables and prose may evolve without + becoming parsing contracts. +9. **Cancellation is not failure.** Operator stop, context cancellation, + component failure, and successful completion remain distinct outcomes. +10. **Durability claims are precise.** Transport receipt, handler completion, + sink acknowledgement, and durable commit must not be conflated. +11. **Process mode comes first.** New abstractions must prove themselves on the + flagship runtime before being generalized to secondary drivers. +12. **No silent degradation.** Failed registration, delivery, persistence, or + explicitly requested observability must be visible and actionable. + +## Target architecture + +The target architecture separates product semantics from adapters: + +```text +CLI / future web UI / automation / agent + | + v + typed application client + / \ + embedded implementation RPC implementation + \ / + application services + | + domain state machines + | + storage + event journal + | + process runtime / components / streams +``` + +Illustrative interfaces: + +```go +type PipelineRuns interface { + Start(ctx context.Context, req StartRunRequest) (PipelineRun, error) + Get(ctx context.Context, id string) (PipelineRun, error) + List(ctx context.Context, filter RunFilter) (RunPage, error) + Stop(ctx context.Context, id string, reason string) (PipelineRun, error) + Wait(ctx context.Context, id string, condition RunCondition) (PipelineRun, error) +} + +type Events interface { + List(ctx context.Context, filter EventFilter) (EventPage, error) + Watch(ctx context.Context, filter EventFilter, cursor string) (EventStream, error) +} +``` + +These interfaces are examples, not an instruction to publish a large Go API +immediately. Begin internally and expose only contracts with demonstrated use. + +## Resource model + +### Pipeline definition + +A pipeline definition is desired topology and configuration. Today it is a +`flowctl/v1` YAML document. It is not the same thing as a running pipeline. + +Recommended identity: + +- stable pipeline ID assigned when stored, if persistent definitions are added; +- human-readable name for discovery; +- immutable definition revision or digest; +- schema/API version; +- creation/update timestamps; +- source metadata without embedded secret values. + +Persistent pipeline definitions are optional for the first phases. A run can +continue to embed the submitted definition while the run model is stabilized. + +### Pipeline run + +A run is one execution attempt with a stable run ID. It owns: + +- pipeline name and definition revision/digest; +- lifecycle state; +- component instances; +- start/end timestamps; +- classified terminal result; +- structured failure context; +- operator stop reason; +- correlation/operation ID; +- aggregate progress that is carefully distinguished from durable sink state. + +### Component instance + +A component instance is a source, processor, or sink participating in one run. +Registration identity must distinguish the logical component from a process +instance or restart attempt. + +### Chunk + +A chunk represents a bounded historical range attempt. The existing chunk +model is a useful foundation, but its transition rules, retry ownership, +verification boundary, and relationship to a pipeline run must be formalized. + +### Operation + +An operation represents an accepted asynchronous mutation when a command cannot +truthfully return a final result immediately. Examples may include future +remote starts, controlled checkpoints, or long stop/drain operations. + +Do not create operation resources for synchronous local work merely for +architectural symmetry. + +### Event + +An event is an immutable description of a committed lifecycle change. It is +not a log line and not the pipeline data stream. + +## State machines + +### Pipeline run state + +The current run enum is a useful base: + +```text +unknown -> starting -> running -> completed + | | + | +----> failed + +-------> stopping -> stopped +``` + +The exact addition of `stopping` should be made only with a protobuf/API +versioning decision. Until then, stopping may remain an operation/event while +the stored state transitions directly to `stopped`. + +Required rules: + +- `completed`, `failed`, and `stopped` are terminal; +- successful bounded delivery alone can produce `completed`; +- component/data-plane failure produces `failed`; +- operator cancellation produces `stopped`, not `failed`; +- terminal states cannot be overwritten by late heartbeats or shutdown updates; +- repeating stop on a stopped run is idempotent; +- stopping a completed or failed run returns the existing terminal state or a + typed conflict, not a fabricated success; +- the first authoritative failure is retained; +- start must not report `running` until required components are registered and + stream wiring has succeeded; +- registration-only mode remains explicit and cannot imply data delivery. + +### Chunk state + +Formalize legal transitions among planned, dispatched, running, completed, +verified, failed, retry-pending, and aborted. At minimum: + +- only completed work can become verified; +- a failed attempt is immutable; +- retry creates or advances an attempt without erasing prior evidence; +- aborted is distinct from failed; +- verification records reference the exact attempt and range; +- overlapping active chunks for the same pipeline/component/range are rejected + unless explicitly supported. + +Use table-driven transition tests as the executable specification. + +## Prioritized implementation plan + +### Priority 1: Define application services and thin CLI adapters + +#### Problem + +Current Cobra handlers often create gRPC connections, use background contexts, +query the control plane, transform domain state, and render directly to global +stdout. This makes embedded and remote behavior harder to keep consistent. + +#### Work + +- Introduce internal typed services for run queries and mutations. +- Introduce a typed client interface implemented by embedded and gRPC clients. +- Move run-ID resolution, filtering, stop semantics, and state validation out + of Cobra handlers. +- Thread `cmd.Context()` through every operation instead of creating + `context.Background()` in command paths. +- Inject clocks and storage where state transition tests require determinism. +- Keep orchestration internals private until an external Go API is justified. +- Use `cmd.OutOrStdout()` and `cmd.ErrOrStderr()` through renderers. + +#### Acceptance criteria + +- Pipeline list/get/stop behavior can be tested without Cobra and without a + live TCP listener. +- The same service tests run against memory and persistent storage. +- Embedded and external clients pass shared contract tests. +- No command-specific business rule exists only in a renderer. + +### Priority 2: Establish stable CLI exit codes and structured errors + +#### Problem + +The root command currently maps every error to exit code 1 and human prose. +Automation cannot distinguish invalid configuration, unavailable control plane, +not-found resources, conflicts, failed runs, authorization, or interruption. + +#### Proposed exit classes + +The exact values require a compatibility decision, but the contract should +distinguish: + +- success; +- general execution failure; +- invalid usage/configuration; +- not found; +- unavailable dependency/control plane; +- conflict or illegal state transition; +- authentication/authorization failure when remote security exists; +- pipeline run failed; +- interruption (conventionally 130 for SIGINT). + +#### Structured error + +```json +{ + "_schema": "flowctl.error.v1", + "code": "pipeline_run_failed", + "exit_code": 7, + "operation": "pipeline.run", + "resource": {"type": "pipeline_run", "id": "..."}, + "message": "Pipeline delivery failed", + "details": {"edge": "source -> sink", "cause": "unavailable"}, + "correlation_id": "..." +} +``` + +#### Work + +- Create typed domain/application errors with stable string codes. +- Map gRPC status codes and local errors into the same taxonomy. +- Add a stable JSON error mode on stderr. +- Redact pipeline environment values and credentials. +- Preserve safe structured delivery failure details. +- Document whether component logs may precede the terminal JSON record; prefer a + dedicated diagnostics stream/file when strict parsing is required. + +#### Acceptance criteria + +- Exit-code behavior is table-tested at the CLI boundary. +- Embedded and remote forms of the same failure yield the same code. +- interruption is never reported as pipeline failure; +- stdout remains a valid selected output format on errors. + +### Priority 3: Stabilize machine-readable output + +#### Problem + +The root advertises `table|yaml|json|wide`, but individual commands do not +consistently honor the global output flag. Human tables and protobuf-derived +JSON are not yet defined as stable CLI contracts. + +#### Work + +- Inventory every command and supported output format. +- Centralize renderers for table, wide, JSON, YAML, and NDJSON. +- Define versioned envelopes such as: + - `flowctl.pipeline_list.v1`; + - `flowctl.pipeline_run.v1`; + - `flowctl.component_list.v1`; + - `flowctl.chunk_list.v1`; + - `flowctl.status.v1`. +- Define pagination fields even if the first local implementation returns one + page. +- Use stable enum strings rather than Go/protobuf debug formatting. +- Preserve exact IDs in machine output; truncation is human presentation only. +- Define null/omitted behavior for timestamps, metrics, and failures. +- Make empty results successful with an empty collection. +- Add JSON Schema or equivalent contract tests. + +#### Compatibility strategy + +Do not silently change existing JSON shapes. Introduce explicit versioned +formats or document a pre-1.0 migration and deprecation window. + +#### Acceptance criteria + +- Every read command honors documented output formats. +- JSON/YAML contain no ANSI escapes, progress lines, or logger output. +- Contract tests cover unknown fields and enum stability. +- human and machine renderers consume the same typed results. + +### Priority 4: Implement `pipeline wait` + +#### Goal + +Replace arbitrary sleeps and polling loops with an explicit synchronization +primitive. + +#### Proposed interface + +```bash +flowctl pipeline wait --for running --timeout 2m +flowctl pipeline wait --for terminal --timeout 30m +flowctl pipeline wait --for completed --output json +``` + +If the existing plural `pipelines` group remains canonical, use +`flowctl pipelines wait`. Settle singular/plural command organization before +adding aliases that become permanent. + +#### Conditions + +- `starting`, `running`, `completed`, `failed`, `stopped`, or `terminal`; +- optionally component-health readiness after its semantics are precise; +- never a vague `ready` condition without a documented definition. + +#### Semantics + +- returns 0 only when the requested condition is satisfied; +- a run reaching an incompatible terminal state returns a typed nonzero result; +- timeout differs from run failure and user interruption; +- deleting or losing the resource is not success; +- remote watch may be used later, but bounded polling is acceptable initially; +- polling includes backoff and honors context cancellation. + +#### Acceptance criteria + +- process integration tests cover success, failure, stopped, timeout, and + interruption; +- scripts no longer require fixed sleeps in the flagship workflow; +- machine output returns the final observed resource. + +### Priority 5: Add a durable lifecycle event model + +#### Goal + +Let clients understand how resources reached their current state and react to +changes without scraping logs or repeatedly listing resources. + +#### Event envelope + +```json +{ + "_schema": "flowctl.event.v1", + "event_id": "...", + "sequence": 42, + "occurred_at": "2026-10-14T14:00:00Z", + "type": "pipeline.run.failed", + "resource": {"type": "pipeline_run", "id": "..."}, + "pipeline_id": "...", + "run_id": "...", + "operation_id": "...", + "correlation_id": "...", + "data": {} +} +``` + +#### Initial event names + +- `pipeline.run.created`; +- `pipeline.run.starting`; +- `pipeline.run.running`; +- `pipeline.run.completed`; +- `pipeline.run.failed`; +- `pipeline.run.stop_requested`; +- `pipeline.run.stopped`; +- `component.registered`; +- `component.health_changed`; +- `delivery.edge_failed`; +- chunk lifecycle events matching committed transitions. + +#### Durability rules + +- Events describe committed state transitions. +- Event IDs and per-stream sequence numbers are stable. +- Storage commits state and its event atomically, or uses an outbox pattern that + guarantees eventual publication without inventing events. +- Retention and compaction are explicit. +- Events contain bounded metadata and no secrets. +- Event history is operational metadata, not the high-volume pipeline data + plane. + +#### Acceptance criteria + +- Restarting the control plane does not erase retained lifecycle history. +- Tests prove state/event atomicity under injected storage failure. +- Duplicate delivery is tolerated through stable event IDs. +- Event order is defined per resource or journal, not implied globally without + evidence. + +### Priority 6: Add event listing and follow + +#### Proposed interface + +```bash +flowctl pipeline events +flowctl pipeline events --follow +flowctl pipeline events --after --output ndjson +``` + +#### Snapshot-plus-watch contract + +A client must be able to: + +1. request the current resource and a corresponding event cursor; +2. subscribe after that cursor; +3. receive every subsequent committed event without a snapshot/watch race; +4. reconnect using the last acknowledged cursor. + +The first implementation may use a gRPC server stream. A future browser-facing +adapter may translate the same application event stream to SSE or WebSocket. + +#### Backpressure and retention + +- Slow event clients must not block pipeline delivery. +- Use bounded subscriber buffers and disconnect lagging clients with a resumable + cursor/error. +- Return a typed cursor-expired response when retention has removed history. +- Keep event payloads small and schema-versioned. + +#### Acceptance criteria + +- follow works across control-plane restart when history is retained; +- reconnect produces no gaps and consumers can deduplicate repeats; +- NDJSON stdout contains events only; +- Ctrl+C exits as interruption without changing run state. + +### Priority 7: Make start/stop idempotent and operation-aware + +#### Goal + +Automation should be able to retry mutations safely after timeouts or uncertain +responses. + +#### Work + +- Accept an idempotency key for start and other creation-like operations. +- Store the key with the resulting run/operation for a bounded retention period. +- Repeating the same key and equivalent request returns the existing result. +- Reusing a key for a different request returns a conflict. +- Make stop naturally idempotent. +- Return an operation ID if a remote mutation is accepted before it reaches its + final state. +- Add `operation get`/`wait` only when asynchronous operations actually exist. +- Never equate idempotent orchestration with exactly-once sink effects. + +#### Acceptance criteria + +- Tests inject response loss and prove a retry does not create a second run. +- idempotency keys are not logged with secret request content; +- local CLI defaults may generate a key for retryable remote calls while still + exposing explicit control to automation. + +### Priority 8: Add `plan` for pipeline definitions + +#### Goal + +Show resolution, validation, and operational consequences before starting +components. + +#### Proposed interface + +```bash +flowctl pipeline plan -f stellar-pipeline.yaml +flowctl pipeline plan -f stellar-pipeline.yaml --output json +``` + +#### Plan content + +- parsed API version and pipeline identity; +- selected runtime driver and support level; +- resolved component references, versions, commands/images, and digests where + available; +- source-to-processor-to-sink edges actually supported by the runtime; +- input/output event compatibility; +- ports, health endpoints, persistence mode, and control-plane mode; +- required secret names without values; +- bounded/unbounded source declarations; +- warnings for experimental drivers or registration-only mode; +- local mutations that start would perform; +- definition digest used to associate a run with the plan. + +#### Non-goal + +Do not call this Terraform-style apply/reconciliation until flowctl stores +persistent desired pipeline definitions and can safely converge actual state. +Initially, `plan` is an immutable launch preview. + +#### Acceptance criteria + +- plan performs no launches or durable mutations; +- secret values never appear; +- start and plan share the same resolver and validator; +- a stale plan digest is detected if start is asked to consume an approved + plan artifact. + +### Priority 9: Add capability introspection + +#### Goal + +Let clients discover what a control plane and component support without +guessing from version strings or help text. + +#### Control-plane capabilities + +Examples: + +- API/protocol versions; +- supported pipeline schema versions; +- supported runtime drivers and maturity; +- event history/watch support; +- idempotency support; +- persistence mode; +- authentication mode; +- maximum message sizes and relevant bounded limits. + +#### Component capabilities + +Build on component metadata and SDK contracts: + +- component kind and version; +- accepted and produced event types; +- bounded-stream semantics; +- health/readiness endpoints; +- supported operations such as checkpoint or drain, only when standardized; +- configuration schema with secret annotations; +- delivery/acknowledgement contract version. + +#### Constraints + +- Keep common operations typed. Do not replace `stop` or `wait` with an opaque + generic method call. +- Extension methods require namespaced identifiers, typed arguments, side-effect + descriptions, and permission requirements. +- Unknown capabilities are ignored by old clients. + +#### Acceptance criteria + +- The CLI can reject an unsupported operation before partially executing it. +- Capability results have a versioned machine format. +- Runtime support level is truthful; experimental Docker behavior is not + advertised as equivalent to process mode. + +### Priority 10: Add `flowctl doctor` + +#### Default checks + +- CLI version and executable path; +- configuration file selected and precedence sources; +- active context and namespace; +- persistence path and permissions; +- control-plane reachability and reported capabilities when configured; +- required ports and conflicting listeners for an embedded run; +- process runtime prerequisites; +- component cache integrity and executable permissions; +- pipeline component resolution without exposing secrets; +- schema/protocol compatibility among flowctl, flowctl-sdk, flow-proto, and + components; +- stale runs whose owning process/control plane no longer exists; +- whether an advertised health endpoint is actually reachable. + +#### Modes + +```bash +flowctl doctor +flowctl doctor --pipeline stellar-pipeline.yaml +flowctl doctor --control-plane 127.0.0.1:8080 +flowctl doctor --output json +``` + +Destructive repair must never be implicit. If repair commands are later added, +show an explicit plan and require confirmation or `--yes`. + +#### Acceptance criteria + +- Offline/local checks work without starting a pipeline. +- Remote checks use bounded timeouts. +- Results distinguish failure, warning, and informational findings. +- Suggested remediation uses commands that are tested and current. + +### Priority 11: Make contexts, namespaces, and configuration real + +#### Problem + +Global flags exist, but their semantics and coverage must be consistent before +they can support local, shared, and future hosted control planes. + +#### Work + +- Define configuration precedence: command flag, environment, selected context, + config file, default. +- Add `flowctl config view --effective` with redaction. +- Add `flowctl context list|get|use` only after context storage and endpoint/TLS + semantics are fully specified. +- Define namespace behavior. If namespaces are not enforced by storage and API, + stop presenting the flag as a production isolation mechanism. +- Keep credentials out of command arguments where process listings could expose + them. +- Support TLS and authentication configuration consistently across status, + pipelines, chunks, and future events commands. + +#### Acceptance criteria + +- All control-plane commands use the same resolved client configuration. +- Effective configuration identifies each value's source. +- namespaces are either enforced end-to-end or clearly marked reserved. +- machine output never contains secret material. + +### Priority 12: Add stable audit metadata before multi-user operation + +Flowctl does not need a full compliance subsystem for local embedded mode, but +state mutations should carry enough attribution to support future shared use: + +- actor identity when authenticated; +- client name/version; +- operation and correlation IDs; +- request source (`cli`, API, automation); +- timestamp; +- target resource; +- redacted reason for stop/retry/checkpoint; +- resulting state/event ID. + +Do not claim an authoritative audit log until events are durable, tamper policy +is defined, retention is configured, and authenticated identity exists. + +## Command organization + +Flowctl currently mixes top-level commands and a plural `pipelines` resource +group. Before expanding the command set, choose and document one grammar. + +Recommended long-term form: + +```text +flowctl pipeline validate +flowctl pipeline plan +flowctl pipeline run +flowctl pipeline list +flowctl pipeline inspect +flowctl pipeline wait +flowctl pipeline events +flowctl pipeline stop + +flowctl component list +flowctl component inspect + +flowctl chunk list +flowctl chunk inspect +``` + +However, compatibility matters. Existing `flowctl run`, `validate`, `status`, +and `pipelines ...` commands should remain aliases through a documented +deprecation period if commands move. Do not rename merely for aesthetic +consistency; make the change only when the new resource model adds value. + +Global flags should behave uniformly: + +- `--output` selects presentation for non-streaming results; +- `--context` selects client/control-plane configuration; +- `--namespace` is meaningful only if enforced; +- `--timeout` should represent an operation deadline, not silently change + pipeline runtime duration; +- `--yes` suppresses confirmation but never validation; +- `--no-color` and `NO_COLOR` affect human presentation only; +- `--log-format` and `--log-level` affect diagnostics, never command data. + +## API and contract strategy + +### Canonical protobuf ownership + +Flowctl currently has local control-plane protobufs while data-plane component +contracts live in `flow-proto`. Establish clear ownership: + +- shared external contracts should live in a versioned contract module; +- generated code should not be the only specification; +- Buf or an equivalent compatibility check should detect breaking protobuf + changes; +- field numbers are never reused; +- enum values are append-only; +- API versions and server capabilities are discoverable. + +Do not move contracts only for repository tidiness. Plan compatibility, +imports, releases, generated code, and migration together. + +### Browser-facing API + +A future dashboard should call the same application services. Connect RPC, +gRPC-Web, or a small HTTP/SSE adapter can be evaluated after the service and +event contracts stabilize. + +The browser API must not become an independent implementation of pipeline +state transitions. + +### SDK boundary + +Flowctl owns orchestration semantics. SDKs own component registration, +heartbeats, health, and stream-service helpers. Keep contract tests across: + +- `flowctl`; +- `flowctl-sdk`; +- `flow-proto`; +- representative source, processor, and sink components; +- JavaScript/TypeScript SDK where supported. + +## Observability model + +### Logs + +- Human logs go to stderr or component log files. +- Structured logs use stable field names but are not a substitute for events. +- Every log line should carry run, component, operation, and correlation IDs + where applicable. +- Environment maps and credentials are never logged wholesale. + +### Progress and metrics + +Define each counter precisely: + +- transport events received/sent; +- handler successes/failures; +- sink acknowledgements; +- durable destination watermark, when reported by the destination; +- bytes and rates with measurement interval. + +Do not populate `events_processed` until its meaning and aggregation are +correct. Prefer omitted/unknown to a misleading zero. + +### Health + +Separate: + +- liveness: process/control-plane responsiveness; +- readiness: able to accept/control work; +- data-plane health: active streams and components functioning; +- durability: sink-specific committed watermark. + +A green control-plane heartbeat is not proof that a pipeline is delivering or +committing data. + +### Status protocol + +Flowctl itself can emit OSC 7501 for foreground `run`, `wait`, plan, component +resolution, and blocking permission/authentication states. OSC is terminal +presentation metadata, not a replacement for durable lifecycle events or JSON +results. + +For headless foreground invocation, consider a versioned status-file stream +only if the durable control-plane event API does not already satisfy the use +case. Avoid two competing machine status contracts. + +## Reliability requirements + +### Supervision + +- every delivery goroutine belongs to a supervised group; +- first non-cancellation failure cancels sibling edges; +- shutdown waits for bounded cleanup; +- no event is dropped to keep a run apparently healthy; +- fan-out applies backpressure or fails explicitly; +- sink close acknowledgement is verified; +- unexpected EOF from an unbounded source is failure; +- bounded completion is declared explicitly by the source contract. + +### Persistence + +- terminal state and first failure survive restart; +- writes are transactionally safe; +- storage migrations are versioned and tested; +- corrupted or incompatible state produces an actionable startup error; +- backup/export is available before destructive migrations; +- embedded persistence defaults are explicit and documented. + +### Recovery + +Flowctl must not claim checkpoint ownership it does not have. A recovery plan +should identify: + +- source restart cursor; +- destination durable watermark; +- component-specific replay behavior; +- orchestration run/attempt identity; +- verification command proving convergence. + +Automatic restart should be introduced only after those contracts are +machine-readable and retries are demonstrably safe. + +## Security requirements + +Before external/shared control-plane use is considered production-ready: + +- TLS configuration is required or an explicit insecure-local mode is used; +- clients and components authenticate; +- mutation authorization is enforced server-side; +- secrets use references or protected environment injection, not stored plain + in run definitions or events; +- APIs have message-size, rate, and concurrency bounds; +- component registration cannot impersonate another run/component; +- local state and sockets have restrictive permissions; +- structured failures, plans, and audit metadata are redacted; +- server-side validation never trusts CLI validation alone. + +Secure local IPC may be considered for an always-running local control plane, +but a daemon should not be added until persistent background operation is a +real supported workflow. + +## Developer experience + +### `flowctl validate` + +Validation should provide: + +- stable diagnostic codes; +- source locations in YAML where practical; +- all independent validation errors in one run; +- human and versioned JSON output; +- strict mode for warnings; +- offline structural validation by default; +- an explicit resolution/connectivity mode for external checks. + +### Schema discovery + +Publish the canonical `flowctl/v1` pipeline schema and expose it through a +command such as: + +```bash +flowctl schema pipeline --version v1 --output json +``` + +This supports editors, agents, CI, and form generation without scraping help. + +### Shell completion + +Generate Bash, Zsh, Fish, and PowerShell completion. Where latency permits, +complete contexts, local pipeline files, run IDs, component IDs, and supported +conditions. Dynamic completion must use short timeouts and fail silently rather +than hanging the shell. + +### Examples + +- Keep a tiny deterministic process-mode pipeline for first success. +- Validate every checked-in pipeline in CI. +- Exercise the canonical init/validate/plan/run/wait/inspect workflow. +- Keep network-dependent live Stellar tests separate and report exact failure. +- Clearly label experimental driver examples. + +## Agent-friendly operation + +Agents should interact through typed, discoverable contracts rather than parse +terminal prose. Required foundations: + +- stable JSON/YAML schemas; +- typed errors and exit codes; +- capability discovery; +- pipeline schema discovery; +- plan output describing side effects; +- idempotency keys; +- wait and event cursors; +- noninteractive operation; +- explicit confirmation and permission requirements; +- bounded queries and pagination; +- correlation IDs; +- redacted diagnostics. + +An MCP adapter can be built on the application client after these contracts +stabilize. Do not make MCP the domain model. + +## TUI and dashboard guidance + +The existing dashboard or a future Bubble Tea TUI should consume the same +queries, mutations, and event stream as the ordinary CLI. Its update loop must +not contain independent orchestration rules. + +Build richer interfaces only after: + +- stable snapshot queries exist; +- event cursors solve reconnect races; +- state transitions are centralized; +- machine errors are typed; +- terminal output does not leak into data contracts. + +A small run dashboard is then a useful architecture test, not the architecture +itself. + +## Explicit deferrals + +Defer these until the flagship process runtime and contracts above are solid: + +- arbitrary DAG scheduling; +- generalized resource-method invocation; +- Kubernetes/operator reconciliation; +- equally supported Docker and process drivers; +- hosted multi-tenant flowctl; +- full event sourcing as the primary state store; +- automatic infinite component restart; +- cross-region control-plane clustering; +- dynamic in-process plugins; +- a GUI-specific backend; +- billing and organization administration. + +## Suggested pull-request sequence + +Keep contract work separate from internal refactors and land one independently +verifiable capability per PR. + +1. Internal run service and injected CLI renderers +2. Stable error taxonomy and exit codes +3. Machine-output inventory and versioned output schemas +4. Shared embedded/gRPC client contract tests +5. Explicit run and chunk state-transition validators +6. `pipeline wait` +7. Durable lifecycle event storage/outbox +8. Event list API and CLI +9. Event follow with cursors and reconnect tests +10. Idempotent start/stop semantics +11. Pipeline plan and definition digest +12. Capability discovery +13. `flowctl doctor` +14. Effective configuration/context commands +15. Schema discovery and dynamic completion +16. Optional dashboard/TUI migration to the shared client + +Do not combine protobuf contract migration, command renaming, and runtime +refactoring in one PR. + +## Verification matrix + +| Area | Required verification | +|---|---| +| Go implementation | `go test ./...`, `go vet ./...`, `go build ./...` | +| State machine | table-driven legal/illegal/idempotent transition tests | +| Storage | memory/Bolt parity, migration, restart, and injected failure tests | +| CLI | exit-code table, stdout/stderr isolation, golden human output where useful | +| Machine output | schema tests, enum stability, unknown-field compatibility | +| Embedded/remote parity | shared client/service contract suite | +| Wait | completed, failed, stopped, timeout, not-found, interruption | +| Events | atomicity, ordering scope, duplicates, cursor resume, retention expiry | +| Process runtime | bounded success, every edge failure, fan-out, cancellation, no goroutine leak | +| Security | redaction, TLS/auth configuration, unauthorized mutation tests | +| Pipeline definitions | validate all checked-in examples and generated presets | +| Cross-repository | tagged flowctl-sdk/source/processor/sink process E2E | +| Live Stellar gate | bounded archive parity before unbounded tip/reconnect testing | + +Tests requiring localhost listeners must run in an environment that permits +binding ephemeral ports. Network-dependent tests must report exact failures and +must not be silently treated as passing. + +## Release gates + +Before declaring the programmable resource model stable: + +- publish compatibility policy for CLI JSON and control-plane protobufs; +- publish supported flowctl/flowctl-sdk/flow-proto version combinations; +- run the full synthetic process E2E; +- run a bounded real Stellar source-to-durable-sink parity test; +- inject source, processor, sink, persistence, and control-plane failures; +- prove restart retains terminal state and events; +- prove event reconnect from a cursor has no gaps; +- verify no secrets appear in plans, errors, events, or persisted run records; +- verify documentation and completion from release artifacts; +- retain a rollback path for storage migrations. + +## Definition of success + +This roadmap is successful when: + +- the CLI, embedded mode, and remote client invoke the same application rules; +- pipelines, runs, components, chunks, operations, and events have clear + identities and lifecycle semantics; +- automation can plan, start, wait, inspect, follow events, and stop without + parsing human text or sleeping arbitrarily; +- retries do not create accidental duplicate runs; +- current state and event subscriptions can be combined without missing + changes; +- failures remain structured, redacted, durable, and attributable to an edge; +- the process runtime remains the trustworthy flagship path; +- a dashboard can be built as another client without duplicating orchestration; +- nebu remains the simple prototyping layer and flowctl remains the production + lifecycle owner. + diff --git a/docs/adr/0000-template.md b/docs/adr/0000-template.md new file mode 100644 index 0000000..8f8623b --- /dev/null +++ b/docs/adr/0000-template.md @@ -0,0 +1,63 @@ +--- +status: proposed +date: YYYY-MM-DD +decision-makers: +consulted: +informed: +--- + +# NNNN. Short title naming the decision + +## Context and problem statement + +What is the situation, and what question does it force? Two or three paragraphs. +Link the issue, the standard rule ID, or the gap this addresses. + +## Decision drivers + +- What matters in choosing, in priority order +- Constraints that rule options in or out + +## Considered options + +1. **Option A** — one line +2. **Option B** — one line +3. **Option C** — one line + +## Decision outcome + +Chosen option: **Option B**, because … + +### Consequences + +- Good: … +- Bad: … +- Neutral: … + +### Contract impact + +Only for decisions that change a contract (see `docs/engineering-profile.md` §2). + +| | | +|---|---| +| Affected interfaces | | +| Change class | Additive / Deprecating / Breaking / Semantic | +| Migration window | | +| Rollback behaviour | | +| Tests proving it | | + +## Pros and cons of the options + +### Option A + +- Good: … +- Bad: … + +### Option B + +- Good: … +- Bad: … + +## More information + +Links, follow-up ADRs, the PR that implemented this. diff --git a/docs/adr/0001-cli-output-and-exit-code-contract.md b/docs/adr/0001-cli-output-and-exit-code-contract.md new file mode 100644 index 0000000..29db144 --- /dev/null +++ b/docs/adr/0001-cli-output-and-exit-code-contract.md @@ -0,0 +1,130 @@ +--- +status: accepted +date: 2026-10-20 +decision-makers: Tillman Mosley III +consulted: +informed: +--- + +# 0001. Adopt an exit-code vocabulary and a machine-readable output contract + +## Context and problem statement + +flowctl advertised a global `--output table|yaml|json|wide` flag at `cmd/root.go` that no +command ever read. Every command printed hand-formatted tables with `fmt.Printf`, mixing +results, progress and hints on stdout. Errors all exited 1. + +That makes flowctl unusable from a script or an agent: there is nothing to parse and +nothing to branch on. It also violates four MUSTs of the OBSRVR Engineering Standard +v2.0 — CLI-5 (stdout carries results only), CLI-6 (JSON and NDJSON with a schema +identifier), CLI-15 (the exit-code table) and CLI-22 (escape-sequence sanitisation) — +recorded as gaps GAP-2, GAP-3, GAP-9 and GAP-18 in `docs/engineering-profile.md`. + +The standard treats JSON output and exit codes as public contracts, as binding as an +API. flowctl therefore needs to define them deliberately, once, rather than letting each +command invent its own. + +## Decision drivers + +- A script must be able to branch on *why* a command failed, not just that it did. +- `flowctl … | jq` must always work, with no flag juggling. +- The shape of machine-readable output must be versioned from the first release that + emits it, so later changes are classifiable (VER-3). +- Component-supplied strings reach the terminal; they are untrusted input. +- The conversion must be incremental. Converting 14 commands in one change is not + reviewable. + +## Considered options + +1. **Per-command JSON marshalling** — each command marshals its own struct with + `encoding/json`. +2. **One output package all commands render through**, owning format selection, schema + identifiers, stream separation, colour suppression and sanitisation. +3. **Adopt an existing CLI framework's output layer** (e.g. kubectl's printers). + +## Decision outcome + +Chosen option: **Option 2** — three small internal packages: + +- `internal/exitcode` — the exit-code vocabulary and error→code mapping. +- `internal/sanitize` — control-character stripping for untrusted strings. +- `internal/cliout` — a `Printer` that renders results as table, JSON or NDJSON, + writes results to stdout and everything else to stderr, and suppresses colour. + +Exit codes follow CLI-15 exactly, with no domain-specific additions yet: + +| Code | Meaning | +|---|---| +| 0 | Success | +| 1 | Operation failed | +| 2 | Usage error | +| 3 | Resource not found | +| 4 | Not authenticated or not permitted | +| 5 | Conflict or failed precondition | +| 6 | Timed out waiting | +| 7 | Dependency unavailable | +| 8 | Data validation failed | +| 130 / 143 | Interrupted (SIGINT / SIGTERM) after clean shutdown | + +Codes 10–63 are reserved for domain-specific additions and must be documented here +when first used. + +Machine-readable output is enveloped. `--output json` emits one document: + +```json +{"schema": "flowctl.service_list.v1", "items": [ … ]} +``` + +`--output ndjson` emits one object per line, each carrying its own schema identifier, +flushed per line. Schema names are `flowctl..v1`, singular for an object and +`_list` for a collection. Human table output is explicitly *not* a contract and +may change at any time. + +Option 1 was rejected because it guarantees divergence: fourteen commands would produce +fourteen envelope conventions, and sanitisation would be forgotten in most of them. +Option 3 was rejected as far more dependency weight than a tool this size needs, and it +would not give us the schema-identifier requirement for free anyway. + +### Consequences + +- Good: one place to change when the standard's output rules change; one place that + can be tested for CLI-5, CLI-7 and CLI-22 compliance for every command at once. +- Good: `exitcode.Of` maps gRPC status codes and `context.DeadlineExceeded` + centrally, so "control plane unreachable" becomes exit 7 everywhere without each + command knowing. +- Bad: commands not yet converted still print ad-hoc output. The conversion is + incremental and the gap stays open until it finishes. Converted commands are listed + in `docs/engineering-profile.md`. +- Bad: `--output yaml|wide` were advertised and are not implemented. They are now + rejected with a usage error rather than silently ignored, which is a visible + behaviour change for anyone who passed them. + +### Contract impact + +| | | +|---|---| +| Affected interfaces | CLI exit codes; `--output` on all converted commands; `init --output`; `processors find --output` | +| Change class | **Breaking** | +| Migration window | None. flowctl is pre-1.0 (v0.0.12); VER-2 permits a minor bump to break. | +| Rollback behaviour | Revert the PR. No persisted state changes format. | +| Tests proving it | Golden tests per schema in `internal/cliout`; exit-code mapping tests in `internal/exitcode`; injection tests in `internal/sanitize`. | + +Specific breaks, all recorded in `CHANGELOG.md`: + +- Commands that previously exited 1 for every failure now exit 2–8 where the cause is + known. A script testing `$? -eq 1` must be updated to test `$? -ne 0`. +- `flowctl init --output ` is renamed to `--file`/`-f`, because CLI-3 forbids + reusing a shared option name for a different meaning. +- `flowctl processors find --output ` is renamed to `--output-type` for the same + reason. +- Neither rename keeps a deprecated alias. A local `--output` shadows the global one, + so an alias would make `--output json` silently mean "write to a file named json" or + "filter for the event type json" — the precise confusion CLI-3 exists to prevent, and + worse than a clean break. VER-2 permits breaking on a minor bump before 1.0. +- `--output yaml` and `--output wide` now error instead of being ignored. + +## More information + +- OBSRVR Engineering Standard v2.0 §3, rules CLI-3, CLI-5, CLI-6, CLI-7, CLI-15, CLI-22. +- `docs/engineering-profile.md` gaps GAP-2, GAP-3, GAP-9, GAP-16, GAP-17, GAP-18. +- Superseded by: none. diff --git a/docs/adr/0002-defer-protobuf-package-versioning.md b/docs/adr/0002-defer-protobuf-package-versioning.md new file mode 100644 index 0000000..4f30eb5 --- /dev/null +++ b/docs/adr/0002-defer-protobuf-package-versioning.md @@ -0,0 +1,98 @@ +--- +status: accepted +date: 2026-10-20 +decision-makers: Tillman Mosley III +consulted: +informed: +--- + +# 0002. Defer the protobuf package rename to `flowctl.v1` + +## Context and problem statement + +flowctl's protobuf files declare `package flowctl`, with no version qualifier. The +OBSRVR Engineering Standard v2.0 requires every record type to have a named, versioned +schema (DATA-5), and buf's own style guidance — which VER-5 adopts through +`buf breaking` — expects a version suffix on the package. + +Renaming `flowctl` to `flowctl.v1` changes the fully-qualified name of every message +and the gRPC path of every method (`/flowctl.ControlPlane/Register` becomes +`/flowctl.v1.ControlPlane/Register`). It is a hard wire break: every deployed component, +every build of flowctl-sdk, and every third-party processor stops talking to the control +plane at the moment one side is upgraded. + +This is recorded as gap GAP-7. + +## Decision drivers + +- The rename is correct and should happen before 1.0. +- flowctl has no version-matrix tests for rolling deploys (GAP-20, and the gap the + standard itself names for this repo under VER-9). We cannot currently prove a + coordinated upgrade works. +- flowctl-sdk and the published processor images are versioned and released separately; + a wire break needs their release trains lined up. +- The value of the rename is forward-compatibility hygiene. Nothing is broken today by + the unversioned name. + +## Considered options + +1. **Rename now**, in the same PR as the rest of the conformance work. +2. **Rename now behind a dual-registration shim** — serve both the old and new service + paths for a deprecation window. +3. **Defer**, add `buf` lint and breaking-change checking against the current package + name, and do the rename as its own coordinated change before 1.0. + +## Decision outcome + +Chosen option: **Option 3, defer.** + +`buf.yaml` and `buf.gen.yaml` land now, and CI runs `buf lint` and `buf breaking` +against the last release. The `PACKAGE_VERSION_SUFFIX` lint rule is explicitly excluded, +with a comment pointing at this ADR, so the exclusion is visible to anyone reading the +config rather than silently absent. + +Option 1 was rejected because we would be making a wire-breaking change at the same +time as a large CLI change, with no version-matrix test to catch the fallout. Option 2 +was rejected as the worst of both: the shim is real work, it doubles the surface to +test, and it would itself need removing later — all to avoid a coordinated release we +have to do anyway. + +### Consequences + +- Good: `buf lint` and `buf breaking` start protecting the proto files immediately, + which is the larger share of the value. +- Good: the rename gets done once, deliberately, with the version-matrix tests that + VER-9 requires already in place. +- Bad: the repo stays non-conformant with DATA-5's versioned-schema requirement until + then. This exception is the reason that is acceptable rather than an oversight. +- Neutral: `reserve` statements and field-number discipline are enforced from now + regardless of the package name. + +### Contract impact + +| | | +|---|---| +| Affected interfaces | `proto/common.proto`, `control_plane.proto`, `source.proto`, `processor.proto`, `sink.proto` | +| Change class | None yet — this ADR defers a Breaking change | +| Migration window | To be set by the superseding ADR; must precede 1.0 | +| Rollback behaviour | N/A | +| Tests proving it | `buf breaking` in CI against the last release tag | + +## Exception record (CONF-1) + +| | | +|---|---| +| Rule | DATA-5 (versioned schema name) | +| Reason | Wire-breaking; requires coordinated release with flowctl-sdk and deployed components | +| Risk | Low. The unversioned name causes no failure today; it forecloses a clean v2 later. | +| Compensating control | `buf lint` and `buf breaking` in CI from this PR; the rename is a release blocker for 1.0 | +| Owner | Tillman Mosley III | +| Expiry | 2027-04-20 | + +## More information + +- OBSRVR Engineering Standard v2.0, DATA-5, VER-5, VER-9, CONF-1. +- `docs/engineering-profile.md` gaps GAP-7, GAP-20. +- The rename should also carry the event-envelope fields in GAP-4 (network, ledger, + correlation ID, cursor), since both are wire-breaking and should cost one migration + rather than two. diff --git a/docs/docker-and-translation.md b/docs/docker-and-translation.md index bf2ce54..a72dfc5 100644 --- a/docs/docker-and-translation.md +++ b/docs/docker-and-translation.md @@ -21,19 +21,19 @@ flowctl can translate pipeline configurations to different deployment formats: ```bash # Translate a pipeline to Docker Compose -./bin/flowctl translate -f examples/docker-pipeline.yaml -o docker-compose +./bin/flowctl translate -f examples/docker-pipeline.yaml -t docker-compose # Save the output to a file -./bin/flowctl translate -f examples/docker-pipeline.yaml -o docker-compose --to-file docker-compose.yml +./bin/flowctl translate -f examples/docker-pipeline.yaml -t docker-compose --to-file docker-compose.yml # Add a resource prefix for naming consistency -./bin/flowctl translate -f examples/docker-pipeline.yaml -o docker-compose --prefix myapp +./bin/flowctl translate -f examples/docker-pipeline.yaml -t docker-compose --prefix myapp # Specify a container registry -./bin/flowctl translate -f examples/docker-pipeline.yaml -o docker-compose --registry ghcr.io/myorg +./bin/flowctl translate -f examples/docker-pipeline.yaml -t docker-compose --registry ghcr.io/myorg # Generate local execution script/config -./bin/flowctl translate -f examples/local-pipeline.yaml -o local --to-file run_pipeline.sh +./bin/flowctl translate -f examples/local-pipeline.yaml -t local --to-file run_pipeline.sh ``` Supported output formats: @@ -57,7 +57,7 @@ The Docker Compose-based local generator creates: ```bash # Generate Docker Compose configuration for local execution -./bin/flowctl translate -f examples/local-pipeline.yaml -o local --to-file docker-compose.yaml +./bin/flowctl translate -f examples/local-pipeline.yaml -t local --to-file docker-compose.yaml # Start the pipeline docker compose --profile local up -d @@ -75,7 +75,7 @@ For compatibility with older workflows, you can still use the bash script genera ```bash export FLOWCTL_LOCAL_GENERATOR_TYPE=bash -./bin/flowctl translate -f examples/local-pipeline.yaml -o local --to-file run_pipeline.sh +./bin/flowctl translate -f examples/local-pipeline.yaml -t local --to-file run_pipeline.sh chmod +x run_pipeline.sh ./run_pipeline.sh ``` diff --git a/docs/engineering-profile.md b/docs/engineering-profile.md new file mode 100644 index 0000000..bf28438 --- /dev/null +++ b/docs/engineering-profile.md @@ -0,0 +1,195 @@ +# flowctl — Engineering Profile + +Profile for [OBSRVR Engineering Standard](https://github.com/withObsrvr/ways-of-working) **v2.0**. + +| | | +|---|---| +| Repo | `github.com/withobsrvr/flowctl` | +| Owner | Tillman Mosley III (sole maintainer) | +| Current release | `v0.0.12` (pre-1.0) | +| Licence | MPL-2.0 (`LICENSE`) | +| Profile last reviewed | 2026-10-20 | +| Conformance assessed at | branch `chore/engineering-standard-conformance` | +| Next review | 2027-01-20 (quarterly, CONF-2) | + +--- + +## 1. System classes and applicable sections + +flowctl is **two system classes at once**: the `flowctl` binary is a CLI tool, and the +control plane it embeds (and serves standalone via `flowctl server`) is a network service. +Both sets of sections apply; where they overlap, the stricter reading wins. + +| Class | Applies | +|---|---| +| Always (§1) | **2, 7, 9, 12** | +| CLI tool | **3, 4, 6, 8, 10, 11** | +| Network service | **4, 5, 6, 8, 10** | + +Union: **§2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12.** Every section of the standard applies to +this repo except §13 (this document) and §14 (conformance process). + +§11 applies narrowly. flowctl orchestrates components; it does not decode XDR itself. +STL-1 binds the *components* flowctl runs, and flowctl's obligation is to carry and +surface their spec declarations (see rule FC-3 below), not to make them. + +--- + +## 2. Contracts owned (VER-1) + +Every contract below is owned by Tillman Mosley III until the team grows. No contract has +a recorded supported-consumer range or deprecation policy yet — closing that is GAP-5. + +| Contract | Source of truth | Version | Consumers | Status | +|---|---|---|---|---| +| CLI grammar, flags and arguments | `cmd/**`, cobra definitions | v0.0.12 | Operators, CI jobs, `obsrvr` umbrella CLI | Unstable; pre-1.0 | +| CLI exit codes | `internal/exitcode` | v1 ([ADR 0001](adr/0001-cli-output-and-exit-code-contract.md)) | Any script branching on failure | Live | +| CLI `--output json` / NDJSON shapes | `internal/cliout` schema ids | `flowctl..v1` | Scripts, agents | Live — every record-producing command converted | +| Pipeline config format (`apiVersion: flowctl/v1`) | `schemas/cue/schema.cue` | `flowctl/v1` | Pipeline authors, `translate`, `run`, `validate` | Live; enforced by `internal/apiversion` on every path | +| Control plane gRPC API | `proto/control_plane.proto` (package `flowctl`) | unversioned | Components, SDK, `flowctl` subcommands | Live; package not version-qualified (GAP-7) | +| Event envelope | `proto/common.proto` → `Event` | `schema_version` field, per-event | Sources, processors, sinks | Live; missing required fields (GAP-4) | +| Component service contracts | `proto/source.proto`, `processor.proto`, `sink.proto` | unversioned | flowctl-sdk, all components | Live | +| Embedded run-history store | `internal/storage/bolt.go` buckets `services`, `pipeline_runs`, `chunk_runs` | **no format version recorded** | `flowctl pipelines`, `chunks`, `status` | Live; upgrade-unsafe (GAP-8) | +| Translated deployment output | `internal/translator`, `internal/generator` | tracks CLI version | docker-compose / local runners | Live | + +**Not owned by this repo:** the `stellar.v1.RawLedger` schema (see the +stellar-raw-ledger-origin profile), and any CAP/SEP decoding, which belongs to the +processors flowctl runs. + +--- + +## 3. Product-specific rules + +These tighten the standard for flowctl. They are rules, not aspirations — a change that +breaks one needs an ADR. + +- **FC-1 Standalone ingestion never depends on the control plane.** A pipeline that has + started MUST continue to run, process and checkpoint when the control plane is + unreachable. Loss of the control plane degrades observability, never delivery. This is + the load-bearing rule of the whole repo (P-8) and every change to + `internal/runner` and `pkg/console/heartbeat` is reviewed against it. +- **FC-2 Events carry provenance.** Every event crossing a component boundary MUST carry + its type and version, the network, the ledger it derives from, a correlation ID and a + resumable cursor (DATA-6, DATA-7). Carried in the envelope, not in free-form metadata. +- **FC-3 flowctl relays component spec declarations verbatim.** When a component declares + the CAP or SEP it implements (STL-1), flowctl MUST surface that declaration unaltered + through registration, `processors show` and machine-readable output. flowctl MUST NOT + infer, default or normalise it. +- **FC-4 Two orchestrators, one behaviour.** The process and container orchestrators MUST + produce the same component lifecycle, the same run states and the same exit behaviour. + Any divergence is a bug, and a fix needs a test covering both (extends TEST-2.5 parity + to the orchestrator axis). +- **FC-5 The control plane is a trust boundary.** It accepts registrations from processes + flowctl did not start. Everything it receives — service names, metadata, relayed errors + — is untrusted input and MUST be sanitised before it reaches a terminal (CLI-22), a log + or persisted state. Commands rendering through `internal/cliout` get this at the point a + cell is formatted; commands not yet converted call `sanitize.Message` on each + control-plane response, and MUST drop that call only when they move to `cliout`. +- **FC-6 Run IDs are stable and public.** A run ID, once issued, identifies that run for + the life of the store and appears in every surface that mentions the run (CLI-4). + +--- + +## 4. Exceptions (§14) + +### Recorded + +| Rule | ADR | Reason | Owner | Expiry | +|---|---|---|---|---| +| DATA-5 (versioned schema name) | [0002](adr/0002-defer-protobuf-package-versioning.md) | Renaming the protobuf package `flowctl` → `flowctl.v1` breaks the wire for every deployed component at once; it needs a coordinated release with flowctl-sdk and the version-matrix tests of VER-9. `buf lint`/`buf breaking` are in CI meanwhile. | Tillman Mosley III | 2027-04-20 | + +### Candidates, not yet recorded + +Each needs an accepted ADR before it counts as an exception under CONF-1. Until then +the underlying rule is simply unmet. + +| Candidate | Rule | Argument | Proposed expiry | +|---|---|---|---| +| No OSC terminal signalling | CLI-17–19 | SHOULDs, not MUSTs; the protocol is a draft at rev 0.4 and the shared package CLI-17 requires does not exist yet in any OBSRVR repo. Build it once, adopt everywhere. | 2027-04-20 | +| Makefile instead of justfile | BUILD-1 | The standard grandfathers existing Makefiles until the repo is next reworked. `nix develop` + `make test` already gets clone-to-tests in two commands. | 2027-04-20 | + +CLI-22 (escape injection) was a candidate in the first draft of this profile and is +not one any more: it is closed. + +--- + +## 5. Conformance status + +Assessed 2026-10-20 against standard v2.0. ✓ met · ◐ partial · ✗ unmet. + +| § | Area | Status | Summary | +|---|---|---|---| +| 2 | Principles | ✓ | Architecture follows P-2, P-3, P-5, P-8 well. Headless core in `internal/`, adapters in `cmd/`, components as separate processes over documented gRPC. | +| 3 | CLI | ◐ | CLI-3, 5, 6, 7, 11, 15, 22 met across every command. CLI-8 now holds too: `init` refuses to prompt when stdin is not a terminal. CLI-1 (grammar), 12 (operation ids, `wait`, `cancel`) and 13 (`doctor`) partial; CLI-14, 17–21 unmet. | +| 4 | Data contracts | ◐ | Proto and CUE are authoritative (DATA-5 ✓). Event envelope lacks network, ledger, correlation ID and cursor (DATA-6, DATA-7 ✗). | +| 5 | Network APIs | ✗ | No health service, readiness or metrics (API-13). Errors carry no stable code or request ID (API-3). mTLS available but off by default (API-11 ◐). | +| 6 | Versioning | ◐ | `buf lint`/`buf breaking` in CI, ADRs in place, changelog rewritten, VER-7 enforced. Tags still unsigned until the next release runs the new workflow (VER-12). Version matrix still missing (VER-9). | +| 7 | Security | ◐ | `govulncheck` and `gitleaks` now gate the merge (SEC-6, SEC-9); signing and SBOM are wired into the release workflow and take effect on the next tag (SEC-10); `LICENSE` and `SECURITY.md` published (SEC-11, SEC-12). The control plane still has no authn/authz — the largest remaining gap (SEC-1–4, GAP-12). | +| 8 | Observability | ✗ | Logs are now JSON off-terminal with `service`, `version` and RFC 3339 UTC timestamps (OBS-1 ◐; still no `trace_id`/`network`/`ledger`). No OpenTelemetry (OBS-2). flowctl still exports no metrics (OBS-3) — the second-largest gap. | +| 9 | Testing | ◐ | Merge gate now complete except change classification, which a PR template prompts for. `-race` runs in CI (TEST-4) and found a real race on first run. Fuzz tests cover the sanitiser and `apiVersion` (TEST-3, partial). Still no real-chain fixtures (TEST-1) and no parity, replay or benchmark tests (TEST-2 levels 5–8). | +| 10 | Builds | ◐ | `toolchain` pinned in `go.mod`; builds use `-trimpath` and embed only the commit date, verified by `make verify-repro` in the release gate (BUILD-3). Containers run non-root with OCI annotations (BUILD-4). CI still uses `setup-go` rather than the flake (BUILD-2, GAP-21). | +| 11 | Stellar | ◐ | Correctly delegated to components. No pubnet default anywhere ✓ (BUILD-7). FC-3 relay path not yet machine-readable, pending GAP-2. | +| 12 | Documentation | ◐ | DOC-1 met. ADRs in place (DOC-4). CLI reference still hand-maintained in the README (DOC-2) and `docs/` still mixes tutorials, reference and internal planning (DOC-3). | + +### Gap register + +Ordered by the standard's own adoption sequence (§14). **Closed** entries are kept so +the register reads as a history rather than only a backlog. + +| ID | Gap | Rule | Phase | Status | +|---|---|---|---|---| +| GAP-1 | No `LICENSE`, `SECURITY.md`, `AGENTS.md`, `CONTRIBUTING.md` | DOC-1, SEC-11, SEC-12 | 1 | **Closed** | +| GAP-2 | `--output` declared and read nowhere; no JSON, NDJSON or schema identifiers | CLI-6 | 2 | **Closed** — every record-producing command emits json/ndjson with a schema id. `dashboard` and `run`/`server` produce no records and reject or ignore machine formats accordingly | +| GAP-3 | Only exit codes 0 and 1; clean signal shutdown exited 0 | CLI-15, CLI-11 | 2 | **Closed** | +| GAP-4 | `Event` envelope has no network, ledger, correlation ID or cursor | DATA-6, DATA-7, FC-2 | 3 | Open — wire-breaking; pair with GAP-7 | +| GAP-5 | No contract owners or change classification; no ADR directory | VER-1, VER-3, VER-8, DOC-4 | 1 | **Closed** — owners in §2, ADRs in `docs/adr/`, classification in the PR template | +| GAP-6 | Three disagreeing `apiVersion` validators; `flowctl run` accepted `v99` | VER-7 | 2 | **Closed** — `internal/apiversion`, with a test asserting the Go pattern matches the CUE schema | +| GAP-7 | Proto package is `flowctl`, not `flowctl.v1`; no buf | DATA-5, VER-5, API-1 | 2 | **Partial** — `buf lint`/`buf breaking`/generated-code check in CI; rename deferred under [ADR 0002](adr/0002-defer-protobuf-package-versioning.md) | +| GAP-8 | BoltDB store records no format version, so an upgrade cannot migrate | VER-7 | 3 | Open | +| GAP-9 | No control-character sanitisation of component-supplied strings | CLI-22, FC-5 | 2 | **Closed** — `internal/sanitize` for formatted values plus `sanitize.Message` at every command's control-plane ingress, fuzzed, and verified end-to-end against a service registered under a hostile name | +| GAP-10 | Merge gate missing gofmt, staticcheck, govulncheck, secret scanning, `-race`, buf | §9 merge gate | 2 | **Closed** — found a real data race on its first run | +| GAP-11 | Unsigned tags, no SBOM or provenance, non-reproducible builds | VER-12, SEC-10, BUILD-3 | 5 | **Partial** — `-trimpath`, commit-date stamping and `make verify-repro` done; signing and SBOM wired into the release workflow but unproven until the next tag | +| GAP-12 | Control plane has no authentication, authorization, actor identity or audit log | SEC-1–4 | 4 | Open — **largest remaining gap** | +| GAP-13 | No gRPC health service, readiness endpoint, or stable error codes/request IDs | API-13, API-3 | 3 | Open | +| GAP-14 | No metrics: no last-ledger, lag, backlog, checkpoint or gap-count signals | OBS-3, OBS-4 | 3 | Open — blocks Lake's freshness/completeness SLOs | +| GAP-15 | `viper.AutomaticEnv` with no prefix; no XDG support; no `config show` | CLI-9 | 2 | **Closed** | +| GAP-16 | `--output` reused for two unrelated meanings | CLI-3 | 2 | **Closed** — renamed to `--file` and `--output-type` | +| GAP-17 | No colour suppression: no `NO_COLOR`, `TERM=dumb` or TTY detection | CLI-7 | 2 | **Closed** — logger, `cliout`, and `status` (which emitted raw ANSI and a screen-clear unconditionally). `dashboard` relies on lipgloss's own detection. | +| GAP-18 | Progress and banners written to stdout alongside results | CLI-5 | 2 | **Closed** — no bare `fmt.Print*` to stdout remains outside one TTY-gated screen clear; `run`, the live monitor and the component downloaders all write progress to stderr | +| GAP-19 | No real-chain fixtures, fuzzing, golden, subprocess, parity or replay tests | TEST-1, TEST-2, TEST-3 | 3 | **Partial** — fuzz tests for the sanitiser and `apiVersion`; golden tests for the output envelopes; TEST-1 and TEST-2 levels 5–8 still open | +| GAP-20 | No version-matrix tests for rolling deploys *(the gap §13 of the standard names)* | VER-9 | 3 | Open | +| GAP-21 | No `toolchain` line; CI uses `setup-go` rather than the repo flake | BUILD-2 | 5 | **Partial** — `toolchain` pinned; CI still does not use the flake | +| GAP-22 | Containers run as root; no OCI annotations | BUILD-4 | 5 | **Closed** | +| GAP-23 | No MCP server or JSON help for agent operation | CLI-14 | 6 | Open | +| GAP-24 | Packages lack package comments (`ST1000` excluded in `staticcheck.conf`) | §9 linters | 2 | Open — mechanical, ~20 packages | +| GAP-25 | `grpc.DialContext`/`WithBlock` deprecated (`SA1019` excluded) | §9 linters | 2 | Open — `NewClient` never blocks, so call sites relying on `WithBlock` change behaviour | + +### Next gap to close + +**GAP-13 and GAP-14 together** — the gRPC health service and the first metrics. + +These are what make flowctl operable rather than merely runnable. GAP-14 is the one with +consequences outside this repo: without last-ledger, lag-behind-tip, backlog and +gap-count signals, Obsrvr Lake cannot publish the freshness and completeness SLOs OBS-4 +requires of it. They belong together because both are served from the same place — a +metrics listener alongside the health service, on a port that is not the gRPC port +(API-13). + +After that, in order: + +1. **GAP-12** — control plane authn/authz. The biggest single gap, and the one needing a + design decision first: workload identity versus per-caller keys, and where the + read/operate split falls. That decision is an ADR, not a patch. +2. **GAP-4 with GAP-7** — the event envelope fields and the `flowctl.v1` rename. Both are + wire-breaking, so they should cost one coordinated migration rather than two. +3. **GAP-19 and GAP-20** — real-ledger fixtures and the version-matrix tests. VER-9 is + the gap §13 of the standard names for this repo, and it gates any confident rollout + of the migration above. + +## 6. Revision history + +| Date | Change | +|---|---| +| 2026-10-20 | Initial profile against standard v2.0. Conformance assessed from a repo audit at `3c954ab`. | +| 2026-10-20 | Updated after the conformance branch: GAP-1, 3, 5, 6, 9, 10, 15, 16, 17, 22 closed; GAP-2, 7, 11, 18, 19, 21 partial; GAP-24 and GAP-25 added for the two linter exclusions. | +| 2026-10-20 | Output conversion finished: GAP-2 and GAP-18 closed. CLI-8 met by the `init` TTY guard. Next gap is now GAP-13/GAP-14. | diff --git a/docs/init-command.md b/docs/init-command.md index b88ab98..fc7962e 100644 --- a/docs/init-command.md +++ b/docs/init-command.md @@ -60,7 +60,7 @@ flowctl init --preset testnet-duckdb flowctl init --non-interactive --network testnet --destination duckdb # Custom output file -flowctl init --non-interactive --network mainnet --destination postgres -o prod-pipeline.yaml +flowctl init --non-interactive --network mainnet --destination postgres -f prod-pipeline.yaml ``` ## Examples @@ -109,7 +109,7 @@ spec: ### Create a Mainnet PostgreSQL Pipeline ```bash -flowctl init --preset mainnet-postgres -o mainnet.yaml +flowctl init --preset mainnet-postgres -f mainnet.yaml ``` ## Component Auto-Download @@ -232,7 +232,7 @@ After creating your first pipeline: 1. **Run it**: `./bin/flowctl run stellar-pipeline.yaml` 2. **Monitor it**: `./bin/flowctl dashboard` 3. **Add processors**: Edit the YAML to add transformation stages -4. **Deploy to production**: `./bin/flowctl translate -f pipeline.yaml -o docker-compose` +4. **Deploy to production**: `./bin/flowctl translate -f pipeline.yaml -t docker-compose` ## See Also diff --git a/docs/processor-discovery.md b/docs/processor-discovery.md index e8b9dd0..9881fe1 100644 --- a/docs/processor-discovery.md +++ b/docs/processor-discovery.md @@ -82,7 +82,7 @@ flowctl processors find [flags] ./bin/flowctl processors find --input stellar.ledger.v1 # Find processors that produce token.transfer events -./bin/flowctl processors find --output stellar.token.transfer.v1 +./bin/flowctl processors find --output-type stellar.token.transfer.v1 # Find processors for a specific network ./bin/flowctl processors find --metadata network=testnet diff --git a/examples/getting-started/README.md b/examples/getting-started/README.md index 9d6b89b..60ff63d 100644 --- a/examples/getting-started/README.md +++ b/examples/getting-started/README.md @@ -166,7 +166,7 @@ Let's walk through understanding a simple pipeline. We'll use the generated star ### Step 1: Examine the Configuration ```bash -flowctl init --preset testnet-duckdb -o /tmp/stellar-pipeline.yaml +flowctl init --preset testnet-duckdb -f /tmp/stellar-pipeline.yaml cat /tmp/stellar-pipeline.yaml ``` @@ -603,7 +603,7 @@ sinks: # Translate image-based pipelines to deployment format # (the default `flowctl init` starter pipelines are meant to run directly with `flowctl run`) -./bin/flowctl translate -f image-based-pipeline.yaml -o docker-compose +./bin/flowctl translate -f image-based-pipeline.yaml -t docker-compose # Check version ./bin/flowctl version diff --git a/examples/quickstart/README.md b/examples/quickstart/README.md index 5b11b3d..185d656 100644 --- a/examples/quickstart/README.md +++ b/examples/quickstart/README.md @@ -60,7 +60,7 @@ flowctl init --preset testnet-duckdb flowctl init --non-interactive --network testnet --destination duckdb # Create a mainnet pipeline with PostgreSQL sink -flowctl init --non-interactive --network mainnet --destination postgres -o mainnet-pipeline.yaml +flowctl init --non-interactive --network mainnet --destination postgres -f mainnet-pipeline.yaml ``` ## Run the Pipeline diff --git a/go.mod b/go.mod index bba5610..bf334d1 100644 --- a/go.mod +++ b/go.mod @@ -2,21 +2,25 @@ module github.com/withobsrvr/flowctl go 1.26.1 +toolchain go1.26.9 + require ( github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 - github.com/docker/docker v28.2.2+incompatible + github.com/docker/docker v28.3.3+incompatible github.com/fatih/color v1.18.0 github.com/fsnotify/fsnotify v1.8.0 github.com/google/go-containerregistry v0.20.6 github.com/google/uuid v1.6.0 + github.com/mattn/go-isatty v0.0.20 github.com/spf13/cobra v1.10.2 github.com/spf13/viper v1.20.1 github.com/withObsrvr/flow-proto v0.1.3 + github.com/withObsrvr/flowctl-sdk v0.1.5 go.etcd.io/bbolt v1.4.0 go.uber.org/zap v1.27.0 - golang.org/x/sync v0.19.0 - google.golang.org/grpc v1.79.2 + golang.org/x/sync v0.23.0 + google.golang.org/grpc v1.83.2 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) @@ -25,7 +29,6 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect github.com/charmbracelet/x/ansi v0.10.1 // indirect @@ -34,7 +37,6 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/cli v28.2.2+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect @@ -45,13 +47,12 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -64,7 +65,6 @@ require ( github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect @@ -74,21 +74,17 @@ require ( github.com/spf13/pflag v1.0.9 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/vbatts/tar-split v0.12.1 // indirect - github.com/withObsrvr/flowctl-sdk v0.1.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect - go.opentelemetry.io/otel v1.42.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect - go.opentelemetry.io/otel/metric v1.42.0 // indirect - go.opentelemetry.io/otel/sdk v1.42.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect - go.opentelemetry.io/otel/trace v1.42.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.50.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa // indirect ) diff --git a/go.sum b/go.sum index 2f5e4ed..7822e33 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsy github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= @@ -62,8 +62,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -170,20 +170,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= -go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= -go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0 h1:wpMfgF8E1rkrT1Z6meFh1NDtownE9Ii3n3X2GJYjsaU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.33.0/go.mod h1:wAy0T/dUbs468uOlkT31xjvqQgEVXv58BRFWEgn5v/0= -go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= -go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= -go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= -go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= -go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= -go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= -go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= -go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -204,13 +204,13 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -219,12 +219,12 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -235,15 +235,15 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= -google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= -google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= -google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1:Kjn0N0tCrDgiAFW+lGO4JZ3ck44CehvJQMAwj9QF0G8= +google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/api/client.go b/internal/api/client.go index 39ce9fa..e7bfd4c 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -15,11 +15,11 @@ import ( type ClientOptions struct { // ServerAddress is the address of the control plane server. ServerAddress string - + // TLSConfig is the TLS configuration to use for the connection. // If nil, an insecure connection will be used. TLSConfig *config.TLSConfig - + // Additional gRPC dial options to use. DialOptions []grpc.DialOption } @@ -29,25 +29,25 @@ func CreateControlPlaneClient(opts *ClientOptions) (pb.ControlPlaneClient, *grpc if opts == nil { return nil, nil, fmt.Errorf("client options cannot be nil") } - + if opts.ServerAddress == "" { return nil, nil, fmt.Errorf("server address is required") } - + // Prepare dial options dialOpts := make([]grpc.DialOption, 0, len(opts.DialOptions)+1) - + // Add TLS options if configured if opts.TLSConfig != nil && opts.TLSConfig.Mode != config.TLSModeDisabled { logger.Info("Configuring TLS for client connection", zap.String("server", opts.ServerAddress), zap.String("mode", string(opts.TLSConfig.Mode))) - + tlsOpt, err := opts.TLSConfig.LoadClientCredentials() if err != nil { return nil, nil, fmt.Errorf("failed to load TLS credentials: %w", err) } - + dialOpts = append(dialOpts, tlsOpt) } else { // Use insecure credentials if TLS is disabled @@ -55,19 +55,19 @@ func CreateControlPlaneClient(opts *ClientOptions) (pb.ControlPlaneClient, *grpc zap.String("server", opts.ServerAddress)) dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - + // Add user-provided dial options dialOpts = append(dialOpts, opts.DialOptions...) - + // Connect to the server conn, err := grpc.Dial(opts.ServerAddress, dialOpts...) if err != nil { return nil, nil, fmt.Errorf("failed to connect to control plane: %w", err) } - + // Create the client client := pb.NewControlPlaneClient(conn) - + return client, conn, nil } @@ -81,6 +81,6 @@ func MustCreateControlPlaneClient(opts *ClientOptions) (pb.ControlPlaneClient, * // to satisfy the compiler panic(err) } - + return client, conn -} \ No newline at end of file +} diff --git a/internal/api/control_plane.go b/internal/api/control_plane.go index af8090b..ea76425 100644 --- a/internal/api/control_plane.go +++ b/internal/api/control_plane.go @@ -320,7 +320,11 @@ func (s *ControlPlaneServer) GetComponentStatus(ctx context.Context, req *flowct return nil, fmt.Errorf("service %s not found", req.ServiceId) } - return service.Status, nil + // Clone rather than returning the pointer. service.Status lives in the map + // guarded by s.mu, and the janitor mutates it when a heartbeat goes stale. + // Handing the live pointer to a caller lets gRPC serialise it after the lock + // is released, racing the janitor — which `go test -race` catches. + return gproto.Clone(service.Status).(*flowctlv1.ComponentStatusResponse), nil } // ListComponents implements the ListComponents RPC @@ -340,7 +344,8 @@ func (s *ControlPlaneServer) ListComponents(ctx context.Context, req *flowctlv1. continue } - components = append(components, service.Status) + // Cloned for the same reason as GetComponentStatus above. + components = append(components, gproto.Clone(service.Status).(*flowctlv1.ComponentStatusResponse)) } return &flowctlv1.ListComponentsResponse{Components: components}, nil @@ -865,7 +870,7 @@ func (w *ControlPlaneWrapper) ListServices(ctx context.Context, req *emptypb.Emp return &flowctlpb.ServiceList{Services: services}, nil } -// Pipeline run tracking methods - delegate directly to the server +// CreatePipelineRun delegates pipeline run tracking directly to the server. func (w *ControlPlaneWrapper) CreatePipelineRun(ctx context.Context, req *flowctlpb.CreatePipelineRunRequest) (*flowctlpb.PipelineRun, error) { return w.server.CreatePipelineRun(ctx, req) } diff --git a/internal/apiversion/apiversion.go b/internal/apiversion/apiversion.go new file mode 100644 index 0000000..2c7e48a --- /dev/null +++ b/internal/apiversion/apiversion.go @@ -0,0 +1,85 @@ +// Package apiversion validates the apiVersion field of a pipeline configuration. +// +// It exists because flowctl had three validators that disagreed. The CUE schema in +// schemas/cue/schema.cue enforced `^flowctl/v[0-9]+$`, while the Go loaders in +// internal/runner and internal/translator only checked the field was non-empty. A +// pipeline declaring `apiVersion: v99` was rejected by `flowctl validate` but +// accepted by `flowctl run`, which then interpreted it as v1 — exactly the silent +// reinterpretation VER-7 forbids. +package apiversion + +import ( + "fmt" + "regexp" + "strconv" + + "github.com/withobsrvr/flowctl/internal/exitcode" +) + +// Current is the apiVersion this build of flowctl writes. +const Current = "flowctl/v1" + +// maxSupportedMajor is the highest pipeline major version this build understands. +// Raising it is a contract change: see docs/adr/ and VER-3. +const maxSupportedMajor = 1 + +// pattern matches the apiVersion grammar. It must stay in step with the same +// expression in schemas/cue/schema.cue, which is the authoritative schema (DATA-5). +var pattern = regexp.MustCompile(`^flowctl/v([0-9]+)$`) + +// Validate reports whether v is an apiVersion this build supports. +// +// Errors are actionable as VER-7 requires: they name what was found, what is +// supported, and what to do about it. All of them classify as exit code 8, data +// validation failed. +func Validate(v string) error { + if v == "" { + return exitcode.Errorf(exitcode.DataInvalid, + "apiVersion is required; expected %q", Current) + } + + match := pattern.FindStringSubmatch(v) + if match == nil { + return exitcode.Errorf(exitcode.DataInvalid, + "apiVersion %q is not valid; expected %q", v, Current) + } + + major, err := strconv.Atoi(match[1]) + if err != nil { + // Unreachable while the pattern only admits digits, but a future + // loosening of the pattern should fail loudly rather than parse as 0. + return exitcode.Errorf(exitcode.DataInvalid, + "apiVersion %q has an unreadable version number", v) + } + + if major > maxSupportedMajor { + return exitcode.Errorf(exitcode.DataInvalid, + "apiVersion %q is newer than this build of flowctl supports (up to %q); upgrade flowctl", + v, Current) + } + if major < 1 { + return exitcode.Errorf(exitcode.DataInvalid, + "apiVersion %q is not a released pipeline version; expected %q", v, Current) + } + + return nil +} + +// Major returns the major version number of a valid apiVersion. +func Major(v string) (int, error) { + if err := Validate(v); err != nil { + return 0, err + } + n, _ := strconv.Atoi(pattern.FindStringSubmatch(v)[1]) + return n, nil +} + +// ValidateKind checks the resource kind alongside the apiVersion, so the two +// always fail with the same error class. +func ValidateKind(kind string) error { + if kind != "Pipeline" { + return exitcode.Errorf(exitcode.DataInvalid, + "kind must be %q, got %q", "Pipeline", fmt.Sprint(kind)) + } + return nil +} diff --git a/internal/apiversion/apiversion_test.go b/internal/apiversion/apiversion_test.go new file mode 100644 index 0000000..d8e7cf6 --- /dev/null +++ b/internal/apiversion/apiversion_test.go @@ -0,0 +1,154 @@ +package apiversion + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/withobsrvr/flowctl/internal/exitcode" +) + +func TestValidateAcceptsCurrent(t *testing.T) { + if err := Validate(Current); err != nil { + t.Fatalf("Validate(%q) = %v, want nil", Current, err) + } +} + +func TestValidateRejects(t *testing.T) { + // VER-7: an unsupported version is rejected with an actionable error, never + // silently reinterpreted. Before this package, "v1" and "v99" both loaded and + // ran as v1. + tests := []struct { + name string + in string + contains string + }{ + {"empty", "", "required"}, + {"bare v1, the old loose form", "v1", "not valid"}, + {"no group", "v1/flowctl", "not valid"}, + {"wrong group", "nebu/v1", "not valid"}, + {"no version", "flowctl/", "not valid"}, + {"non-numeric", "flowctl/vX", "not valid"}, + {"trailing content", "flowctl/v1beta", "not valid"}, + {"leading space", " flowctl/v1", "not valid"}, + {"future major", "flowctl/v2", "upgrade flowctl"}, + {"far future major", "flowctl/v99", "upgrade flowctl"}, + {"zero", "flowctl/v0", "not a released"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := Validate(tc.in) + if err == nil { + t.Fatalf("Validate(%q) = nil, want an error", tc.in) + } + if !strings.Contains(err.Error(), tc.contains) { + t.Errorf("Validate(%q) = %q, want it to mention %q", tc.in, err, tc.contains) + } + // Bad configuration is a data validation failure, not a generic one. + if got := exitcode.Of(err); got != exitcode.DataInvalid { + t.Errorf("Validate(%q) exit code = %d, want %d", tc.in, got, exitcode.DataInvalid) + } + }) + } +} + +func TestErrorsAreActionable(t *testing.T) { + // VER-7 requires the error to name the supported range or an upgrade hint. + err := Validate("flowctl/v2") + if !strings.Contains(err.Error(), Current) { + t.Errorf("error should name the supported version: %q", err) + } +} + +func TestMajor(t *testing.T) { + got, err := Major("flowctl/v1") + if err != nil || got != 1 { + t.Errorf("Major = (%d, %v), want (1, nil)", got, err) + } + if _, err := Major("v1"); err == nil { + t.Error("Major should reject an invalid apiVersion") + } +} + +func TestValidateKind(t *testing.T) { + if err := ValidateKind("Pipeline"); err != nil { + t.Errorf("ValidateKind(Pipeline) = %v", err) + } + err := ValidateKind("pipeline") + if err == nil { + t.Fatal("ValidateKind should be case-sensitive") + } + if got := exitcode.Of(err); got != exitcode.DataInvalid { + t.Errorf("exit code = %d, want %d", got, exitcode.DataInvalid) + } +} + +// TestPatternMatchesCUESchema guards the duplication between this package and the +// authoritative CUE schema. DATA-5 allows generated or checked copies of a schema, +// never hand-edited ones that can drift; this is the check that keeps them honest. +func TestPatternMatchesCUESchema(t *testing.T) { + path := filepath.Join("..", "..", "schemas", "cue", "schema.cue") + data, err := os.ReadFile(path) + if err != nil { + t.Skipf("cannot read %s: %v", path, err) + } + + // apiVersion: string & =~"^flowctl/v[0-9]+$" + re := regexp.MustCompile(`apiVersion:\s*string\s*&\s*=~"([^"]+)"`) + match := re.FindSubmatch(data) + if match == nil { + t.Fatalf("could not find the apiVersion constraint in %s; if the schema was "+ + "restructured, update this test and apiversion.pattern together", path) + } + + // The Go pattern adds a capture group around the version number so Major can + // read it. Strip it before comparing: the two must describe the same language, + // not be byte-identical. + ours := strings.NewReplacer("(", "", ")", "").Replace(pattern.String()) + if got := string(match[1]); got != ours { + t.Errorf("CUE schema constrains apiVersion to %q but this package uses %q; "+ + "they must match (DATA-5)", got, ours) + } +} + +// TestCurrentSatisfiesPattern catches a Current constant that was bumped without +// widening maxSupportedMajor. +func TestCurrentSatisfiesPattern(t *testing.T) { + major, err := Major(Current) + if err != nil { + t.Fatalf("Current = %q is not valid: %v", Current, err) + } + if major != maxSupportedMajor { + t.Errorf("Current is %q (major %d) but maxSupportedMajor is %d", + Current, major, maxSupportedMajor) + } +} + +func FuzzValidateNeverPanics(f *testing.F) { + // TEST-3: apiVersion comes from a user-supplied config file. + for _, seed := range []string{ + "", "flowctl/v1", "v1", "flowctl/v", "flowctl/v99999999999999999999", + "flowctl/v-1", "flowctl/v1\n", "\x00", "FLOWCTL/V1", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, in string) { + err := Validate(in) + if err == nil { + // The real invariant: anything accepted must resolve to a major + // version this build supports. It need not be spelled exactly like + // Current — the authoritative CUE pattern admits zero-padded forms + // such as "flowctl/v001", and the two validators must agree. + major, mErr := Major(in) + if mErr != nil { + t.Fatalf("Validate(%q) accepted an input Major rejects: %v", in, mErr) + } + if major < 1 || major > maxSupportedMajor { + t.Fatalf("Validate(%q) accepted unsupported major %d", in, major) + } + } + }) +} diff --git a/internal/apiversion/testdata/fuzz/FuzzValidateNeverPanics/df6407f52831dfa4 b/internal/apiversion/testdata/fuzz/FuzzValidateNeverPanics/df6407f52831dfa4 new file mode 100644 index 0000000..fb0e884 --- /dev/null +++ b/internal/apiversion/testdata/fuzz/FuzzValidateNeverPanics/df6407f52831dfa4 @@ -0,0 +1,2 @@ +go test fuzz v1 +string("flowctl/v00000000000001") diff --git a/internal/cliout/cliout.go b/internal/cliout/cliout.go new file mode 100644 index 0000000..cccd09d --- /dev/null +++ b/internal/cliout/cliout.go @@ -0,0 +1,336 @@ +// Package cliout renders command results. +// +// It exists so that the rules in §3 of the OBSRVR Engineering Standard v2.0 are +// implemented once rather than in every command: +// +// - CLI-5: stdout carries results only. Progress, hints and warnings go to stderr, +// so `flowctl … | jq` always works. +// - CLI-6: --output json emits one JSON document, --output ndjson one object per +// line flushed as it is written, and both carry a schema identifier. +// - CLI-7: colour and animation are off when stdout is not a terminal, when +// NO_COLOR is set, or when TERM=dumb. +// - CLI-22: values that came from a registered component are stripped of terminal +// control characters before they reach a terminal. +// +// Human table output is deliberately not a contract and may change at any time. The +// JSON and NDJSON shapes are contracts: changing one is a breaking change under +// VER-3. See docs/adr/0001-cli-output-and-exit-code-contract.md. +package cliout + +import ( + "encoding/json" + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/mattn/go-isatty" + "github.com/withobsrvr/flowctl/internal/exitcode" + "github.com/withobsrvr/flowctl/internal/sanitize" +) + +// Format is an output format selected by --output. +type Format string + +const ( + // FormatTable is the human-readable default. Not a contract. + FormatTable Format = "table" + // FormatJSON emits a single JSON document. + FormatJSON Format = "json" + // FormatNDJSON emits one JSON object per line, flushed per line. + FormatNDJSON Format = "ndjson" +) + +// ParseFormat resolves the value of --output. +// +// Unknown values are a usage error rather than a silent fallback: a script that asked +// for JSON and got a table would otherwise parse garbage. "yaml" and "wide" were +// advertised by earlier versions and never implemented; they are named explicitly so +// the error can say so. +func ParseFormat(s string) (Format, error) { + switch strings.ToLower(strings.TrimSpace(s)) { + case "", "table": + return FormatTable, nil + case "json": + return FormatJSON, nil + case "ndjson": + return FormatNDJSON, nil + case "yaml", "wide": + return "", exitcode.Errorf(exitcode.Usage, + "output format %q is not implemented; use table, json or ndjson", s) + default: + return "", exitcode.Errorf(exitcode.Usage, + "unknown output format %q; use table, json or ndjson", s) + } +} + +// Printer renders results to stdout and everything else to stderr. +type Printer struct { + out io.Writer + errw io.Writer + format Format + color bool +} + +// New returns a Printer writing results to out and messages to errw. +func New(format Format, out, errw io.Writer) *Printer { + return &Printer{ + out: out, + errw: errw, + format: format, + color: format == FormatTable && ShouldColor(out), + } +} + +// Format reports the selected output format. +func (p *Printer) Format() Format { return p.format } + +// Machine reports whether output is being produced for a program rather than a person. +// Use it to skip decorative work, never to change what data is emitted. +func (p *Printer) Machine() bool { return p.format != FormatTable } + +// Color reports whether ANSI colour may be written. +func (p *Printer) Color() bool { return p.color } + +// Out returns the result stream. Prefer List and Object; reach for this only when +// emitting something that is genuinely not a record, such as generated YAML. +func (p *Printer) Out() io.Writer { return p.out } + +// Err returns the message stream. +func (p *Printer) Err() io.Writer { return p.errw } + +// Info writes a human-facing message to stderr. It never appears on stdout, so it +// cannot corrupt piped output, and it is written in every format: a person watching a +// terminal still wants progress even when the result is JSON. +func (p *Printer) Info(format string, args ...any) { + fmt.Fprintln(p.errw, sanitize.String(fmt.Sprintf(format, args...))) +} + +// Row is one record. Keys are the field names used in JSON output and referenced by +// Column.Key and Field.Key. +type Row map[string]any + +// Column describes one table column and the field it reads. +type Column struct { + // Header is the column heading in table output. + Header string + // Key is the Row key to read. + Key string + // Format converts the value to a cell. Defaults to Value. + Format func(any) string +} + +// List is a collection result. +type List struct { + // Schema identifies the collection, e.g. "flowctl.service_list.v1". + Schema string + // ItemSchema identifies one element, e.g. "flowctl.service.v1". + ItemSchema string + // Columns define table rendering, in order. + Columns []Column + // Rows are the records, already converted to plain values. + Rows []Row + // Empty is the message shown on a terminal when there are no rows. It goes to + // stderr, because "nothing matched" is a message, not a result: machine output + // still emits a well-formed empty collection. + Empty string +} + +// List renders a collection. +func (p *Printer) List(l List) error { + switch p.format { + case FormatJSON: + return p.encode(map[string]any{ + "schema": l.Schema, + "items": itemsOf(l.Rows), + }) + case FormatNDJSON: + for _, row := range l.Rows { + if err := p.encode(map[string]any{ + "schema": l.ItemSchema, + "data": map[string]any(row), + }); err != nil { + return err + } + } + return nil + default: + if len(l.Rows) == 0 { + if l.Empty != "" { + p.Info("%s", l.Empty) + } + return nil + } + return p.table(l) + } +} + +// Field describes one labelled field of an object in table output. +type Field struct { + Label string + Key string + Format func(any) string +} + +// Object is a single-record result. +type Object struct { + // Schema identifies the record, e.g. "flowctl.version.v1". + Schema string + // Fields define table rendering, in order. + Fields []Field + // Data is the record. + Data Row +} + +// Object renders a single record. In NDJSON it is a single line, which keeps +// `flowctl … --output ndjson` uniform regardless of arity. +func (p *Printer) Object(o Object) error { + switch p.format { + case FormatJSON, FormatNDJSON: + return p.encode(map[string]any{ + "schema": o.Schema, + "data": map[string]any(o.Data), + }) + default: + // Pad to the longest label including its colon, so values line up. + width := 0 + for _, f := range o.Fields { + if n := len(f.Label) + 1; n > width { + width = n + } + } + for _, f := range o.Fields { + format := f.Format + if format == nil { + format = Value + } + if _, err := fmt.Fprintf(p.out, "%-*s %s\n", + width, f.Label+":", format(o.Data[f.Key])); err != nil { + return err + } + } + return nil + } +} + +// table writes tab-aligned columns to stdout. +func (p *Printer) table(l List) error { + w := tabwriter.NewWriter(p.out, 0, 0, 2, ' ', 0) + + headers := make([]string, len(l.Columns)) + for i, c := range l.Columns { + headers[i] = strings.ToUpper(c.Header) + } + if _, err := fmt.Fprintln(w, strings.Join(headers, "\t")); err != nil { + return err + } + + for _, row := range l.Rows { + cells := make([]string, len(l.Columns)) + for i, c := range l.Columns { + format := c.Format + if format == nil { + format = Value + } + cells[i] = format(row[c.Key]) + } + if _, err := fmt.Fprintln(w, strings.Join(cells, "\t")); err != nil { + return err + } + } + return w.Flush() +} + +// encode writes one JSON document followed by a newline, flushing as it goes so that +// NDJSON consumers see each line as it is produced rather than at exit (CLI-6). +func (p *Printer) encode(v any) error { + enc := json.NewEncoder(p.out) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + return fmt.Errorf("encoding output: %w", err) + } + if f, ok := p.out.(interface{ Sync() error }); ok { + // Best effort: a closed or non-syncable stream is the caller's problem, + // and failing the command over it would be worse than the missed flush. + _ = f.Sync() + } + return nil +} + +func itemsOf(rows []Row) []map[string]any { + // Always a non-nil slice, so an empty collection marshals as [] and not null. + items := make([]map[string]any, 0, len(rows)) + for _, r := range rows { + items = append(items, map[string]any(r)) + } + return items +} + +// Value renders a field for table output. +// +// Every string passes through sanitize.Line, because most values reaching here were +// supplied by a registered component and are untrusted (CLI-22). Times render as +// RFC 3339 UTC to match DATA-3. +func Value(v any) string { + switch t := v.(type) { + case nil: + return "-" + case string: + if t == "" { + return "-" + } + return sanitize.Line(t) + case time.Time: + if t.IsZero() { + return "-" + } + return t.UTC().Format(time.RFC3339) + case *time.Time: + if t == nil { + return "-" + } + return Value(*t) + case time.Duration: + return t.String() + case bool: + if t { + return "yes" + } + return "no" + case float64: + return strings.TrimSuffix(strings.TrimRight(fmt.Sprintf("%.2f", t), "0"), ".") + default: + return sanitize.Line(fmt.Sprint(v)) + } +} + +// ShouldColor reports whether ANSI colour may be written to w. +// +// Colour is off unless w is a terminal, and off regardless when NO_COLOR is set to +// any value (per no-color.org) or TERM is "dumb". +func ShouldColor(w io.Writer) bool { + if _, set := os.LookupEnv("NO_COLOR"); set { + return false + } + if os.Getenv("TERM") == "dumb" { + return false + } + f, ok := w.(*os.File) + if !ok { + return false + } + return isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd()) +} + +// IsTerminal reports whether w is an interactive terminal. Commands use it to decide +// whether prompting is allowed: CLI-8 requires a tool to run without prompts when +// stdin is not a terminal. +func IsTerminal(w any) bool { + f, ok := w.(*os.File) + if !ok { + return false + } + return isatty.IsTerminal(f.Fd()) +} diff --git a/internal/cliout/cliout_test.go b/internal/cliout/cliout_test.go new file mode 100644 index 0000000..e1eb590 --- /dev/null +++ b/internal/cliout/cliout_test.go @@ -0,0 +1,319 @@ +package cliout + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/withobsrvr/flowctl/internal/exitcode" +) + +func newTestPrinter(format Format) (*Printer, *bytes.Buffer, *bytes.Buffer) { + var out, errw bytes.Buffer + return New(format, &out, &errw), &out, &errw +} + +func sampleList() List { + return List{ + Schema: "flowctl.service_list.v1", + ItemSchema: "flowctl.service.v1", + Columns: []Column{ + {Header: "ID", Key: "id"}, + {Header: "Type", Key: "type"}, + }, + Rows: []Row{ + {"id": "src-1", "type": "SOURCE"}, + {"id": "sink-1", "type": "SINK"}, + }, + Empty: "Nothing here.", + } +} + +// --- CLI-6: machine-readable shapes are contracts ------------------------------ + +func TestJSONListIsOneDocumentWithSchema(t *testing.T) { + p, out, errw := newTestPrinter(FormatJSON) + if err := p.List(sampleList()); err != nil { + t.Fatal(err) + } + + var doc struct { + Schema string `json:"schema"` + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(out.Bytes(), &doc); err != nil { + t.Fatalf("output is not one JSON document: %v\ngot: %s", err, out.String()) + } + if doc.Schema != "flowctl.service_list.v1" { + t.Errorf("schema = %q", doc.Schema) + } + if len(doc.Items) != 2 || doc.Items[0]["id"] != "src-1" { + t.Errorf("items = %#v", doc.Items) + } + if errw.Len() != 0 { + t.Errorf("stderr should be empty, got %q", errw.String()) + } +} + +func TestNDJSONIsOneObjectPerLineEachWithSchema(t *testing.T) { + p, out, _ := newTestPrinter(FormatNDJSON) + if err := p.List(sampleList()); err != nil { + t.Fatal(err) + } + + lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n") + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d: %q", len(lines), out.String()) + } + for i, line := range lines { + var rec struct { + Schema string `json:"schema"` + Data map[string]any `json:"data"` + } + if err := json.Unmarshal([]byte(line), &rec); err != nil { + t.Fatalf("line %d is not JSON: %v", i, err) + } + if rec.Schema != "flowctl.service.v1" { + t.Errorf("line %d schema = %q, want the item schema", i, rec.Schema) + } + if rec.Data["id"] == nil { + t.Errorf("line %d lost its payload: %s", i, line) + } + } +} + +func TestEmptyListStillEmitsWellFormedJSON(t *testing.T) { + // A consumer must get [] rather than null, and the "nothing here" message + // must not land on stdout where it would break parsing (CLI-5). + p, out, errw := newTestPrinter(FormatJSON) + l := sampleList() + l.Rows = nil + if err := p.List(l); err != nil { + t.Fatal(err) + } + if got := strings.TrimSpace(out.String()); got != `{"items":[],"schema":"flowctl.service_list.v1"}` { + t.Errorf("got %s", got) + } + if errw.Len() != 0 { + t.Errorf("stderr should be empty in machine mode, got %q", errw.String()) + } +} + +func TestEmptyListInTableModeWritesMessageToStderr(t *testing.T) { + p, out, errw := newTestPrinter(FormatTable) + l := sampleList() + l.Rows = nil + if err := p.List(l); err != nil { + t.Fatal(err) + } + if out.Len() != 0 { + t.Errorf("stdout should carry results only, got %q", out.String()) + } + if !strings.Contains(errw.String(), "Nothing here.") { + t.Errorf("stderr = %q", errw.String()) + } +} + +func TestObjectCarriesSchemaInBothMachineFormats(t *testing.T) { + for _, format := range []Format{FormatJSON, FormatNDJSON} { + p, out, _ := newTestPrinter(format) + err := p.Object(Object{ + Schema: "flowctl.version.v1", + Fields: []Field{{Label: "version", Key: "version"}}, + Data: Row{"version": "v0.0.12"}, + }) + if err != nil { + t.Fatal(err) + } + var rec struct { + Schema string `json:"schema"` + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(out.Bytes(), &rec); err != nil { + t.Fatalf("%s: %v", format, err) + } + if rec.Schema != "flowctl.version.v1" || rec.Data["version"] != "v0.0.12" { + t.Errorf("%s: got %#v", format, rec) + } + } +} + +// --- CLI-5: stdout carries results only ---------------------------------------- + +func TestInfoNeverReachesStdout(t *testing.T) { + for _, format := range []Format{FormatTable, FormatJSON, FormatNDJSON} { + p, out, errw := newTestPrinter(format) + p.Info("connecting to %s", "localhost:8080") + if out.Len() != 0 { + t.Errorf("%s: Info wrote %q to stdout", format, out.String()) + } + if !strings.Contains(errw.String(), "connecting to localhost:8080") { + t.Errorf("%s: stderr = %q", format, errw.String()) + } + } +} + +// --- CLI-22: untrusted values cannot reach the terminal as control codes -------- + +func TestTableCellsAreSanitised(t *testing.T) { + p, out, _ := newTestPrinter(FormatTable) + err := p.List(List{ + Schema: "flowctl.service_list.v1", + ItemSchema: "flowctl.service.v1", + Columns: []Column{{Header: "ID", Key: "id"}}, + // A hostile component registering under this name. + Rows: []Row{{"id": "svc\x1b]0;pwned\x07\nFAKE\tROW"}}, + }) + if err != nil { + t.Fatal(err) + } + if strings.ContainsRune(out.String(), 0x1b) { + t.Errorf("ESC reached the terminal: %q", out.String()) + } + // One header line plus one data line; the forged row must not appear. + if n := strings.Count(strings.TrimRight(out.String(), "\n"), "\n"); n != 1 { + t.Errorf("expected 2 lines, got %d: %q", n+1, out.String()) + } +} + +func TestJSONIsNotSanitisedButIsEscaped(t *testing.T) { + // encoding/json escapes control characters, so they cannot reach the terminal + // as control codes. Stripping them here would corrupt data the consumer is + // entitled to receive intact. + p, out, _ := newTestPrinter(FormatJSON) + err := p.List(List{ + Schema: "flowctl.service_list.v1", + ItemSchema: "flowctl.service.v1", + Rows: []Row{{"id": "svc\x1b[2K"}}, + }) + if err != nil { + t.Fatal(err) + } + if strings.ContainsRune(out.String(), 0x1b) { + t.Errorf("raw ESC byte in JSON output: %q", out.String()) + } + if !strings.Contains(out.String(), `\u001b`) { + t.Errorf("expected the ESC to be u-escaped, got %q", out.String()) + } + var doc struct { + Items []map[string]any `json:"items"` + } + if err := json.Unmarshal(out.Bytes(), &doc); err != nil { + t.Fatal(err) + } + if doc.Items[0]["id"] != "svc\x1b[2K" { + t.Errorf("JSON round-trip lost data: %q", doc.Items[0]["id"]) + } +} + +// --- CLI-6: format parsing ------------------------------------------------------ + +func TestParseFormat(t *testing.T) { + for _, tc := range []struct { + in string + want Format + }{ + {"", FormatTable}, + {"table", FormatTable}, + {"json", FormatJSON}, + {"ndjson", FormatNDJSON}, + {" JSON ", FormatJSON}, + } { + got, err := ParseFormat(tc.in) + if err != nil { + t.Errorf("ParseFormat(%q): %v", tc.in, err) + continue + } + if got != tc.want { + t.Errorf("ParseFormat(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestParseFormatRejectsUnimplementedFormatsAsUsageErrors(t *testing.T) { + // Falling back to a table when a script asked for YAML would feed it garbage. + for _, in := range []string{"yaml", "wide", "xml", "nonsense"} { + _, err := ParseFormat(in) + if err == nil { + t.Errorf("ParseFormat(%q) should fail", in) + continue + } + if got := exitcode.Of(err); got != exitcode.Usage { + t.Errorf("ParseFormat(%q) exit code = %d, want %d", in, got, exitcode.Usage) + } + } +} + +// --- value rendering ------------------------------------------------------------ + +func TestValue(t *testing.T) { + ts := time.Date(2026, 10, 20, 14, 30, 0, 0, time.FixedZone("CEST", 2*60*60)) + for _, tc := range []struct { + name string + in any + want string + }{ + {"nil", nil, "-"}, + {"empty string", "", "-"}, + {"string", "src-1", "src-1"}, + {"true", true, "yes"}, + {"false", false, "no"}, + // DATA-3: RFC 3339, UTC, Z suffix. + {"time", ts, "2026-10-20T12:30:00Z"}, + {"zero time", time.Time{}, "-"}, + {"duration", 90 * time.Second, "1m30s"}, + {"float trims zeros", 12.50, "12.5"}, + {"float whole", 12.00, "12"}, + {"int", 42, "42"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := Value(tc.in); got != tc.want { + t.Errorf("Value(%v) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// --- CLI-7: colour -------------------------------------------------------------- + +func TestColorOffWhenNotATerminal(t *testing.T) { + p, _, _ := newTestPrinter(FormatTable) + if p.Color() { + t.Error("colour should be off when stdout is a buffer") + } +} + +func TestColorOffInMachineFormats(t *testing.T) { + for _, format := range []Format{FormatJSON, FormatNDJSON} { + p, _, _ := newTestPrinter(format) + if p.Color() { + t.Errorf("%s: colour should never be on in a machine format", format) + } + } +} + +func TestShouldColorHonoursNoColor(t *testing.T) { + t.Setenv("NO_COLOR", "") + if ShouldColor(nil) { + t.Error("NO_COLOR set to empty string must still disable colour (no-color.org)") + } + t.Setenv("NO_COLOR", "1") + if ShouldColor(nil) { + t.Error("NO_COLOR must disable colour") + } +} + +func TestMachineReportsFormatClass(t *testing.T) { + table, _, _ := newTestPrinter(FormatTable) + if table.Machine() { + t.Error("table is not a machine format") + } + for _, format := range []Format{FormatJSON, FormatNDJSON} { + p, _, _ := newTestPrinter(format) + if !p.Machine() { + t.Errorf("%s should be a machine format", format) + } + } +} diff --git a/internal/cliout/context.go b/internal/cliout/context.go new file mode 100644 index 0000000..88786b0 --- /dev/null +++ b/internal/cliout/context.go @@ -0,0 +1,29 @@ +package cliout + +import ( + "context" + "os" +) + +type contextKey struct{} + +// NewContext returns ctx carrying p, so subcommands in other packages can reach the +// Printer the root command built from the global --output flag without a package +// variable. +func NewContext(ctx context.Context, p *Printer) context.Context { + return context.WithValue(ctx, contextKey{}, p) +} + +// FromContext returns the Printer in ctx. +// +// It never returns nil. A command invoked outside the normal root wiring — in a test, +// say — gets a plain table Printer on the real streams rather than a panic, which +// keeps cliout usable from a unit test with no setup. +func FromContext(ctx context.Context) *Printer { + if ctx != nil { + if p, ok := ctx.Value(contextKey{}).(*Printer); ok && p != nil { + return p + } + } + return New(FormatTable, os.Stdout, os.Stderr) +} diff --git a/internal/cliout/grpcerr.go b/internal/cliout/grpcerr.go new file mode 100644 index 0000000..6c78ee5 --- /dev/null +++ b/internal/cliout/grpcerr.go @@ -0,0 +1,46 @@ +package cliout + +import ( + "context" + + "github.com/withobsrvr/flowctl/internal/exitcode" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// UnreachableHint turns a "cannot reach the control plane" failure into an +// error that says what to do about it. +// +// grpc.NewClient connects lazily, so a dead control plane no longer fails at +// dial time the way grpc.DialContext with WithBlock did — it fails on the first +// RPC, as a bare "connection refused". Without this, the single most common +// error an operator hits loses its hint. +func UnreachableHint(endpoint string, err error) error { + if err == nil { + return nil + } + if status.Code(err) != codes.Unavailable { + return err + } + return exitcode.Errorf(exitcode.Unavailable, + "no control plane is reachable at %s\n"+ + "start one with: flowctl run \n"+ + "or point at a different one with --control-plane-address/--control-plane-port", + endpoint) +} + +// UnreachableInterceptor applies UnreachableHint to every unary call on a +// connection, so each command does not have to remember to. +func UnreachableInterceptor(endpoint string) grpc.DialOption { + return grpc.WithChainUnaryInterceptor(func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + return UnreachableHint(endpoint, invoker(ctx, method, req, reply, cc, opts...)) + }) +} diff --git a/internal/components/metadata.go b/internal/components/metadata.go index 5299c48..ba1c99d 100644 --- a/internal/components/metadata.go +++ b/internal/components/metadata.go @@ -11,9 +11,9 @@ type Metadata struct { Name string `json:"name"` Version string `json:"version"` Description string `json:"description"` - Type string `json:"type"` // "source", "processor", "sink" - Produces []string `json:"produces"` // Event types produced - Consumes []string `json:"consumes"` // Event types consumed + Type string `json:"type"` // "source", "processor", "sink" + Produces []string `json:"produces"` // Event types produced + Consumes []string `json:"consumes"` // Event types consumed InputTypes []string `json:"input_types"` OutputTypes []string `json:"output_types"` EnvMapping map[string]string `json:"env_mapping"` // Config key -> ENV var mapping diff --git a/internal/components/resolver.go b/internal/components/resolver.go index 2337c44..604c621 100644 --- a/internal/components/resolver.go +++ b/internal/components/resolver.go @@ -17,9 +17,9 @@ import ( // - OCI image download and extraction // - Binary extraction and permission setting type Resolver struct { - client *registry.Client - cacheDir string - defaultOrg string // Default organization for short refs (e.g., "withobsrvr") + client *registry.Client + cacheDir string + defaultOrg string // Default organization for short refs (e.g., "withobsrvr") } // NewResolver creates a new component resolver. @@ -90,7 +90,7 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (*Component, error) } // Download and extract component - fmt.Printf("→ Downloading %s@%s...\n", name, version) + fmt.Fprintf(os.Stderr, "→ Downloading %s@%s...\n", name, version) if err := r.download(ctx, name, version); err != nil { return nil, fmt.Errorf("failed to download component %s@%s: %w", name, version, err) } @@ -103,7 +103,7 @@ func (r *Resolver) Resolve(ctx context.Context, ref string) (*Component, error) // Load metadata metadata := r.loadMetadataWithFallback(name, version, metadataPath) - fmt.Printf("✓ Downloaded %s@%s\n", name, version) + fmt.Fprintf(os.Stderr, "✓ Downloaded %s@%s\n", name, version) return &Component{ Name: name, @@ -195,7 +195,7 @@ func (r *Resolver) download(ctx context.Context, name, version string) error { destMetadataPath := r.componentMetadataPath(name, version) if err := copyFile(sourceMetadataPath, destMetadataPath); err != nil { // Non-fatal - metadata is optional - fmt.Printf(" Warning: failed to copy metadata.json: %v\n", err) + fmt.Fprintf(os.Stderr, " Warning: failed to copy metadata.json: %v\n", err) } } diff --git a/internal/config/config.go b/internal/config/config.go index 051e6fe..ed26389 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,7 +18,7 @@ type Config struct { Processors []ProcessorConfig `yaml:"processors"` Sink SinkConfig `yaml:"sink"` // TLS configuration for gRPC communication - TLS *TLSConfig `yaml:"tls,omitempty"` + TLS *TLSConfig `yaml:"tls,omitempty"` } // SourceConfig represents source configuration @@ -43,10 +43,10 @@ type SinkConfig struct { // LoadFromFile loads configuration from a YAML file func LoadFromFile(path string) (*Config, error) { logger.Debug("Loading config from file", zap.String("path", path)) - + data, err := os.ReadFile(path) if err != nil { - logger.Error("Failed to read config file", + logger.Error("Failed to read config file", zap.String("path", path), zap.Error(err)) return nil, fmt.Errorf("failed to read config file: %w", err) @@ -54,17 +54,17 @@ func LoadFromFile(path string) (*Config, error) { var cfg Config if err := yaml.Unmarshal(data, &cfg); err != nil { - logger.Error("Failed to parse config file", + logger.Error("Failed to parse config file", zap.String("path", path), zap.Error(err)) return nil, fmt.Errorf("failed to parse config file: %w", err) } - + // Set default TLS config if not provided if cfg.TLS == nil { cfg.TLS = DefaultTLSConfig() } - + // Resolve relative certificate paths based on config file location if cfg.TLS.Mode != TLSModeDisabled { baseDir := filepath.Dir(path) @@ -75,8 +75,8 @@ func LoadFromFile(path string) (*Config, error) { } } - logger.Debug("Successfully loaded config file", - zap.String("path", path), + logger.Debug("Successfully loaded config file", + zap.String("path", path), zap.String("version", cfg.Version), zap.String("tls_mode", string(cfg.TLS.Mode))) return &cfg, nil @@ -85,7 +85,7 @@ func LoadFromFile(path string) (*Config, error) { // Validate checks if the configuration is valid func (c *Config) Validate() error { logger.Debug("Validating configuration") - + if c.Version == "" { logger.Error("Invalid configuration", zap.String("reason", "missing version")) return fmt.Errorf("version is required") @@ -105,7 +105,7 @@ func (c *Config) Validate() error { logger.Error("Invalid configuration", zap.String("reason", "missing sink type")) return fmt.Errorf("sink type is required") } - + // Validate TLS configuration if set if c.TLS != nil { if err := c.TLS.Validate(); err != nil { @@ -114,7 +114,7 @@ func (c *Config) Validate() error { } } - logger.Debug("Configuration validated successfully", + logger.Debug("Configuration validated successfully", zap.String("version", c.Version), zap.String("source", c.Source.Type), zap.Int("processors", len(c.Processors)), diff --git a/internal/config/global_config.go b/internal/config/global_config.go index e4e6056..6d7e7c9 100644 --- a/internal/config/global_config.go +++ b/internal/config/global_config.go @@ -11,10 +11,10 @@ import ( const ( // EnvLocalGeneratorType is the environment variable for the local generator type EnvLocalGeneratorType = "FLOWCTL_LOCAL_GENERATOR_TYPE" - + // LocalGeneratorDocker is the Docker Compose generator type LocalGeneratorDocker = "docker" - + // LocalGeneratorBash is the bash script generator type LocalGeneratorBash = "bash" ) @@ -35,13 +35,13 @@ func DefaultGlobalConfig() GlobalConfig { // LoadGlobalConfig loads the global configuration from environment variables func LoadGlobalConfig() GlobalConfig { cfg := DefaultGlobalConfig() - + // Load local generator type from environment if genType := os.Getenv(EnvLocalGeneratorType); genType != "" { genType = strings.ToLower(genType) if genType == LocalGeneratorBash || genType == LocalGeneratorDocker { cfg.LocalGeneratorType = genType - logger.Debug("Using local generator type from environment", + logger.Debug("Using local generator type from environment", zap.String("type", genType)) } else { logger.Warn("Unknown local generator type in environment, using default", @@ -49,7 +49,7 @@ func LoadGlobalConfig() GlobalConfig { zap.String("default", cfg.LocalGeneratorType)) } } - + return cfg } @@ -61,4 +61,4 @@ func (c GlobalConfig) UseDockerComposeGenerator() bool { // UseBashScriptGenerator returns true if the bash script generator should be used func (c GlobalConfig) UseBashScriptGenerator() bool { return c.LocalGeneratorType == LocalGeneratorBash -} \ No newline at end of file +} diff --git a/internal/controlplane/embedded_server_test.go b/internal/controlplane/embedded_server_test.go index d4f5cf8..adc7bec 100644 --- a/internal/controlplane/embedded_server_test.go +++ b/internal/controlplane/embedded_server_test.go @@ -119,4 +119,4 @@ func TestEmbeddedControlPlane_WaitForComponent(t *testing.T) { if got := err.Error(); got == "timeout waiting for component non-existent to register" || len(got) == 0 { t.Errorf("Expected detailed timeout error message, got: %v", err) } -} \ No newline at end of file +} diff --git a/internal/core/dag.go b/internal/core/dag.go index 1e910f1..6f08dd4 100644 --- a/internal/core/dag.go +++ b/internal/core/dag.go @@ -371,4 +371,4 @@ func (d *DAG) Stop() error { logger.Info("DAG stopped") return nil -} \ No newline at end of file +} diff --git a/internal/core/dag_test.go b/internal/core/dag_test.go index 8aacb2f..de7f4b4 100644 --- a/internal/core/dag_test.go +++ b/internal/core/dag_test.go @@ -14,56 +14,56 @@ import ( func TestDAGBasicFlow(t *testing.T) { // Create a new DAG dag := NewDAG(10) - + // Create a mock source mockSrc := &mockTestSource{id: "source1"} err := dag.AddSource("source1", mockSrc) if err != nil { t.Fatalf("Failed to add source: %v", err) } - + // Create a mock processor mockProc := &mockProcessor{id: "processor1"} err = dag.AddProcessor("processor1", mockProc, []string{"test-event"}) if err != nil { t.Fatalf("Failed to add processor: %v", err) } - + // Add a sink channel sinkChan := dag.AddSinkChannel("sink1") - + // Connect source to processor err = dag.ConnectSourceToProcessor("source1", "processor1", "test-event") if err != nil { t.Fatalf("Failed to connect source to processor: %v", err) } - + // Connect processor to sink err = dag.ConnectProcessorToSink("processor1", "sink1", "processed-event") if err != nil { t.Fatalf("Failed to connect processor to sink: %v", err) } - + // Create a context with cancellation ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - + // Start the DAG err = dag.Start(ctx) if err != nil { cancel() // Cancel the context before failing t.Fatalf("Failed to start DAG: %v", err) } - + // Create a channel to signal when we've received a message messageReceived := make(chan struct{}) - + // Collect messages from the sink channel var receivedMessages []*processor.Message - + // Start a goroutine to collect messages go func() { defer close(messageReceived) - + // Wait for a message with a timeout select { case msg, ok := <-sinkChan: @@ -75,7 +75,7 @@ func TestDAGBasicFlow(t *testing.T) { return } }() - + // Wait for message or timeout select { case <-messageReceived: @@ -83,26 +83,26 @@ func TestDAGBasicFlow(t *testing.T) { case <-time.After(2 * time.Second): // Timeout waiting for message } - + // Cancel context and stop the DAG cancel() err = dag.Stop() if err != nil { t.Fatalf("Failed to stop DAG: %v", err) } - + // Wait a bit for goroutines to finish time.Sleep(100 * time.Millisecond) - + // Verify we received at least one message if not timed out if ctx.Err() == context.DeadlineExceeded { t.Fatal("Test timed out waiting for messages") } - + if len(receivedMessages) == 0 { t.Fatal("No messages received from the sink") } - + // Verify the message content msg := receivedMessages[0] if msg.Type != "processed-event" { @@ -114,90 +114,90 @@ func TestDAGBasicFlow(t *testing.T) { func TestDAGComplexFlow(t *testing.T) { // Create a new DAG dag := NewDAG(10) - + // Create a mock source mockSrc := &mockTestSource{id: "source1"} err := dag.AddSource("source1", mockSrc) if err != nil { t.Fatalf("Failed to add source: %v", err) } - + // Create mock processors proc1 := &mockProcessor{id: "processor1"} proc2 := &mockProcessor{id: "processor2"} proc3 := &mockProcessor{id: "processor3"} - + // Add processors to DAG err = dag.AddProcessor("processor1", proc1, []string{"test-event"}) if err != nil { t.Fatalf("Failed to add processor1: %v", err) } - + err = dag.AddProcessor("processor2", proc2, []string{"processed-event"}) if err != nil { t.Fatalf("Failed to add processor2: %v", err) } - + err = dag.AddProcessor("processor3", proc3, []string{"processed-event"}) if err != nil { t.Fatalf("Failed to add processor3: %v", err) } - + // Add sink channels sink1Chan := dag.AddSinkChannel("sink1") sink2Chan := dag.AddSinkChannel("sink2") - + // Connect source to processor1 err = dag.ConnectSourceToProcessor("source1", "processor1", "test-event") if err != nil { t.Fatalf("Failed to connect source to processor1: %v", err) } - + // Connect processor1 to processor2 and processor3 (fan-out) err = dag.Connect("processor1", "processor2", "processed-event") if err != nil { t.Fatalf("Failed to connect processor1 to processor2: %v", err) } - + err = dag.Connect("processor1", "processor3", "processed-event") if err != nil { t.Fatalf("Failed to connect processor1 to processor3: %v", err) } - + // Connect processor2 and processor3 to sinks err = dag.ConnectProcessorToSink("processor2", "sink1", "processed-event") if err != nil { t.Fatalf("Failed to connect processor2 to sink1: %v", err) } - + err = dag.ConnectProcessorToSink("processor3", "sink2", "processed-event") if err != nil { t.Fatalf("Failed to connect processor3 to sink2: %v", err) } - + // Create a context with cancellation ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - + // Start the DAG err = dag.Start(ctx) if err != nil { cancel() // Cancel context before failing t.Fatalf("Failed to start DAG: %v", err) } - + // Define a mutex to protect our message collections var mu sync.Mutex - + // Collect messages from both sinks var sink1Messages []*processor.Message var sink2Messages []*processor.Message - + // Channel to signal when both sinks have received messages bothReceived := make(chan struct{}) - + // Track how many sinks have received messages receiveCount := 0 - + // Start goroutines to collect messages from each sink go func() { select { @@ -215,7 +215,7 @@ func TestDAGComplexFlow(t *testing.T) { // Context canceled } }() - + go func() { select { case msg, ok := <-sink2Chan: @@ -232,7 +232,7 @@ func TestDAGComplexFlow(t *testing.T) { // Context canceled } }() - + // Wait for messages with a shorter timeout select { case <-bothReceived: @@ -240,80 +240,37 @@ func TestDAGComplexFlow(t *testing.T) { case <-time.After(2 * time.Second): // Timeout waiting for messages } - + // Cancel context and stop DAG cancel() err = dag.Stop() if err != nil { t.Fatalf("Failed to stop DAG: %v", err) } - + // Wait a bit for goroutines to finish time.Sleep(100 * time.Millisecond) - + // Check results mu.Lock() defer mu.Unlock() - + if len(sink1Messages) == 0 { t.Fatal("No messages received from sink1") } - + if len(sink2Messages) == 0 { t.Fatal("No messages received from sink2") } - + // Success - both sinks received messages t.Logf("Sink1 received %d messages", len(sink1Messages)) t.Logf("Sink2 received %d messages", len(sink2Messages)) } -// mockTestProcessor is a simpler mock processor for testing -type mockTestProcessor struct { - id string - inputCh chan *source.EventEnvelope - outputCh chan *processor.Message -} - -func newMockTestProcessor(id string) *mockTestProcessor { - return &mockTestProcessor{ - id: id, - inputCh: make(chan *source.EventEnvelope, 10), - outputCh: make(chan *processor.Message, 10), - } -} - -func (p *mockTestProcessor) Init(cfg map[string]any) error { - return nil -} - -func (p *mockTestProcessor) Process(ctx context.Context, e *source.EventEnvelope) ([]*processor.Message, error) { - msg := &processor.Message{ - Type: "processed-event", - Payload: []byte(`{"processor":"` + p.id + `","data":"processed"}`), - } - return []*processor.Message{msg}, nil -} - -func (p *mockTestProcessor) Flush(ctx context.Context) error { - return nil -} - -func (p *mockTestProcessor) Name() string { - return p.id -} - // mockTestSource is a simpler mock source for testing type mockTestSource struct { - id string - outputCh chan source.EventEnvelope -} - -func newMockTestSource(id string) *mockTestSource { - return &mockTestSource{ - id: id, - outputCh: make(chan source.EventEnvelope, 10), - } + id string } func (s *mockTestSource) Open(ctx context.Context) error { @@ -332,7 +289,7 @@ func (s *mockTestSource) Events(ctx context.Context, out chan<- source.EventEnve Payload: []byte(`{"source":"` + s.id + `","data":"test-data"}`), Cursor: "test-cursor", } - + // Try to send with a timeout to avoid blocking forever select { case <-ctx.Done(): @@ -346,7 +303,7 @@ func (s *mockTestSource) Events(ctx context.Context, out chan<- source.EventEnve } } } - + // Keep the channel open until context is cancelled <-ctx.Done() return ctx.Err() @@ -358,4 +315,4 @@ func (s *mockTestSource) Close() error { func (s *mockTestSource) Healthy() error { return nil -} \ No newline at end of file +} diff --git a/internal/core/factory.go b/internal/core/factory.go index d376021..4142c81 100644 --- a/internal/core/factory.go +++ b/internal/core/factory.go @@ -16,10 +16,10 @@ type PipelineType string const ( // PipelineTypeSimple uses the original simple pipeline implementation PipelineTypeSimple PipelineType = "simple" - + // PipelineTypeDAG uses the new DAG-based pipeline implementation PipelineTypeDAG PipelineType = "dag" - + // EnvPipelineType is the environment variable to control which pipeline type to use EnvPipelineType = "FLOWCTL_PIPELINE_TYPE" ) @@ -46,7 +46,7 @@ func DeterminePipelineType() PipelineType { return PipelineTypeSimple } } - + // Default to DAG-based implementation return PipelineTypeDAG } @@ -55,7 +55,7 @@ func DeterminePipelineType() PipelineType { type PipelineInterface interface { // Start begins processing data through the pipeline Start(ctx context.Context) error - + // Stop gracefully shuts down the pipeline Stop() error } @@ -63,7 +63,7 @@ type PipelineInterface interface { // CreatePipeline creates a pipeline using the appropriate implementation func CreatePipeline(cfg *config.Config, pipeline *model.Pipeline) (PipelineInterface, error) { pipelineType := DeterminePipelineType() - + switch pipelineType { case PipelineTypeSimple: // For simple pipeline, we could enhance the legacy implementation to use the pipeline model @@ -74,4 +74,4 @@ func CreatePipeline(cfg *config.Config, pipeline *model.Pipeline) (PipelineInter default: return nil, fmt.Errorf("unknown pipeline type: %s", pipelineType) } -} \ No newline at end of file +} diff --git a/internal/core/pipeline.go b/internal/core/pipeline.go index fc9c08f..220d8c1 100644 --- a/internal/core/pipeline.go +++ b/internal/core/pipeline.go @@ -3,6 +3,8 @@ package core import ( "context" "fmt" + "github.com/withobsrvr/flowctl/internal/utils/logger" + "go.uber.org/zap" "sync" "github.com/withobsrvr/flowctl/internal/config" @@ -95,7 +97,7 @@ func (p *Pipeline) Stop() error { func (p *Pipeline) runSource(ctx context.Context) { defer p.wg.Done() if err := p.src.Events(ctx, p.eventChan); err != nil { - // TODO: Handle error + logger.Error("Source stopped producing events", zap.Error(err)) } } @@ -136,7 +138,8 @@ func (p *Pipeline) runSink(ctx context.Context) { return } if err := p.snk.Write(ctx, msgs); err != nil { - // TODO: Handle error + logger.Error("Sink write failed; messages were dropped", + zap.Int("messages", len(msgs)), zap.Error(err)) } } } diff --git a/internal/core/pipeline_dag.go b/internal/core/pipeline_dag.go index 9cbd6c0..025593e 100644 --- a/internal/core/pipeline_dag.go +++ b/internal/core/pipeline_dag.go @@ -16,13 +16,12 @@ import ( // DAGPipeline represents a data processing pipeline implemented with a DAG of buffered channels type DAGPipeline struct { - cfg *config.Config - pipeline *model.Pipeline - dag *DAG - sources map[string]source.Source - processors map[string]processor.Processor - sinks map[string]sink.Sink - sinkChan chan *processor.Message + cfg *config.Config + pipeline *model.Pipeline + dag *DAG + sources map[string]source.Source + processors map[string]processor.Processor + sinks map[string]sink.Sink } // NewDAGPipeline creates a new pipeline from configuration @@ -70,10 +69,10 @@ func (p *DAGPipeline) buildPipelineGraph() error { } p.processors[procConfig.ID] = proc - + // Determine input event types from the inputs inputTypes := determineInputEventTypes(procConfig, p.pipeline) - + if err := p.dag.AddProcessor(procConfig.ID, proc, inputTypes); err != nil { return fmt.Errorf("failed to add processor to DAG: %w", err) } @@ -87,7 +86,7 @@ func (p *DAGPipeline) buildPipelineGraph() error { } p.sinks[sinkConfig.ID] = snk - + // Create a sink channel for each sink p.dag.AddSinkChannel(sinkConfig.ID) } @@ -107,12 +106,12 @@ func (p *DAGPipeline) buildPipelineGraph() error { for _, src := range p.pipeline.Spec.Sources { sourceMap[src.ID] = src } - + processorMap := make(map[string]model.Component) for _, proc := range p.pipeline.Spec.Processors { processorMap[proc.ID] = proc } - + // Connect sources to processors for _, procConfig := range p.pipeline.Spec.Processors { // For each input, find the source or processor @@ -127,7 +126,7 @@ func (p *DAGPipeline) buildPipelineGraph() error { } continue // Found a source, no need to check processors } - + // Check if input is a processor if upstreamProc, exists := processorMap[inputID]; exists { // Connect upstream processor to this processor @@ -178,7 +177,7 @@ func (p *DAGPipeline) Start(ctx context.Context) error { sinkChan := p.dag.SinkChannels[id] go func(sinkID string, s sink.Sink, ch chan *processor.Message) { ctx := context.Background() // Use a separate context for sinks - + for { select { case <-ctx.Done(): @@ -211,7 +210,7 @@ func (p *DAGPipeline) Start(ctx context.Context) error { // Stop gracefully shuts down the pipeline func (p *DAGPipeline) Stop() error { logger.Info("Stopping DAG pipeline") - + // Stop the DAG first if err := p.dag.Stop(); err != nil { return fmt.Errorf("failed to stop DAG: %w", err) @@ -259,30 +258,26 @@ func createSink(component model.Component) (sink.Sink, error) { // determineInputEventTypes determines the input event types for a processor func determineInputEventTypes(processor model.Component, pipeline *model.Pipeline) []string { var inputTypes []string - + // For each input, find the corresponding component and get its output event types for _, inputID := range processor.Inputs { // Check sources for _, src := range pipeline.Spec.Sources { if src.ID == inputID { // Add output event types from this source - for _, eventType := range src.OutputEventTypes { - inputTypes = append(inputTypes, eventType) - } + inputTypes = append(inputTypes, src.OutputEventTypes...) } } - + // Check processors for _, proc := range pipeline.Spec.Processors { if proc.ID == inputID { // Add output event types from this processor - for _, eventType := range proc.OutputEventTypes { - inputTypes = append(inputTypes, eventType) - } + inputTypes = append(inputTypes, proc.OutputEventTypes...) } } } - + return inputTypes } @@ -345,7 +340,7 @@ func (s *mockSource) Events(ctx context.Context, out chan<- source.EventEnvelope Payload: []byte(`{"data": "test event from ` + s.id + `"}`), Cursor: "test-cursor", } - + // Try to send, but check context again before sending if ctxCanceled { return ctx.Err() @@ -402,4 +397,4 @@ func (p *mockProcessor) Flush(ctx context.Context) error { func (p *mockProcessor) Name() string { return p.id -} \ No newline at end of file +} diff --git a/internal/drivers/docker/docker.go b/internal/drivers/docker/docker.go index 8ab2a0c..0d990ca 100644 --- a/internal/drivers/docker/docker.go +++ b/internal/drivers/docker/docker.go @@ -3,7 +3,7 @@ package docker import ( "context" "fmt" - + "github.com/withobsrvr/flowctl/internal/drivers" "github.com/withobsrvr/flowctl/internal/interfaces" "github.com/withobsrvr/flowctl/internal/model" @@ -28,19 +28,19 @@ func NewDockerDriver() drivers.Driver { // Apply deploys a pipeline using Docker func (d *DockerDriver) Apply(ctx context.Context, pipeline *model.Pipeline, options map[string]string) error { logger.Info("Deploying pipeline with Docker driver") - + // Generate Docker Compose file _, err := d.translator.Translate(ctx, pipeline, model.DockerCompose) if err != nil { return fmt.Errorf("failed to translate pipeline to Docker Compose: %w", err) } - + // TODO: Implement Docker Compose deployment // This would involve: // 1. Writing the Docker Compose file to disk // 2. Running 'docker-compose up -d' command // 3. Checking for successful deployment - + logger.Info("Docker driver implementation is incomplete") return fmt.Errorf("docker driver implementation is incomplete") } @@ -48,13 +48,13 @@ func (d *DockerDriver) Apply(ctx context.Context, pipeline *model.Pipeline, opti // Delete removes a deployed pipeline func (d *DockerDriver) Delete(ctx context.Context, pipelineName string, namespace string, options map[string]string) error { logger.Info("Removing pipeline with Docker driver") - + // TODO: Implement Docker Compose teardown // This would involve: // 1. Locating the Docker Compose file for the pipeline // 2. Running 'docker-compose down' command // 3. Cleaning up any additional resources - + logger.Info("Docker driver implementation is incomplete") return fmt.Errorf("docker driver implementation is incomplete") } @@ -62,12 +62,12 @@ func (d *DockerDriver) Delete(ctx context.Context, pipelineName string, namespac // Status retrieves the status of a deployed pipeline func (d *DockerDriver) Status(ctx context.Context, pipelineName string, namespace string, options map[string]string) (*drivers.PipelineStatus, error) { logger.Info("Getting pipeline status with Docker driver") - + // TODO: Implement Docker status check // This would involve: // 1. Running 'docker-compose ps' to get container statuses // 2. Mapping those to our PipelineStatus structure - + logger.Info("Docker driver implementation is incomplete") return nil, fmt.Errorf("docker driver implementation is incomplete") } @@ -75,4 +75,4 @@ func (d *DockerDriver) Status(ctx context.Context, pipelineName string, namespac func init() { // Register the Docker driver drivers.Register("docker", NewDockerDriver) -} \ No newline at end of file +} diff --git a/internal/drivers/driver.go b/internal/drivers/driver.go index 3ebaa09..ff746b7 100644 --- a/internal/drivers/driver.go +++ b/internal/drivers/driver.go @@ -2,35 +2,35 @@ package drivers import ( "context" - + "github.com/withobsrvr/flowctl/internal/model" ) // Driver represents a driver for deploying and managing pipelines type Driver interface { - // Apply deploys a pipeline + // Apply deploys a pipeline Apply(ctx context.Context, pipeline *model.Pipeline, options map[string]string) error - + // Delete removes a deployed pipeline Delete(ctx context.Context, pipelineName string, namespace string, options map[string]string) error - + // Status retrieves the status of a deployed pipeline Status(ctx context.Context, pipelineName string, namespace string, options map[string]string) (*PipelineStatus, error) } // PipelineStatus represents the status of a deployed pipeline type PipelineStatus struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - State string `json:"state"` // running, failed, pending + Name string `json:"name"` + Namespace string `json:"namespace"` + State string `json:"state"` // running, failed, pending Components map[string]ComponentStatus `json:"components"` - Message string `json:"message,omitempty"` + Message string `json:"message,omitempty"` } // ComponentStatus represents the status of a pipeline component type ComponentStatus struct { ID string `json:"id"` - Type string `json:"type"` // source, processor, sink + Type string `json:"type"` // source, processor, sink State string `json:"state"` // running, failed, pending, stopped Replicas int `json:"replicas"` Ready int `json:"ready"` @@ -65,4 +65,4 @@ func List() []string { names = append(names, name) } return names -} \ No newline at end of file +} diff --git a/internal/exitcode/exitcode.go b/internal/exitcode/exitcode.go new file mode 100644 index 0000000..18583bd --- /dev/null +++ b/internal/exitcode/exitcode.go @@ -0,0 +1,166 @@ +// Package exitcode defines flowctl's process exit codes and maps errors onto them. +// +// The codes are a public contract: scripts and AI operators branch on them to decide +// whether a failure is worth retrying. They follow the OBSRVR Engineering Standard +// v2.0, rule CLI-15. See docs/adr/0001-cli-output-and-exit-code-contract.md. +// +// Commands should return errors wrapped with Wrap or Errorf so the cause survives to +// the top level, where Execute calls Of to pick the process exit code. An unwrapped +// error exits 1, which is always a safe default: "it failed, cause unclassified". +package exitcode + +import ( + "context" + "errors" + "fmt" + "io/fs" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Exit codes. Codes 1 and 2 follow shell convention; 128+n means the process ended on +// signal n. Codes 10-63 are reserved for domain-specific additions, which must be +// documented in docs/adr/0001-cli-output-and-exit-code-contract.md before first use. +const ( + // OK means the operation succeeded. + OK = 0 + // Failure means the operation failed for an unclassified reason. Whether it is + // worth retrying depends on the error message. + Failure = 1 + // Usage means a bad flag, argument or config file. Never retry. + Usage = 2 + // NotFound means the named resource does not exist. Never retry. + NotFound = 3 + // Unauthorized means not authenticated, or authenticated without permission. + // Never retry. + Unauthorized = 4 + // Conflict means a failed precondition: already exists, wrong state, version + // mismatch. Retry only after resolving the conflict. + Conflict = 5 + // Timeout means a wait expired. Safe to retry. + Timeout = 6 + // Unavailable means a dependency could not be reached: network, RPC, storage. + // Retry with backoff. + Unavailable = 7 + // DataInvalid means validation failed: a gap, a duplicate, a bad checksum, a + // schema violation. Do not retry; investigate. + DataInvalid = 8 + // Interrupted means the process shut down cleanly after SIGINT. Safe to retry; + // it resumes from its checkpoint. + Interrupted = 130 + // Terminated means the process shut down cleanly after SIGTERM. Safe to retry; + // it resumes from its checkpoint. + Terminated = 143 +) + +// Error is an error carrying the exit code flowctl should exit with. +type Error struct { + Code int + Err error +} + +func (e *Error) Error() string { return e.Err.Error() } +func (e *Error) Unwrap() error { return e.Err } + +// Wrap tags err with an exit code. It returns nil if err is nil, so it is safe to +// write `return exitcode.Wrap(exitcode.NotFound, err)` on a path that may not have +// failed. +func Wrap(code int, err error) error { + if err == nil { + return nil + } + return &Error{Code: code, Err: err} +} + +// Errorf formats a new error and tags it with an exit code. +func Errorf(code int, format string, args ...any) error { + return &Error{Code: code, Err: fmt.Errorf(format, args...)} +} + +// Of returns the exit code for err. +// +// An explicit code set by Wrap or Errorf always wins. Otherwise Of infers one from +// well-known error types — gRPC status codes, context deadlines, filesystem errors — +// so that callers get a useful code without every call site having to classify by +// hand. Anything unrecognised is Failure. +func Of(err error) int { + if err == nil { + return OK + } + + // An explicit code anywhere in the chain wins over inference. + if coded, ok := errors.AsType[*Error](err); ok { + return coded.Code + } + if sig, ok := errors.AsType[*SignalError](err); ok { + return sig.Code() + } + + // gRPC is how every remote call in flowctl fails, so classify it precisely. + // status.FromError only reports ok for real status errors, which is what we + // want: a plain error that happens to wrap one is still inspected via errors.As + // by the grpcStatusOf helper below. + if code, ok := grpcStatusOf(err); ok { + switch code { + case codes.OK: + return OK + case codes.NotFound: + return NotFound + case codes.Unauthenticated, codes.PermissionDenied: + return Unauthorized + case codes.AlreadyExists, codes.FailedPrecondition, codes.Aborted: + return Conflict + case codes.DeadlineExceeded: + return Timeout + case codes.Unavailable: + return Unavailable + case codes.InvalidArgument, codes.OutOfRange: + return Usage + case codes.DataLoss: + return DataInvalid + } + return Failure + } + + switch { + case errors.Is(err, context.DeadlineExceeded): + return Timeout + case errors.Is(err, fs.ErrNotExist): + return NotFound + case errors.Is(err, fs.ErrPermission): + return Unauthorized + case errors.Is(err, fs.ErrExist): + return Conflict + } + + // context.Canceled is deliberately not mapped to Interrupted. Cancellation has + // many causes besides a signal, and the signal paths set Interrupted or + // Terminated explicitly. + return Failure +} + +// grpcStatusOf reports the gRPC code carried by err, if any. It walks the error chain +// rather than calling status.FromError directly, because flowctl wraps RPC errors with +// %w to add context and status.FromError does not unwrap. +func grpcStatusOf(err error) (codes.Code, bool) { + for e := err; e != nil; e = errors.Unwrap(e) { + if se, ok := e.(interface{ GRPCStatus() *status.Status }); ok { + return se.GRPCStatus().Code(), true + } + } + return codes.Unknown, false +} + +// FromSignal returns the exit code for a process that shut down cleanly after sig. +// Per CLI-15 the convention is 128+n. Signals flowctl does not handle report Failure, +// since reaching here with one is a bug rather than a clean shutdown. +func FromSignal(sig string) int { + switch sig { + case "interrupt", "SIGINT": + return Interrupted + case "terminated", "SIGTERM": + return Terminated + } + return Failure +} diff --git a/internal/exitcode/exitcode_test.go b/internal/exitcode/exitcode_test.go new file mode 100644 index 0000000..168ad6d --- /dev/null +++ b/internal/exitcode/exitcode_test.go @@ -0,0 +1,166 @@ +package exitcode + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestOfClassifiesGRPCStatus(t *testing.T) { + // Every remote call flowctl makes fails as a gRPC status, so this mapping is + // what gives a script something to branch on (CLI-15). + tests := []struct { + code codes.Code + want int + }{ + {codes.OK, OK}, + {codes.NotFound, NotFound}, + {codes.Unauthenticated, Unauthorized}, + {codes.PermissionDenied, Unauthorized}, + {codes.AlreadyExists, Conflict}, + {codes.FailedPrecondition, Conflict}, + {codes.Aborted, Conflict}, + {codes.DeadlineExceeded, Timeout}, + {codes.Unavailable, Unavailable}, + {codes.InvalidArgument, Usage}, + {codes.OutOfRange, Usage}, + {codes.DataLoss, DataInvalid}, + {codes.Internal, Failure}, + {codes.Unknown, Failure}, + } + + for _, tc := range tests { + t.Run(tc.code.String(), func(t *testing.T) { + err := status.Error(tc.code, "boom") + if got := Of(err); got != tc.want { + t.Errorf("Of(%v) = %d, want %d", tc.code, got, tc.want) + } + }) + } +} + +func TestOfUnwrapsWrappedGRPCStatus(t *testing.T) { + // Commands add context with %w. status.FromError does not unwrap, so a naive + // implementation would silently classify every wrapped RPC failure as 1. + err := fmt.Errorf("listing services: %w", + fmt.Errorf("dialling: %w", status.Error(codes.Unavailable, "refused"))) + if got := Of(err); got != Unavailable { + t.Errorf("Of(wrapped Unavailable) = %d, want %d", got, Unavailable) + } +} + +func TestExplicitCodeWinsOverInference(t *testing.T) { + // A command that knows better than the transport must be able to say so. + err := Wrap(DataInvalid, status.Error(codes.Unavailable, "refused")) + if got := Of(err); got != DataInvalid { + t.Errorf("Of = %d, want %d", got, DataInvalid) + } +} + +func TestExplicitCodeFoundThroughWrapping(t *testing.T) { + err := fmt.Errorf("outer: %w", Errorf(NotFound, "no such run")) + if got := Of(err); got != NotFound { + t.Errorf("Of = %d, want %d", got, NotFound) + } +} + +func TestOfStandardErrors(t *testing.T) { + tests := []struct { + name string + err error + want int + }{ + {"nil", nil, OK}, + {"deadline", context.DeadlineExceeded, Timeout}, + {"wrapped deadline", fmt.Errorf("waiting: %w", context.DeadlineExceeded), Timeout}, + {"missing file", fmt.Errorf("open x: %w", fs.ErrNotExist), NotFound}, + {"permission", fmt.Errorf("open x: %w", fs.ErrPermission), Unauthorized}, + {"exists", fmt.Errorf("mkdir: %w", fs.ErrExist), Conflict}, + {"plain", errors.New("something broke"), Failure}, + // Cancellation has causes other than a signal; the signal paths set + // Interrupted explicitly rather than relying on inference. + {"cancelled", context.Canceled, Failure}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := Of(tc.err); got != tc.want { + t.Errorf("Of(%v) = %d, want %d", tc.err, got, tc.want) + } + }) + } +} + +func TestOfRealFilesystemError(t *testing.T) { + _, err := os.Open("/definitely/not/a/real/path/pipeline.yaml") + if err == nil { + t.Fatal("expected an error opening a missing file") + } + if got := Of(fmt.Errorf("reading pipeline: %w", err)); got != NotFound { + t.Errorf("Of = %d, want %d", got, NotFound) + } +} + +func TestWrapNilStaysNil(t *testing.T) { + // So `return exitcode.Wrap(code, err)` is safe on a path that may not fail. + if err := Wrap(NotFound, nil); err != nil { + t.Errorf("Wrap(_, nil) = %v, want nil", err) + } +} + +func TestErrorPreservesMessageAndChain(t *testing.T) { + inner := errors.New("inner") + err := Wrap(Conflict, fmt.Errorf("outer: %w", inner)) + if err.Error() != "outer: inner" { + t.Errorf("Error() = %q", err.Error()) + } + if !errors.Is(err, inner) { + t.Error("errors.Is could not reach the wrapped cause") + } +} + +func TestFromSignal(t *testing.T) { + tests := []struct { + sig string + want int + }{ + {"interrupt", Interrupted}, + {"SIGINT", Interrupted}, + {"terminated", Terminated}, + {"SIGTERM", Terminated}, + {"SIGHUP", Failure}, + } + for _, tc := range tests { + if got := FromSignal(tc.sig); got != tc.want { + t.Errorf("FromSignal(%q) = %d, want %d", tc.sig, got, tc.want) + } + } +} + +func TestCodesMatchTheStandard(t *testing.T) { + // CLI-15 fixes these numbers. Changing one is a breaking change to a public + // contract, so it should fail here first and force an ADR. + for name, pair := range map[string][2]int{ + "OK": {OK, 0}, + "Failure": {Failure, 1}, + "Usage": {Usage, 2}, + "NotFound": {NotFound, 3}, + "Unauthorized": {Unauthorized, 4}, + "Conflict": {Conflict, 5}, + "Timeout": {Timeout, 6}, + "Unavailable": {Unavailable, 7}, + "DataInvalid": {DataInvalid, 8}, + "Interrupted": {Interrupted, 130}, + "Terminated": {Terminated, 143}, + } { + if pair[0] != pair[1] { + t.Errorf("%s = %d, the standard says %d", name, pair[0], pair[1]) + } + } +} diff --git a/internal/exitcode/signal.go b/internal/exitcode/signal.go new file mode 100644 index 0000000..4c6bc52 --- /dev/null +++ b/internal/exitcode/signal.go @@ -0,0 +1,45 @@ +package exitcode + +import ( + "errors" + "fmt" + "os" + "syscall" +) + +// SignalError reports that the process shut down cleanly in response to a signal. +// +// It is not a failure. CLI-11 requires a tool to stop starting new work, finish or +// roll back the current unit, write its checkpoint and exit 130 or 143 — so the +// process must exit non-zero to say "interrupted, resume me", without printing an +// error the operator did not cause. +type SignalError struct { + Signal os.Signal +} + +func (e *SignalError) Error() string { + return fmt.Sprintf("interrupted by %s", e.Signal) +} + +// Code returns 130 for SIGINT and 143 for SIGTERM, per the 128+n convention. +func (e *SignalError) Code() int { + switch e.Signal { + case os.Interrupt, syscall.SIGINT: + return Interrupted + case syscall.SIGTERM: + return Terminated + } + return Failure +} + +// Signal returns an error representing a clean shutdown in response to sig. +func Signal(sig os.Signal) error { + return &SignalError{Signal: sig} +} + +// IsSignal reports whether err represents a clean signal shutdown rather than a +// failure. Execute uses it to decide whether to print an error message. +func IsSignal(err error) bool { + _, ok := errors.AsType[*SignalError](err) + return ok +} diff --git a/internal/generator/docker_compose.go b/internal/generator/docker_compose.go index 2ee7061..2ad4ccf 100644 --- a/internal/generator/docker_compose.go +++ b/internal/generator/docker_compose.go @@ -22,17 +22,17 @@ type DockerComposeConfig struct { // DockerComposeService represents a service in Docker Compose type DockerComposeService struct { - Image string `yaml:"image"` - ContainerName string `yaml:"container_name,omitempty"` - Command []string `yaml:"command,omitempty"` - Environment map[string]string `yaml:"environment,omitempty"` - Ports []string `yaml:"ports,omitempty"` - Volumes []string `yaml:"volumes,omitempty"` - DependsOn []string `yaml:"depends_on,omitempty"` - Networks []string `yaml:"networks,omitempty"` + Image string `yaml:"image"` + ContainerName string `yaml:"container_name,omitempty"` + Command []string `yaml:"command,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + Ports []string `yaml:"ports,omitempty"` + Volumes []string `yaml:"volumes,omitempty"` + DependsOn []string `yaml:"depends_on,omitempty"` + Networks []string `yaml:"networks,omitempty"` HealthCheck *DockerHealthCheck `yaml:"healthcheck,omitempty"` - Restart string `yaml:"restart,omitempty"` - Labels map[string]string `yaml:"labels,omitempty"` + Restart string `yaml:"restart,omitempty"` + Labels map[string]string `yaml:"labels,omitempty"` } // DockerHealthCheck represents a Docker health check configuration @@ -84,7 +84,7 @@ func (g *DockerComposeGenerator) Generate(pipeline *model.Pipeline, opts model.T Restart: "unless-stopped", Labels: map[string]string{ "com.obsrvr.component": "source", - "com.obsrvr.pipeline": pipeline.Metadata.Name, + "com.obsrvr.pipeline": pipeline.Metadata.Name, }, } @@ -143,7 +143,7 @@ func (g *DockerComposeGenerator) Generate(pipeline *model.Pipeline, opts model.T Restart: "unless-stopped", Labels: map[string]string{ "com.obsrvr.component": "processor", - "com.obsrvr.pipeline": pipeline.Metadata.Name, + "com.obsrvr.pipeline": pipeline.Metadata.Name, "com.obsrvr.order": fmt.Sprintf("%d", i), }, } @@ -217,7 +217,7 @@ func (g *DockerComposeGenerator) Generate(pipeline *model.Pipeline, opts model.T Restart: "unless-stopped", Labels: map[string]string{ "com.obsrvr.component": "sink", - "com.obsrvr.pipeline": pipeline.Metadata.Name, + "com.obsrvr.pipeline": pipeline.Metadata.Name, }, } @@ -297,7 +297,7 @@ func (g *DockerComposeGenerator) Generate(pipeline *model.Pipeline, opts model.T Restart: getRestartPolicy(pipelineComp.RestartPolicy, "pipeline"), Labels: map[string]string{ "com.obsrvr.component": "pipeline", - "com.obsrvr.pipeline": pipeline.Metadata.Name, + "com.obsrvr.pipeline": pipeline.Metadata.Name, }, } @@ -512,7 +512,7 @@ func getRestartPolicy(policy string, componentType string) string { } // Default restart policies if componentType == "pipeline" { - return "no" // Pipelines typically run to completion: after the container finishes, it exits and remains stopped; no automatic cleanup or restart occurs + return "no" // Pipelines typically run to completion: after the container finishes, it exits and remains stopped; no automatic cleanup or restart occurs } - return "unless-stopped" // Other components should restart -} \ No newline at end of file + return "unless-stopped" // Other components should restart +} diff --git a/internal/generator/generator.go b/internal/generator/generator.go index eab3fd9..cc1b290 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -1 +1 @@ -package generator \ No newline at end of file +package generator diff --git a/internal/generator/local_docker_compose.go b/internal/generator/local_docker_compose.go index 832803f..bc5353d 100644 --- a/internal/generator/local_docker_compose.go +++ b/internal/generator/local_docker_compose.go @@ -35,10 +35,10 @@ type LocalExecutionOptions struct { // DockerComposeConfigWithProfiles extends DockerComposeConfig to use services with profiles type DockerComposeConfigWithProfiles struct { - Version string `yaml:"version"` + Version string `yaml:"version"` Services map[string]DockerComposeServiceWithProfiles `yaml:"services"` - Networks map[string]DockerComposeNetwork `yaml:"networks,omitempty"` - Volumes map[string]interface{} `yaml:"volumes,omitempty"` + Networks map[string]DockerComposeNetwork `yaml:"networks,omitempty"` + Volumes map[string]interface{} `yaml:"volumes,omitempty"` } // Generate produces Docker Compose configuration for local execution @@ -72,13 +72,13 @@ func (g *DockerComposeLocalGenerator) Generate(pipeline *model.Pipeline, opts mo "local-data": map[string]interface{}{ "labels": map[string]string{ "com.obsrvr.flow.pipeline": pipelineName, - "com.obsrvr.flow.local": "true", + "com.obsrvr.flow.local": "true", }, }, "local-logs": map[string]interface{}{ "labels": map[string]string{ "com.obsrvr.flow.pipeline": pipelineName, - "com.obsrvr.flow.local": "true", + "com.obsrvr.flow.local": "true", }, }, }, @@ -105,7 +105,7 @@ func (g *DockerComposeLocalGenerator) Generate(pipeline *model.Pipeline, opts mo // Process processors for _, proc := range pipeline.Spec.Processors { service := createLocalDockerComposeService(proc, pipelineName, localOpts, "processor") - + // Add dependencies if len(proc.Inputs) > 0 { service.DependsOn = make([]string, len(proc.Inputs)) @@ -113,14 +113,14 @@ func (g *DockerComposeLocalGenerator) Generate(pipeline *model.Pipeline, opts mo service.DependsOn[i] = sanitizeLocalServiceName(input) } } - + composeConfig.Services[sanitizeLocalServiceName(proc.ID)] = service } // Process sinks for _, sink := range pipeline.Spec.Sinks { service := createLocalDockerComposeService(sink, pipelineName, localOpts, "sink") - + // Add dependencies if len(sink.Inputs) > 0 { service.DependsOn = make([]string, len(sink.Inputs)) @@ -128,7 +128,7 @@ func (g *DockerComposeLocalGenerator) Generate(pipeline *model.Pipeline, opts mo service.DependsOn[i] = sanitizeLocalServiceName(input) } } - + composeConfig.Services[sanitizeLocalServiceName(sink.ID)] = service } @@ -137,7 +137,7 @@ func (g *DockerComposeLocalGenerator) Generate(pipeline *model.Pipeline, opts mo if opts.OutputPath != "" { envFilePath = filepath.Join(filepath.Dir(opts.OutputPath), localOpts.EnvFile) } - + if err := generateEnvFile(pipeline, pipelineName, envFilePath, localOpts); err != nil { logger.Warn("Failed to write environment file", zap.Error(err)) } @@ -215,12 +215,12 @@ func (g *DockerComposeLocalGenerator) Validate(pipeline *model.Pipeline) error { } // createLocalDockerComposeService creates a Docker Compose service for local execution -func createLocalDockerComposeService(component model.Component, pipelineName string, +func createLocalDockerComposeService(component model.Component, pipelineName string, opts LocalExecutionOptions, componentType string) DockerComposeServiceWithProfiles { sanitizedID := sanitizeLocalServiceName(component.ID) containerName := fmt.Sprintf("local-%s-%s", pipelineName, sanitizedID) - + // Create base service service := DockerComposeServiceWithProfiles{ DockerComposeService: DockerComposeService{ @@ -237,7 +237,7 @@ func createLocalDockerComposeService(component model.Component, pipelineName str "com.obsrvr.flow.local": "true", }, }, - Profiles: []string{"local"}, + Profiles: []string{"local"}, } // Add environment variables for configuration @@ -279,11 +279,11 @@ func createLocalDockerComposeService(component model.Component, pipelineName str } else { portStr = fmt.Sprintf("%d:%d", port.ContainerPort, port.ContainerPort) } - + if port.Protocol != "" && strings.ToLower(port.Protocol) != "tcp" { portStr = fmt.Sprintf("%s/%s", portStr, strings.ToLower(port.Protocol)) } - + service.Ports = append(service.Ports, portStr) } } @@ -309,7 +309,7 @@ func createLocalDockerComposeService(component model.Component, pipelineName str // Add health check if specified if component.HealthCheck != "" { healthPort := component.HealthPort - + // Default to a reasonable port if not specified if healthPort == 0 { // Try to use the first port if available @@ -348,12 +348,12 @@ func sanitizeLocalServiceName(id string) string { } return '_' }, id) - + // Ensure the name starts with a letter or underscore if len(sanitized) > 0 && !((sanitized[0] >= 'a' && sanitized[0] <= 'z') || (sanitized[0] >= 'A' && sanitized[0] <= 'Z') || sanitized[0] == '_') { sanitized = "_" + sanitized } - + return sanitized } @@ -362,48 +362,48 @@ func generateEnvFile(pipeline *model.Pipeline, pipelineName, envFilePath string, // Create environment variables map envVars := map[string]string{ "FLOW_PIPELINE_NAME": pipelineName, - "FLOW_LOG_DIR": opts.LogDirectory, + "FLOW_LOG_DIR": opts.LogDirectory, } - + // Add component-specific environment variables for _, src := range pipeline.Spec.Sources { prefix := fmt.Sprintf("FLOW_SOURCE_%s_", strings.ToUpper(sanitizeLocalServiceName(src.ID))) envVars[prefix+"ID"] = src.ID - + if src.Env != nil { for key, value := range src.Env { envVars[key] = value } } } - + for _, proc := range pipeline.Spec.Processors { prefix := fmt.Sprintf("FLOW_PROCESSOR_%s_", strings.ToUpper(sanitizeLocalServiceName(proc.ID))) envVars[prefix+"ID"] = proc.ID - + if proc.Env != nil { for key, value := range proc.Env { envVars[key] = value } } } - + for _, sink := range pipeline.Spec.Sinks { prefix := fmt.Sprintf("FLOW_SINK_%s_", strings.ToUpper(sanitizeLocalServiceName(sink.ID))) envVars[prefix+"ID"] = sink.ID - + if sink.Env != nil { for key, value := range sink.Env { envVars[key] = value } } } - + // Build env file content var sb strings.Builder sb.WriteString(fmt.Sprintf("# Environment variables for %s\n", pipelineName)) sb.WriteString("# Generated by flowctl\n\n") - + sb.WriteString("# Global environment variables\n") keys := make([]string, 0, len(envVars)) for key := range envVars { @@ -413,17 +413,17 @@ func generateEnvFile(pipeline *model.Pipeline, pipelineName, envFilePath string, for _, key := range keys { sb.WriteString(fmt.Sprintf("%s=%s\n", key, envVars[key])) } - + // Create directory if it doesn't exist dir := filepath.Dir(envFilePath) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create directory for env file: %w", err) } - + // Write env file if err := os.WriteFile(envFilePath, []byte(sb.String()), 0644); err != nil { return fmt.Errorf("failed to write env file: %w", err) } - + return nil -} \ No newline at end of file +} diff --git a/internal/generator/local_docker_compose_test.go b/internal/generator/local_docker_compose_test.go index cddfc53..2a8ec9b 100644 --- a/internal/generator/local_docker_compose_test.go +++ b/internal/generator/local_docker_compose_test.go @@ -20,9 +20,9 @@ func TestDockerComposeLocalGenerator_Generate(t *testing.T) { Spec: model.Spec{ Sources: []model.Component{ { - ID: "test-source", - Image: "test-source:latest", - Command: []string{"test-source-cmd", "--option"}, + ID: "test-source", + Image: "test-source:latest", + Command: []string{"test-source-cmd", "--option"}, HealthCheck: "/health", HealthPort: 8080, Env: map[string]string{ @@ -39,9 +39,9 @@ func TestDockerComposeLocalGenerator_Generate(t *testing.T) { }, Processors: []model.Component{ { - ID: "test-processor", - Image: "test-processor:latest", - Inputs: []string{"test-source"}, + ID: "test-processor", + Image: "test-processor:latest", + Inputs: []string{"test-source"}, Command: []string{"test-processor-cmd"}, Env: map[string]string{ "PROCESSOR_ENV": "value", @@ -50,9 +50,9 @@ func TestDockerComposeLocalGenerator_Generate(t *testing.T) { }, Sinks: []model.Component{ { - ID: "test-sink", - Image: "test-sink:latest", - Inputs: []string{"test-processor"}, + ID: "test-sink", + Image: "test-sink:latest", + Inputs: []string{"test-processor"}, Command: []string{"test-sink-cmd"}, Volumes: []model.Volume{ { @@ -141,7 +141,7 @@ func TestDockerComposeLocalGenerator_Generate(t *testing.T) { t.Errorf("Could not parse %s service from Docker Compose output", serviceName) continue } - + // Verify profile exists in the YAML output using string search if !strings.Contains(output, "profiles:") || !strings.Contains(output, "- local") { t.Errorf("Service %s doesn't have expected 'local' profile format", serviceName) @@ -261,4 +261,4 @@ func TestDockerComposeLocalGenerator_Validate(t *testing.T) { if err := generator.Validate(invalidPipeline3); err == nil { t.Error("Pipeline with sink missing inputs passed validation") } -} \ No newline at end of file +} diff --git a/internal/generator/local_legacy.go b/internal/generator/local_legacy.go index d1afcf3..5ac8484 100644 --- a/internal/generator/local_legacy.go +++ b/internal/generator/local_legacy.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" "text/template" - + "github.com/withobsrvr/flowctl/internal/model" "github.com/withobsrvr/flowctl/internal/utils/logger" "go.uber.org/zap" @@ -232,7 +232,7 @@ func (g *LegacyLocalGenerator) Generate(pipeline *model.Pipeline, opts model.Tra for i, proc := range pipeline.Spec.Processors { procID := sanitizeID(proc.ID) processorIDs[i] = procID - + // Determine dependencies var dependencies []string if len(proc.Inputs) > 0 { @@ -245,7 +245,7 @@ func (g *LegacyLocalGenerator) Generate(pipeline *model.Pipeline, opts model.Tra // Default to depending on all sources dependencies = sourceIDs } - + procComponent := buildLocalComponent(procID, proc.ID, "processor", proc, dependencies) runConfig.Components = append(runConfig.Components, procComponent) } @@ -253,7 +253,7 @@ func (g *LegacyLocalGenerator) Generate(pipeline *model.Pipeline, opts model.Tra // Process sinks for _, sink := range pipeline.Spec.Sinks { sinkID := sanitizeID(sink.ID) - + // Determine dependencies var dependencies []string if len(sink.Inputs) > 0 { @@ -269,7 +269,7 @@ func (g *LegacyLocalGenerator) Generate(pipeline *model.Pipeline, opts model.Tra // If no processors, depend on sources dependencies = sourceIDs } - + sinkComponent := buildLocalComponent(sinkID, sink.ID, "sink", sink, dependencies) runConfig.Components = append(runConfig.Components, sinkComponent) } @@ -412,8 +412,8 @@ func buildLocalComponent(id, name, cType string, component model.Component, depe // Format full health check URL if !strings.HasPrefix(localComponent.HealthCheck, "http") { - localComponent.HealthCheck = fmt.Sprintf("http://localhost:%d%s", - localComponent.HealthCheckPort, + localComponent.HealthCheck = fmt.Sprintf("http://localhost:%d%s", + localComponent.HealthCheckPort, localComponent.HealthCheck) } } @@ -440,4 +440,4 @@ func writeToFile(path string, data []byte) error { return fmt.Errorf("failed to create directory: %w", err) } return os.WriteFile(path, data, 0644) -} \ No newline at end of file +} diff --git a/internal/interfaces/interfaces.go b/internal/interfaces/interfaces.go index b41415b..b2b2e43 100644 --- a/internal/interfaces/interfaces.go +++ b/internal/interfaces/interfaces.go @@ -13,7 +13,7 @@ type Parser interface { Parse(data []byte) (*model.Pipeline, error) // ParseReader parses from an io.Reader - ParseReader(r io.Reader) (*model.Pipeline, error) + ParseReader(r io.Reader) (*model.Pipeline, error) } // Generator is the interface for generating target-specific output @@ -53,4 +53,4 @@ type Translator interface { // ValidFormats returns all supported output formats ValidFormats() []model.Format -} \ No newline at end of file +} diff --git a/internal/model/format.go b/internal/model/format.go index 9781eb3..6a676f3 100644 --- a/internal/model/format.go +++ b/internal/model/format.go @@ -24,4 +24,4 @@ type TranslationOptions struct { RegistryPrefix string // OutputPath is the path where output files should be written OutputPath string -} \ No newline at end of file +} diff --git a/internal/model/legacy.go b/internal/model/legacy.go index e68c913..e90bbd0 100644 --- a/internal/model/legacy.go +++ b/internal/model/legacy.go @@ -118,4 +118,4 @@ func (p *Pipeline) ToConfig() *Config { } return cfg -} \ No newline at end of file +} diff --git a/internal/orchestrator/docker.go b/internal/orchestrator/docker.go index 45f0153..0745b64 100644 --- a/internal/orchestrator/docker.go +++ b/internal/orchestrator/docker.go @@ -25,19 +25,19 @@ type DockerOrchestrator struct { // NewDockerOrchestrator creates a new Docker orchestrator func NewDockerOrchestrator(controlPlaneEndpoint string) (*DockerOrchestrator, error) { log := logger.GetLogger().Named("docker-orchestrator") - + // Create Docker client dockerClient, err := runner.NewDockerClient(log) if err != nil { return nil, fmt.Errorf("failed to create Docker client: %w", err) } - + // Create control plane client controlPlaneClient, err := runner.NewControlPlaneClient(controlPlaneEndpoint, log) if err != nil { return nil, fmt.Errorf("failed to create control plane client: %w", err) } - + return &DockerOrchestrator{ controlPlaneEndpoint: controlPlaneEndpoint, pipelines: make(map[string]*runner.PipelineRunner), @@ -56,20 +56,20 @@ func (o *DockerOrchestrator) GetType() string { func (o *DockerOrchestrator) StartComponent(ctx context.Context, component *Component) error { o.mu.Lock() defer o.mu.Unlock() - + // Check if component is already running if _, exists := o.pipelines[component.ID]; exists { return fmt.Errorf("component %s is already running", component.ID) } - + o.logger.Info("Starting Docker component", zap.String("id", component.ID), zap.String("type", component.Type), zap.String("image", component.Image)) - + // Convert orchestrator component to model component modelComponent := o.convertToModelComponent(component) - + // Add control plane endpoint to environment if it's a pipeline type if component.Type == "pipeline" { if modelComponent.Env == nil { @@ -83,18 +83,18 @@ func (o *DockerOrchestrator) StartComponent(ctx context.Context, component *Comp } } } - + // Create pipeline runner for the component pipelineRunner := runner.NewPipelineRunner(modelComponent, o.dockerClient, o.controlPlaneClient, o.logger) - + // Start the component if err := pipelineRunner.Start(ctx); err != nil { return fmt.Errorf("failed to start component %s: %w", component.ID, err) } - + // Store the runner o.pipelines[component.ID] = pipelineRunner - + return nil } @@ -102,12 +102,12 @@ func (o *DockerOrchestrator) StartComponent(ctx context.Context, component *Comp func (o *DockerOrchestrator) StopComponent(ctx context.Context, componentID string) error { o.mu.Lock() defer o.mu.Unlock() - + runner, exists := o.pipelines[componentID] if !exists { return &ComponentNotFoundError{ComponentID: componentID} } - + // Stop the component if err := runner.Stop(ctx); err != nil { return &OrchestratorError{ @@ -116,7 +116,7 @@ func (o *DockerOrchestrator) StopComponent(ctx context.Context, componentID stri Err: err, } } - + delete(o.pipelines, componentID) return nil } @@ -125,21 +125,21 @@ func (o *DockerOrchestrator) StopComponent(ctx context.Context, componentID stri func (o *DockerOrchestrator) StopAll(ctx context.Context) error { o.mu.Lock() defer o.mu.Unlock() - + var errors []error for id, runner := range o.pipelines { if err := runner.Stop(ctx); err != nil { errors = append(errors, fmt.Errorf("failed to stop component %s: %w", id, err)) } } - + // Clear all pipelines o.pipelines = make(map[string]*runner.PipelineRunner) - + if len(errors) > 0 { return fmt.Errorf("errors stopping components: %v", errors) } - + return nil } @@ -147,12 +147,12 @@ func (o *DockerOrchestrator) StopAll(ctx context.Context) error { func (o *DockerOrchestrator) GetStatus(ctx context.Context, componentID string) (*ComponentStatus, error) { o.mu.RLock() defer o.mu.RUnlock() - + runner, exists := o.pipelines[componentID] if !exists { return nil, &ComponentNotFoundError{ComponentID: componentID} } - + // For now, return a basic status // TODO: Implement proper status retrieval from Docker status := &ComponentStatus{ @@ -164,7 +164,7 @@ func (o *DockerOrchestrator) GetStatus(ctx context.Context, componentID string) "image": runner.Pipeline.Image, }, } - + return status, nil } @@ -172,7 +172,7 @@ func (o *DockerOrchestrator) GetStatus(ctx context.Context, componentID string) func (o *DockerOrchestrator) GetAllStatus(ctx context.Context) (map[string]*ComponentStatus, error) { o.mu.RLock() defer o.mu.RUnlock() - + statuses := make(map[string]*ComponentStatus) for id := range o.pipelines { status, err := o.GetStatus(ctx, id) @@ -181,7 +181,7 @@ func (o *DockerOrchestrator) GetAllStatus(ctx context.Context) (map[string]*Comp } statuses[id] = status } - + return statuses, nil } @@ -189,12 +189,12 @@ func (o *DockerOrchestrator) GetAllStatus(ctx context.Context) (map[string]*Comp func (o *DockerOrchestrator) IsHealthy(ctx context.Context, componentID string) (bool, error) { o.mu.RLock() defer o.mu.RUnlock() - + _, exists := o.pipelines[componentID] if !exists { return false, &ComponentNotFoundError{ComponentID: componentID} } - + // TODO: Implement actual health checking return true, nil } @@ -204,7 +204,7 @@ func (o *DockerOrchestrator) WaitForHealthy(ctx context.Context, componentID str deadline := time.Now().Add(timeout) ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() - + for { select { case <-ctx.Done(): @@ -213,12 +213,12 @@ func (o *DockerOrchestrator) WaitForHealthy(ctx context.Context, componentID str if time.Now().After(deadline) { return fmt.Errorf("timeout waiting for component %s to become healthy", componentID) } - + healthy, err := o.IsHealthy(ctx, componentID) if err != nil { return err } - + if healthy { return nil } @@ -230,12 +230,12 @@ func (o *DockerOrchestrator) WaitForHealthy(ctx context.Context, componentID str func (o *DockerOrchestrator) GetLogs(ctx context.Context, componentID string, follow bool) (LogStream, error) { o.mu.RLock() defer o.mu.RUnlock() - + runner, exists := o.pipelines[componentID] if !exists { return nil, &ComponentNotFoundError{ComponentID: componentID} } - + return runner.GetLogs(ctx, follow) } @@ -250,7 +250,7 @@ func (o *DockerOrchestrator) convertToModelComponent(orchComp *Component) model. ReadOnly: v.ReadOnly, }) } - + // Convert ports ports := make([]model.Port, 0, len(orchComp.Ports)) for _, p := range orchComp.Ports { @@ -260,7 +260,7 @@ func (o *DockerOrchestrator) convertToModelComponent(orchComp *Component) model. Protocol: p.Protocol, }) } - + // Use restart policy from component, with defaults based on type restartPolicy := orchComp.RestartPolicy if restartPolicy == "" { @@ -273,7 +273,7 @@ func (o *DockerOrchestrator) convertToModelComponent(orchComp *Component) model. restartPolicy = "unless-stopped" } } - + return model.Component{ ID: orchComp.ID, Type: orchComp.Type, @@ -285,4 +285,4 @@ func (o *DockerOrchestrator) convertToModelComponent(orchComp *Component) model. Ports: ports, RestartPolicy: restartPolicy, } -} \ No newline at end of file +} diff --git a/internal/orchestrator/interface.go b/internal/orchestrator/interface.go index 2b43706..3225d20 100644 --- a/internal/orchestrator/interface.go +++ b/internal/orchestrator/interface.go @@ -55,14 +55,14 @@ type HTTPCheck struct { // ComponentStatus represents the status of a component type ComponentStatus struct { - ID string `json:"id"` - Status string `json:"status"` - PID int `json:"pid,omitempty"` + ID string `json:"id"` + Status string `json:"status"` + PID int `json:"pid,omitempty"` ContainerID string `json:"container_id,omitempty"` - Endpoint string `json:"endpoint,omitempty"` - StartTime time.Time `json:"start_time,omitempty"` - Error string `json:"error,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + StartTime time.Time `json:"start_time,omitempty"` + Error string `json:"error,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` } // Orchestrator defines the interface for component orchestration @@ -99,7 +99,7 @@ type Orchestrator interface { type LogStream interface { // Read reads log data Read(p []byte) (n int, err error) - + // Close closes the log stream Close() error } @@ -130,19 +130,19 @@ func (e *ComponentNotFoundError) Error() string { // ComponentStatus constants const ( - StatusUnknown = "unknown" - StatusStarting = "starting" - StatusRunning = "running" - StatusStopping = "stopping" - StatusStopped = "stopped" - StatusFailed = "failed" - StatusHealthy = "healthy" + StatusUnknown = "unknown" + StatusStarting = "starting" + StatusRunning = "running" + StatusStopping = "stopping" + StatusStopped = "stopped" + StatusFailed = "failed" + StatusHealthy = "healthy" StatusUnhealthy = "unhealthy" ) // Orchestrator type constants const ( - TypeProcess = "process" - TypeDocker = "docker" + TypeProcess = "process" + TypeDocker = "docker" TypeKubernetes = "kubernetes" -) \ No newline at end of file +) diff --git a/internal/registry/processor_resolver.go b/internal/registry/processor_resolver.go index 185f30f..83a93fa 100644 --- a/internal/registry/processor_resolver.go +++ b/internal/registry/processor_resolver.go @@ -14,9 +14,9 @@ import ( // - OCI image download and extraction // - Binary extraction and permission setting type ProcessorResolver struct { - client *Client - cacheDir string - defaultOrg string // Default organization for short refs (e.g., "stellar") + client *Client + cacheDir string + defaultOrg string // Default organization for short refs (e.g., "stellar") } // NewProcessorResolver creates a new processor resolver. @@ -70,7 +70,7 @@ func (r *ProcessorResolver) Resolve(ctx context.Context, ref string) (string, er } // Download and extract processor - fmt.Printf("⬇ Downloading %s...\n", ref) + fmt.Fprintf(os.Stderr, "⬇ Downloading %s...\n", ref) if err := r.download(ctx, imgRef); err != nil { return "", fmt.Errorf("failed to download processor %s: %w", ref, err) } @@ -80,7 +80,7 @@ func (r *ProcessorResolver) Resolve(ctx context.Context, ref string) (string, er return "", fmt.Errorf("processor binary not found after download: %s", binaryPath) } - fmt.Printf("✓ Downloaded %s\n", ref) + fmt.Fprintf(os.Stderr, "✓ Downloaded %s\n", ref) return binaryPath, nil } diff --git a/internal/runner/loader.go b/internal/runner/loader.go index 7313b57..6a38cae 100644 --- a/internal/runner/loader.go +++ b/internal/runner/loader.go @@ -5,6 +5,8 @@ import ( "fmt" "os" + "github.com/withobsrvr/flowctl/internal/apiversion" + "github.com/withobsrvr/flowctl/internal/components" "github.com/withobsrvr/flowctl/internal/model" "github.com/withobsrvr/flowctl/internal/utils/logger" @@ -139,12 +141,12 @@ func contains(s, substr string) bool { // validatePipeline validates the pipeline structure func validatePipeline(pipeline *model.Pipeline) error { // Check required fields - if pipeline.APIVersion == "" { - return fmt.Errorf("apiVersion is required") + if err := apiversion.Validate(pipeline.APIVersion); err != nil { + return err } - if pipeline.Kind != "Pipeline" { - return fmt.Errorf("kind must be 'Pipeline', got: %s", pipeline.Kind) + if err := apiversion.ValidateKind(pipeline.Kind); err != nil { + return err } if pipeline.Metadata.Name == "" { @@ -253,4 +255,4 @@ func LoadPipelineFromBytes(data []byte) (*model.Pipeline, error) { } return &pipeline, nil -} \ No newline at end of file +} diff --git a/internal/runner/log_aggregator.go b/internal/runner/log_aggregator.go index fabdd1e..9f6eb0f 100644 --- a/internal/runner/log_aggregator.go +++ b/internal/runner/log_aggregator.go @@ -15,14 +15,14 @@ import ( // LogAggregator aggregates logs from multiple components type LogAggregator struct { - streams map[string]io.ReadCloser - colors map[string]*color.Color - mu sync.RWMutex - quit chan struct{} - logCh chan LogEntry - writer *bufio.Writer - lastFlush time.Time - flushMu sync.Mutex + streams map[string]io.ReadCloser + colors map[string]*color.Color + mu sync.RWMutex + quit chan struct{} + logCh chan LogEntry + writer *bufio.Writer + lastFlush time.Time + flushMu sync.Mutex flushTicker *time.Ticker } @@ -189,8 +189,8 @@ func (r *PipelineRunner) StartAggregatedLogging() *LogAggregator { for _, componentID := range componentIDs { stream, err := r.orchestrator.GetLogs(r.ctx, componentID, true) if err != nil { - logger.Error("Failed to get logs for component", - zap.String("component", componentID), + logger.Error("Failed to get logs for component", + zap.String("component", componentID), zap.Error(err)) continue } @@ -200,4 +200,4 @@ func (r *PipelineRunner) StartAggregatedLogging() *LogAggregator { }() return aggregator -} \ No newline at end of file +} diff --git a/internal/runner/monitor.go b/internal/runner/monitor.go index 4869439..5bb9036 100644 --- a/internal/runner/monitor.go +++ b/internal/runner/monitor.go @@ -3,6 +3,7 @@ package runner import ( "context" "fmt" + "github.com/withobsrvr/flowctl/internal/cliout" "os" "os/signal" "syscall" @@ -15,10 +16,10 @@ import ( // Monitor provides real-time monitoring and status display for the pipeline type Monitor struct { - runner *PipelineRunner - displayCh chan bool - logCh chan LogEntry - quit chan struct{} + runner *PipelineRunner + displayCh chan bool + logCh chan LogEntry + quit chan struct{} clearScreen bool } @@ -95,14 +96,18 @@ func (m *Monitor) displayStatus() { if m.clearScreen { // Clear screen and move cursor to top - fmt.Print("\033[2J\033[H") + // Only clear a real terminal: writing escape codes into a redirected + // stream corrupts it (CLI-7). + if cliout.ShouldColor(os.Stderr) { + fmt.Fprint(os.Stderr, "\033[2J\033[H") + } } // Header - fmt.Printf("🚀 Pipeline: %s\n", m.runner.pipeline.Metadata.Name) - fmt.Printf("⏰ Status as of: %s\n", time.Now().Format("15:04:05")) - fmt.Printf("📡 Control Plane: %s\n", m.runner.controlPlane.GetEndpoint()) - fmt.Println() + fmt.Fprintf(os.Stderr, "🚀 Pipeline: %s\n", m.runner.pipeline.Metadata.Name) + fmt.Fprintf(os.Stderr, "⏰ Status as of: %s\n", time.Now().Format("15:04:05")) + fmt.Fprintf(os.Stderr, "📡 Control Plane: %s\n", m.runner.controlPlane.GetEndpoint()) + fmt.Fprintln(os.Stderr) // Component status healthyCount := 0 @@ -127,39 +132,39 @@ func (m *Monitor) displayStatus() { } componentColor.Printf("%-30s ", componentId) statusColor.Printf("%s", status) - fmt.Println() + fmt.Fprintln(os.Stderr) // Show metrics if available if len(service.Metrics) > 0 { for k, v := range service.Metrics { - fmt.Printf(" %s: %s\n", k, v) + fmt.Fprintf(os.Stderr, " %s: %s\n", k, v) } } // Show last heartbeat if service.LastHeartbeat != nil { - fmt.Printf(" Last seen: %s\n", service.LastHeartbeat.AsTime().Format("15:04:05")) + fmt.Fprintf(os.Stderr, " Last seen: %s\n", service.LastHeartbeat.AsTime().Format("15:04:05")) } - fmt.Println() + fmt.Fprintln(os.Stderr) } // Summary summaryColor := color.New(color.FgYellow, color.Bold) summaryColor.Printf("📊 Summary: %d/%d components healthy\n", healthyCount, len(services)) - fmt.Println() + fmt.Fprintln(os.Stderr) // Instructions - fmt.Println("Press Ctrl+C to stop pipeline") - fmt.Println("=====================================") + fmt.Fprintln(os.Stderr, "Press Ctrl+C to stop pipeline") + fmt.Fprintln(os.Stderr, "=====================================") } // displayLogEntry displays a log entry func (m *Monitor) displayLogEntry(entry LogEntry) { timestamp := entry.Timestamp.Format("15:04:05") - + // Color code by component componentColor := m.getComponentColor(entry.ComponentID) - + // Color code by log level var levelColor *color.Color switch entry.Level { @@ -176,7 +181,7 @@ func (m *Monitor) displayLogEntry(entry LogEntry) { prefix := fmt.Sprintf("[%s] %s:", timestamp, entry.ComponentID) componentColor.Printf("%-30s ", prefix) levelColor.Printf("%-5s ", entry.Level) - fmt.Printf("%s\n", entry.Message) + fmt.Fprintf(os.Stderr, "%s\n", entry.Message) } // getComponentColor returns a color for a component @@ -257,4 +262,4 @@ func (r *PipelineRunner) StartTUIMonitoring() error { } return r.RunWithIntegratedMonitoring() -} \ No newline at end of file +} diff --git a/internal/runner/pipeline_runner.go b/internal/runner/pipeline_runner.go index 79afde3..823f450 100644 --- a/internal/runner/pipeline_runner.go +++ b/internal/runner/pipeline_runner.go @@ -19,6 +19,7 @@ import ( "github.com/google/uuid" flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" "github.com/withobsrvr/flowctl/internal/controlplane" + "github.com/withobsrvr/flowctl/internal/exitcode" "github.com/withobsrvr/flowctl/internal/model" "github.com/withobsrvr/flowctl/internal/orchestrator" "github.com/withobsrvr/flowctl/internal/storage" @@ -168,6 +169,16 @@ func resolveEmbeddedDBPath(dbPath string) (string, error) { // Run starts the pipeline with embedded or external control plane func (r *PipelineRunner) Run() error { + // Register for signals before starting anything. CLI-11 requires a tool to + // stop starting new work and shut down cleanly on SIGINT or SIGTERM; if the + // handler is only installed once startup finishes, a signal arriving during + // startup kills the process outright, leaving child components orphaned and + // the run row marked running. The channel is buffered so a signal that + // arrives before the wait below is still delivered. + sigChan := make(chan os.Signal, 1) + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) + defer signal.Stop(sigChan) + if r.config.UseExternalCP { logger.Info("Starting pipeline with external control plane", zap.String("pipeline", r.pipeline.Metadata.Name), @@ -197,7 +208,7 @@ func (r *PipelineRunner) Run() error { return fmt.Errorf("control plane not ready: %w", err) } - fmt.Printf("✓ Control plane ready at %s\n", r.GetControlPlaneEndpoint()) + fmt.Fprintf(os.Stderr, "✓ Control plane ready at %s\n", r.GetControlPlaneEndpoint()) } else { // Verify external control plane is accessible if err := r.waitForControlPlaneReady(); err != nil { @@ -205,14 +216,14 @@ func (r *PipelineRunner) Run() error { r.config.ControlPlaneAddress, r.config.ControlPlanePort, err) } logger.Info("External control plane is accessible") - fmt.Printf("✓ Connected to external control plane at %s\n", r.GetControlPlaneEndpoint()) + fmt.Fprintf(os.Stderr, "✓ Connected to external control plane at %s\n", r.GetControlPlaneEndpoint()) } // Create pipeline run record r.createPipelineRun() // Start components in dependency order - fmt.Println("→ Launching components...") + fmt.Fprintln(os.Stderr, "→ Launching components...") if err := r.startComponents(); err != nil { r.updateRunStatus(flowctlpb.RunStatus_RUN_STATUS_FAILED) _ = r.shutdown() @@ -220,7 +231,7 @@ func (r *PipelineRunner) Run() error { } // Wait for all components to register - fmt.Println("→ Waiting for components to register...") + fmt.Fprintln(os.Stderr, "→ Waiting for components to register...") if err := r.waitForComponentRegistration(); err != nil { r.updateRunStatus(flowctlpb.RunStatus_RUN_STATUS_FAILED) _ = r.shutdown() @@ -231,7 +242,7 @@ func (r *PipelineRunner) Run() error { logger.Warn("Failed to enrich registered component metadata", zap.Error(err)) } - fmt.Println("→ Wiring data flow between components...") + fmt.Fprintln(os.Stderr, "→ Wiring data flow between components...") streamOrch := NewStreamOrchestrator(r.ctx, r.controlPlane, r.pipeline) streamOrch.runID = r.runID streamOrch.controlPlaneEndpoint = r.GetControlPlaneEndpoint() @@ -280,23 +291,21 @@ func (r *PipelineRunner) Run() error { } logger.Info("Pipeline started successfully with control plane integration") - fmt.Println("✓ Pipeline is running") - fmt.Println() - fmt.Println("Next, in another terminal:") - fmt.Printf(" flowctl status --control-plane-address %s --control-plane-port %d\n", r.config.ControlPlaneAddress, r.config.ControlPlanePort) - fmt.Printf(" flowctl pipelines active --control-plane-address %s --control-plane-port %d\n", r.config.ControlPlaneAddress, r.config.ControlPlanePort) - fmt.Println() - fmt.Printf("Component logs are being written under %s/\n", r.config.LogDir) - fmt.Println("Press Ctrl+C to stop the pipeline.") - - // Set up signal handling - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - defer signal.Stop(sigChan) - // Wait for shutdown or for all delivery edges to terminate. + fmt.Fprintln(os.Stderr, "✓ Pipeline is running") + fmt.Fprintln(os.Stderr) + fmt.Fprintln(os.Stderr, "Next, in another terminal:") + fmt.Fprintf(os.Stderr, " flowctl status --control-plane-address %s --control-plane-port %d\n", r.config.ControlPlaneAddress, r.config.ControlPlanePort) + fmt.Fprintf(os.Stderr, " flowctl pipelines active --control-plane-address %s --control-plane-port %d\n", r.config.ControlPlaneAddress, r.config.ControlPlanePort) + fmt.Fprintln(os.Stderr) + fmt.Fprintf(os.Stderr, "Component logs are being written under %s/\n", r.config.LogDir) + fmt.Fprintln(os.Stderr, "Press Ctrl+C to stop the pipeline.") + + // Wait for shutdown or for all delivery edges to terminate. sigChan was + // registered at the top of Run, so a signal sent during startup is already + // buffered and taken here rather than killing the process. var shutdownStatus flowctlpb.RunStatus var deliveryErr error + var receivedSignal os.Signal select { case deliveryErr = <-terminal: if deliveryErr != nil { @@ -306,8 +315,11 @@ func (r *PipelineRunner) Run() error { } else { shutdownStatus = flowctlpb.RunStatus_RUN_STATUS_COMPLETED } - case <-sigChan: - logger.Info("Received shutdown signal") + case sig := <-sigChan: + logger.Info("Received shutdown signal", zap.String("signal", sig.String())) + // Remembered so the process can exit 130/143 after a clean shutdown, + // telling a supervisor it was interrupted and can resume (CLI-11). + receivedSignal = sig shutdownStatus = flowctlpb.RunStatus_RUN_STATUS_STOPPED case <-r.ctx.Done(): logger.Info("Context cancelled") @@ -337,7 +349,16 @@ func (r *PipelineRunner) Run() error { if deliveryErr != nil { return deliveryErr } - return shutdownErr + if shutdownErr != nil { + return shutdownErr + } + // A clean shutdown that was asked for by a signal still exits non-zero, so a + // supervisor can tell "interrupted, resume from checkpoint" apart from + // "finished" (CLI-11, CLI-15). + if receivedSignal != nil { + return exitcode.Signal(receivedSignal) + } + return nil } // waitForTerminalOnStop preserves a failure already reported or captured by diff --git a/internal/runner/pipeline_runner_test.go b/internal/runner/pipeline_runner_test.go index f8e0f4a..1767cce 100644 --- a/internal/runner/pipeline_runner_test.go +++ b/internal/runner/pipeline_runner_test.go @@ -10,7 +10,7 @@ import ( func TestNewPipelineRunner(t *testing.T) { // Create a test pipeline pipeline := &model.Pipeline{ - APIVersion: "v1", + APIVersion: "flowctl/v1", Kind: "Pipeline", Metadata: model.Metadata{ Name: "test-pipeline", @@ -85,10 +85,10 @@ func TestNewPipelineRunner(t *testing.T) { func TestNewPipelineRunnerPreservesAutoPortSelection(t *testing.T) { pipeline := &model.Pipeline{ - APIVersion: "v1", + APIVersion: "flowctl/v1", Kind: "Pipeline", - Metadata: model.Metadata{Name: "test-pipeline"}, - Spec: model.Spec{}, + Metadata: model.Metadata{Name: "test-pipeline"}, + Spec: model.Spec{}, } runner, err := NewPipelineRunner(pipeline, Config{ @@ -110,7 +110,7 @@ func TestNewPipelineRunnerPreservesAutoPortSelection(t *testing.T) { } func TestLoadPipelineFromBytes(t *testing.T) { - testYAML := `apiVersion: v1 + testYAML := `apiVersion: flowctl/v1 kind: Pipeline metadata: name: test-pipeline @@ -154,7 +154,7 @@ spec: func TestValidatePipeline(t *testing.T) { // Test valid pipeline validPipeline := &model.Pipeline{ - APIVersion: "v1", + APIVersion: "flowctl/v1", Kind: "Pipeline", Metadata: model.Metadata{ Name: "test-pipeline", @@ -195,7 +195,7 @@ func TestValidatePipeline(t *testing.T) { // Test invalid pipeline - wrong kind invalidKindPipeline := &model.Pipeline{ - APIVersion: "v1", + APIVersion: "flowctl/v1", Kind: "WrongKind", Metadata: model.Metadata{ Name: "test-pipeline", @@ -216,7 +216,7 @@ func TestValidatePipeline(t *testing.T) { // Test invalid pipeline - no sources noSourcesPipeline := &model.Pipeline{ - APIVersion: "v1", + APIVersion: "flowctl/v1", Kind: "Pipeline", Metadata: model.Metadata{ Name: "test-pipeline", @@ -229,4 +229,4 @@ func TestValidatePipeline(t *testing.T) { if err := validatePipeline(noSourcesPipeline); err == nil { t.Error("Pipeline with no sources should fail validation") } -} \ No newline at end of file +} diff --git a/internal/sandbox/config/config.go b/internal/sandbox/config/config.go index 3e13cc9..6efe81f 100644 --- a/internal/sandbox/config/config.go +++ b/internal/sandbox/config/config.go @@ -98,7 +98,7 @@ func getDefaultConfig() *SandboxConfig { // validateConfig validates the sandbox configuration func validateConfig(config *SandboxConfig) error { - if config.Services == nil || len(config.Services) == 0 { + if len(config.Services) == 0 { return fmt.Errorf("no services defined") } @@ -130,4 +130,4 @@ func validateConfig(config *SandboxConfig) error { } return nil -} \ No newline at end of file +} diff --git a/internal/sandbox/runtime/runtime.go b/internal/sandbox/runtime/runtime.go index f62b7ce..ca02459 100644 --- a/internal/sandbox/runtime/runtime.go +++ b/internal/sandbox/runtime/runtime.go @@ -22,7 +22,6 @@ type Runtime struct { network string containers map[string]*Container watcher *watcher.Watcher - mu sync.RWMutex } // Config represents runtime configuration @@ -158,7 +157,7 @@ func (r *Runtime) performPreflightChecks() error { // Check for daemon not running if strings.Contains(errMsg, "Cannot connect to the Docker daemon") || strings.Contains(errMsg, "Is the docker daemon running") { - return fmt.Errorf("Docker daemon is not running. Please start Docker and try again.\n\nOriginal error: %s", errMsg) + return fmt.Errorf("docker daemon is not running; start Docker and try again\n\nOriginal error: %s", errMsg) } // Check for command not found @@ -241,12 +240,12 @@ func (r *Runtime) createNetwork() error { // Check for daemon not running if strings.Contains(errMsg, "Cannot connect to the Docker daemon") || strings.Contains(errMsg, "Is the docker daemon running") { - return fmt.Errorf("Docker daemon is not running. Please start Docker and try again.\n\nOriginal error: %s", errMsg) + return fmt.Errorf("docker daemon is not running; start Docker and try again\n\nOriginal error: %s", errMsg) } // Check for command not found if !r.config.UseSystemRuntime && strings.Contains(err.Error(), "no such file or directory") { - return fmt.Errorf("failed to list networks: %w\n\nThis error often occurs when the bundled runtime is not properly set up. Please use the --use-system-runtime flag instead.", err) + return fmt.Errorf("failed to list networks: %w\n\nThis often means the bundled runtime is not set up; use --use-system-runtime instead", err) } return fmt.Errorf("failed to list networks: %s", errMsg) diff --git a/internal/sandbox/watcher/watcher.go b/internal/sandbox/watcher/watcher.go index 7903986..588c977 100644 --- a/internal/sandbox/watcher/watcher.go +++ b/internal/sandbox/watcher/watcher.go @@ -91,8 +91,8 @@ func (w *Watcher) handleEvent(event fsnotify.Event) { // Debounce the reload to prevent rapid-fire triggers w.debouncer.Debounce(func() { if err := w.reloadFunc(event.Name); err != nil { - logger.Error("Failed to reload after file change", - zap.String("file", event.Name), + logger.Error("Failed to reload after file change", + zap.String("file", event.Name), zap.Error(err)) } else { logger.Info("Reloaded after file change", zap.String("file", event.Name)) @@ -116,4 +116,4 @@ func (w *Watcher) Close() error { w.debouncer.timer.Stop() } return w.watcher.Close() -} \ No newline at end of file +} diff --git a/internal/sanitize/proto.go b/internal/sanitize/proto.go new file mode 100644 index 0000000..fdd1062 --- /dev/null +++ b/internal/sanitize/proto.go @@ -0,0 +1,100 @@ +package sanitize + +import ( + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" +) + +// Message returns a copy of m with every string field stripped of terminal +// control characters, ready to be printed to a terminal (CLI-22). +// +// Use it on anything that arrived from the control plane. Service IDs, component +// names, descriptions, endpoints, event-type strings, metadata keys and values, +// and relayed error messages are all written by processes flowctl did not start, +// and several commands print them straight to the operator's terminal. One +// crafted value is enough to emit its own OSC sequence. +// +// The original message is not modified: a command that also emits JSON must send +// the unsanitised value, because encoding/json escapes control characters anyway +// and stripping them would corrupt data the consumer is entitled to receive +// intact. Commands rendering through internal/cliout do not need this at all — +// cliout sanitises at the point it formats a cell, which is the better place. +// This exists for the commands not yet converted to it. +// +// Bytes fields are left alone: they carry encoded payloads, not display text. +func Message[T proto.Message](m T) T { + if !m.ProtoReflect().IsValid() { + // A typed nil. Returning it unchanged keeps callers from having to + // nil-check before sanitising. + return m + } + clone := proto.Clone(m) + sanitizeMessage(clone.ProtoReflect()) + return clone.(T) +} + +// sanitizeMessage walks every populated field of m, in place. +func sanitizeMessage(m protoreflect.Message) { + m.Range(func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { + switch { + case fd.IsMap(): + sanitizeMap(fd, v.Map()) + case fd.IsList(): + sanitizeList(fd, v.List()) + case fd.Kind() == protoreflect.StringKind: + m.Set(fd, protoreflect.ValueOfString(Line(v.String()))) + case fd.Kind() == protoreflect.MessageKind || fd.Kind() == protoreflect.GroupKind: + sanitizeMessage(v.Message()) + } + return true + }) +} + +func sanitizeList(fd protoreflect.FieldDescriptor, list protoreflect.List) { + for i := 0; i < list.Len(); i++ { + switch fd.Kind() { + case protoreflect.StringKind: + list.Set(i, protoreflect.ValueOfString(Line(list.Get(i).String()))) + case protoreflect.MessageKind, protoreflect.GroupKind: + sanitizeMessage(list.Get(i).Message()) + } + } +} + +func sanitizeMap(fd protoreflect.FieldDescriptor, mp protoreflect.Map) { + keyKind := fd.MapKey().Kind() + valKind := fd.MapValue().Kind() + + // Keys cannot be rewritten in place, so collect first and rebuild. A + // hostile component controls metadata keys as well as values, and a key + // is just as capable of carrying an escape sequence. + type entry struct { + key protoreflect.MapKey + val protoreflect.Value + } + var entries []entry + mp.Range(func(k protoreflect.MapKey, v protoreflect.Value) bool { + entries = append(entries, entry{k, v}) + return true + }) + + for _, e := range entries { + newKey := e.key + if keyKind == protoreflect.StringKind { + newKey = protoreflect.ValueOfString(Line(e.key.String())).MapKey() + } + + newVal := e.val + switch valKind { + case protoreflect.StringKind: + newVal = protoreflect.ValueOfString(Line(e.val.String())) + case protoreflect.MessageKind, protoreflect.GroupKind: + sanitizeMessage(e.val.Message()) + } + + if newKey.Interface() != e.key.Interface() { + mp.Clear(e.key) + } + mp.Set(newKey, newVal) + } +} diff --git a/internal/sanitize/proto_test.go b/internal/sanitize/proto_test.go new file mode 100644 index 0000000..a1e81cd --- /dev/null +++ b/internal/sanitize/proto_test.go @@ -0,0 +1,158 @@ +package sanitize + +import ( + "strings" + "testing" + + flowctlv1 "github.com/withObsrvr/flow-proto/go/gen/flowctl/v1" + flowctlpb "github.com/withobsrvr/flowctl/proto" + "google.golang.org/protobuf/proto" +) + +// hostile is the kind of value a component can register under. It sets the +// window title, then erases the line so the forgery is not visible. +const hostile = "svc\x1b]0;pwned\x07\x1b[2Kinnocent" + +func TestMessageSanitisesStringFields(t *testing.T) { + in := &flowctlpb.ServiceInfo{ + ServiceId: hostile, + } + out := Message(in) + + if strings.ContainsRune(out.ServiceId, 0x1b) { + t.Errorf("ServiceId still contains ESC: %q", out.ServiceId) + } + // ESC and BEL are removed; the inert remnants of the sequences stay, which + // is the point — they are now ordinary printable text. + if out.ServiceId != "svc]0;pwned[2Kinnocent" { + t.Errorf("ServiceId = %q", out.ServiceId) + } + for _, r := range out.ServiceId { + if isControl(r) { + t.Errorf("control char %U survived", r) + } + } +} + +func TestMessageDoesNotMutateTheOriginal(t *testing.T) { + // A command that also emits JSON must send the unsanitised value, because + // encoding/json escapes control characters anyway and stripping them would + // corrupt data the consumer is entitled to receive intact. + in := &flowctlpb.ServiceInfo{ServiceId: hostile} + _ = Message(in) + + if in.ServiceId != hostile { + t.Errorf("the input was mutated: %q", in.ServiceId) + } +} + +func TestMessageSanitisesRepeatedStrings(t *testing.T) { + in := &flowctlpb.ServiceInfo{ + ServiceId: "ok", + InputEventTypes: []string{"clean.v1", "evil\x1b[2K.v1"}, + OutputEventTypes: []string{"out\x07.v1"}, + } + out := Message(in) + + for _, list := range [][]string{out.InputEventTypes, out.OutputEventTypes} { + for _, v := range list { + for _, r := range v { + if isControl(r) { + t.Errorf("control char %U survived in %q", r, v) + } + } + } + } + if out.InputEventTypes[0] != "clean.v1" { + t.Errorf("clean value was altered: %q", out.InputEventTypes[0]) + } +} + +func TestMessageSanitisesMapKeysAndValues(t *testing.T) { + // A hostile component controls metadata keys as well as values, and a key + // is just as capable of carrying an escape sequence. + in := &flowctlpb.ServiceInfo{ + ServiceId: "ok", + Metadata: map[string]string{ + "ke\x1by": "value", + "network": "test\x1b]52;c;x\x07net", + }, + } + out := Message(in) + + for k, v := range out.Metadata { + for _, r := range k { + if isControl(r) { + t.Errorf("control char %U survived in key %q", r, k) + } + } + for _, r := range v { + if isControl(r) { + t.Errorf("control char %U survived in value %q", r, v) + } + } + } + if _, ok := out.Metadata["key"]; !ok { + t.Errorf("sanitised key missing: %#v", out.Metadata) + } + if got := out.Metadata["network"]; got != "test]52;c;xnet" { + t.Errorf("network = %q", got) + } +} + +func TestMessageRecursesIntoNestedMessages(t *testing.T) { + in := &flowctlv1.ComponentStatusResponse{ + Component: &flowctlv1.ComponentInfo{ + Id: hostile, + Name: "name\x1b[1A", + Endpoint: "host\x00:8080", + }, + } + out := Message(in) + + for _, v := range []string{out.Component.Id, out.Component.Name, out.Component.Endpoint} { + for _, r := range v { + if isControl(r) { + t.Fatalf("control char %U survived in %q", r, v) + } + } + } +} + +func TestMessageLeavesBytesFieldsAlone(t *testing.T) { + // Bytes carry encoded payloads, not display text. Stripping them would + // corrupt a ledger. + payload := []byte{0x00, 0x1b, 0x7f, 0x9b, 0xff} + in := &flowctlpb.Event{ + EventType: "evil\x1b[2K", + Payload: payload, + } + out := Message(in) + + if strings.ContainsRune(out.EventType, 0x1b) { + t.Errorf("EventType not sanitised: %q", out.EventType) + } + if string(out.Payload) != string(payload) { + t.Errorf("Payload was altered: % x", out.Payload) + } +} + +func TestMessageHandlesNil(t *testing.T) { + // So callers need not nil-check before sanitising. + var in *flowctlpb.ServiceInfo + if out := Message(in); out != nil { + t.Errorf("Message(nil) = %v, want nil", out) + } +} + +func TestMessageLeavesCleanMessagesEqual(t *testing.T) { + in := &flowctlpb.ServiceInfo{ + ServiceId: "stellar-live-source", + InputEventTypes: []string{"stellar.ledger.v1"}, + Metadata: map[string]string{"network": "testnet"}, + } + out := Message(in) + if !proto.Equal(in, out) { + t.Errorf("a clean message was altered:\n in: %v\nout: %v", in, out) + } +} diff --git a/internal/sanitize/sanitize.go b/internal/sanitize/sanitize.go new file mode 100644 index 0000000..1cdc7b2 --- /dev/null +++ b/internal/sanitize/sanitize.go @@ -0,0 +1,116 @@ +// Package sanitize strips terminal control characters from untrusted strings. +// +// flowctl's control plane accepts registrations from processes it did not start, and +// prints what they send — service names, component metadata, relayed error messages — +// to the operator's terminal. Without stripping, a crafted service name can emit its +// own escape sequences: change the window title, plant a misleading OSC 8 hyperlink, +// or on permissive terminals write to the clipboard with OSC 52. +// +// This implements rule CLI-22 of the OBSRVR Engineering Standard v2.0. The characters +// removed are C0 (U+0000-U+001F) except newline and tab, DEL (U+007F), and C1 +// (U+0080-U+009F). Removing ESC (U+001B) is what defeats the injection; the rest of +// the range is removed because it is never legitimate in a name or a message and +// terminals disagree about how to render it. +// +// JSON output does not need this: encoding/json escapes control characters, so they +// cannot terminate a string or reach the terminal as control codes. Applying it anyway +// would silently corrupt data that a consumer is entitled to receive intact. +package sanitize + +import ( + "strings" + "unicode/utf8" +) + +// String removes control characters from s, keeping newline and tab. +// +// Invalid UTF-8 is replaced with U+FFFD, so the result is always well-formed: a +// terminal fed a lone continuation byte may otherwise consume the following byte and +// resynchronise somewhere the caller did not intend. +func String(s string) string { + if isClean(s, true) { + return s + } + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + switch { + case r == utf8.RuneError: + // Either real U+FFFD or a decode failure; both render safely. + b.WriteRune(utf8.RuneError) + case r == '\n' || r == '\t': + b.WriteRune(r) + case isControl(r): + // dropped + default: + b.WriteRune(r) + } + } + return b.String() +} + +// Line removes control characters from s, including newline and tab. +// +// Use it for anything rendered into a single cell of a table or a single log field, +// where an embedded newline would let untrusted input forge an extra row or log line. +func Line(s string) string { + if isClean(s, false) { + return s + } + var b strings.Builder + b.Grow(len(s)) + for _, r := range s { + switch { + case r == utf8.RuneError: + b.WriteRune(utf8.RuneError) + case isControl(r): + // dropped + default: + b.WriteRune(r) + } + } + return b.String() +} + +// Strings applies Line to every element of in, returning a new slice. +func Strings(in []string) []string { + out := make([]string, len(in)) + for i, s := range in { + out[i] = Line(s) + } + return out +} + +// Map applies Line to every key and value of in, returning a new map. Component +// metadata arrives as a map[string]string and both halves reach the terminal. +func Map(in map[string]string) map[string]string { + if in == nil { + return nil + } + out := make(map[string]string, len(in)) + for k, v := range in { + out[Line(k)] = Line(v) + } + return out +} + +// isControl reports whether r is C0, DEL or C1. +func isControl(r rune) bool { + return r < 0x20 || r == 0x7f || (r >= 0x80 && r <= 0x9f) +} + +// isClean reports whether s needs no rewriting, so the common case allocates nothing. +func isClean(s string, allowWhitespace bool) bool { + if !utf8.ValidString(s) { + return false + } + for _, r := range s { + if allowWhitespace && (r == '\n' || r == '\t') { + continue + } + if isControl(r) { + return false + } + } + return true +} diff --git a/internal/sanitize/sanitize_test.go b/internal/sanitize/sanitize_test.go new file mode 100644 index 0000000..dd466c3 --- /dev/null +++ b/internal/sanitize/sanitize_test.go @@ -0,0 +1,178 @@ +package sanitize + +import ( + "strings" + "testing" +) + +func TestStringStripsInjectionAttempts(t *testing.T) { + // Each input is something a hostile component could put in a service name or + // relay in an error message. None of the output may contain ESC or any other + // control character, or the terminal will act on it (CLI-22). + tests := []struct { + name string + input string + want string + }{ + { + name: "OSC 7501 status spoof", + input: "svc\x1b]7501;state=done\x1b\\", + want: "svc]7501;state=done\\", + }, + { + name: "OSC 8 misleading hyperlink", + input: "ledger\x1b]8;;https://evil.example\x07click\x1b]8;;\x07", + want: "ledger]8;;https://evil.exampleclick]8;;", + }, + { + name: "OSC 52 clipboard write", + input: "\x1b]52;c;cm0gLXJmIH4=\x07", + want: "]52;c;cm0gLXJmIH4=", + }, + { + name: "window title set", + input: "\x1b]0;pwned\x07name", + want: "]0;pwnedname", + }, + { + name: "CSI cursor movement and erase", + input: "real\x1b[2K\x1b[1Afake", + want: "real[2K[1Afake", + }, + { + name: "carriage return overwrite", + input: "HEALTHY\rUNHEALTHY", + want: "HEALTHYUNHEALTHY", + }, + { + name: "DEL and C1 CSI", + input: "a\x7fb\u009bc", + want: "abc", + }, + { + name: "bell", + input: "ding\x07", + want: "ding", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := String(tc.input) + if got != tc.want { + t.Errorf("String(%q) = %q, want %q", tc.input, got, tc.want) + } + if strings.ContainsRune(got, 0x1b) { + t.Errorf("String(%q) left an ESC in %q", tc.input, got) + } + for _, r := range got { + if isControl(r) { + t.Errorf("String(%q) left control char %U in %q", tc.input, r, got) + } + } + }) + } +} + +func TestStringKeepsNewlineAndTab(t *testing.T) { + in := "line one\nline two\tcolumn" + if got := String(in); got != in { + t.Errorf("String(%q) = %q, want it unchanged", in, got) + } +} + +func TestLineStripsNewlineAndTab(t *testing.T) { + // A newline in a table cell lets untrusted input forge an extra row. + in := "svc\nFAKE-ID\tHEALTHY" + want := "svcFAKE-IDHEALTHY" + if got := Line(in); got != want { + t.Errorf("Line(%q) = %q, want %q", in, got, want) + } +} + +func TestPreservesLegitimateText(t *testing.T) { + // Sanitising must not mangle ordinary content, including non-ASCII, which + // appears in asset codes and contract metadata. + for _, in := range []string{ + "stellar-live-source", + "ttp-processor-v2", + "Test SDF Network ; September 2015", + "café résumé", + "日本語", + "emoji 🟢 status", + "GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3QJZNSR", + } { + if got := String(in); got != in { + t.Errorf("String(%q) = %q, want it unchanged", in, got) + } + if got := Line(in); got != in { + t.Errorf("Line(%q) = %q, want it unchanged", in, got) + } + } +} + +func TestInvalidUTF8BecomesReplacementChar(t *testing.T) { + // A lone continuation byte can make a terminal swallow the next byte and + // resynchronise somewhere unintended. + got := Line("ok\xffbad") + if !strings.ContainsRune(got, '�') { + t.Errorf("Line(%q) = %q, want a U+FFFD replacement", "ok\xffbad", got) + } + if strings.ContainsRune(got, 0xff) { + t.Errorf("Line left a raw 0xff byte in %q", got) + } +} + +func TestCleanStringsAreNotReallocated(t *testing.T) { + // The common case is clean input; it should not pay for a rebuild. + in := "already-clean-value" + if got := String(in); got != in { + t.Fatalf("String(%q) = %q", in, got) + } + if got := Line(in); got != in { + t.Fatalf("Line(%q) = %q", in, got) + } +} + +func TestMapSanitisesKeysAndValues(t *testing.T) { + in := map[string]string{"na\x1bme": "va\x1blue"} + got := Map(in) + if _, ok := got["name"]; !ok { + t.Errorf("Map did not sanitise the key: %#v", got) + } + if got["name"] != "value" { + t.Errorf("Map did not sanitise the value: %#v", got) + } + if Map(nil) != nil { + t.Error("Map(nil) should stay nil") + } +} + +func TestStrings(t *testing.T) { + got := Strings([]string{"a\x1bb", "clean"}) + if got[0] != "ab" || got[1] != "clean" { + t.Errorf("Strings = %#v", got) + } +} + +func FuzzLineNeverEmitsControlCharacters(f *testing.F) { + // TEST-3: the sanitiser is the last line of defence for untrusted input, so + // its invariant is fuzzed rather than only sampled. + for _, seed := range []string{ + "", "plain", "\x1b]52;c;x\x07", "\xff\xfe", "tab\there", "日本語", "\x9b31m", + } { + f.Add(seed) + } + f.Fuzz(func(t *testing.T, in string) { + for _, r := range Line(in) { + if isControl(r) { + t.Fatalf("Line(%q) emitted control char %U", in, r) + } + } + for _, r := range String(in) { + if r != '\n' && r != '\t' && isControl(r) { + t.Fatalf("String(%q) emitted control char %U", in, r) + } + } + }) +} diff --git a/internal/storage/memory_test.go b/internal/storage/memory_test.go index 4a1de10..91a9ea8 100644 --- a/internal/storage/memory_test.go +++ b/internal/storage/memory_test.go @@ -43,28 +43,28 @@ func createTestMemoryService(id string) *ServiceInfo { func TestMemoryStorage_Basic(t *testing.T) { ctx := context.Background() storage := NewMemoryStorage() - + if err := storage.Open(); err != nil { t.Fatalf("Failed to open storage: %v", err) } defer storage.Close() - + // Test registration service := createTestMemoryService("memory-test-1") if err := storage.RegisterService(ctx, service); err != nil { t.Fatalf("Failed to register service: %v", err) } - + // Test retrieval retrieved, err := storage.GetService(ctx, service.Info.Id) if err != nil { t.Fatalf("Failed to get service: %v", err) } if retrieved.Info.Id != service.Info.Id { - t.Errorf("Retrieved service ID does not match: got %s, want %s", + t.Errorf("Retrieved service ID does not match: got %s, want %s", retrieved.Info.Id, service.Info.Id) } - + // Test update err = storage.UpdateService(ctx, service.Info.Id, func(s *ServiceInfo) error { s.Status.Status = flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY @@ -73,7 +73,7 @@ func TestMemoryStorage_Basic(t *testing.T) { if err != nil { t.Fatalf("Failed to update service: %v", err) } - + // Verify update retrieved, err = storage.GetService(ctx, service.Info.Id) if err != nil { @@ -82,7 +82,7 @@ func TestMemoryStorage_Basic(t *testing.T) { if retrieved.Status.Status != flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY { t.Errorf("Update to service health did not take effect") } - + // Test list services, err := storage.ListServices(ctx) if err != nil { @@ -91,18 +91,18 @@ func TestMemoryStorage_Basic(t *testing.T) { if len(services) != 1 { t.Errorf("Expected 1 service, got %d", len(services)) } - + // Test delete if err := storage.DeleteService(ctx, service.Info.Id); err != nil { t.Fatalf("Failed to delete service: %v", err) } - + // Verify deletion _, err = storage.GetService(ctx, service.Info.Id) if !IsNotFound(err) { t.Errorf("Expected NotFound error after deletion, got: %v", err) } - + // List should be empty now services, err = storage.ListServices(ctx) if err != nil { @@ -116,16 +116,16 @@ func TestMemoryStorage_Basic(t *testing.T) { func TestMemoryStorage_Transaction(t *testing.T) { ctx := context.Background() storage := NewMemoryStorage() - + if err := storage.Open(); err != nil { t.Fatalf("Failed to open storage: %v", err) } defer storage.Close() - + // Test transaction service1 := createTestMemoryService("memory-txn-1") service2 := createTestMemoryService("memory-txn-2") - + err := storage.WithTransaction(ctx, func(txn Transaction) error { if err := txn.RegisterService(service1); err != nil { return err @@ -135,7 +135,7 @@ func TestMemoryStorage_Transaction(t *testing.T) { if err != nil { t.Fatalf("Transaction failed: %v", err) } - + // Verify both services were registered services, err := storage.ListServices(ctx) if err != nil { @@ -144,7 +144,7 @@ func TestMemoryStorage_Transaction(t *testing.T) { if len(services) != 2 { t.Errorf("Expected 2 services after transaction, got %d", len(services)) } - + // Test transaction with error err = storage.WithTransaction(ctx, func(txn Transaction) error { // Update a service @@ -154,18 +154,18 @@ func TestMemoryStorage_Transaction(t *testing.T) { }); err != nil { return err } - + // Try to update a non-existent service, which should fail return txn.UpdateService("non-existent", func(s *ServiceInfo) error { return nil }) }) - + // The transaction should have failed if err == nil { t.Errorf("Expected transaction to fail, but it succeeded") } - + // In memory storage doesn't support true transactions (rollback) // so we need to check if the update was applied despite the error retrieved, err := storage.GetService(ctx, service1.Info.Id) @@ -175,4 +175,4 @@ func TestMemoryStorage_Transaction(t *testing.T) { if retrieved.Status.Status != flowctlv1.HealthStatus_HEALTH_STATUS_UNHEALTHY { t.Errorf("Memory transaction doesn't support rollback, but this test still verifies behavior") } -} \ No newline at end of file +} diff --git a/internal/testfixtures/mocks.go b/internal/testfixtures/mocks.go index b2fa07b..5d38e49 100644 --- a/internal/testfixtures/mocks.go +++ b/internal/testfixtures/mocks.go @@ -42,4 +42,4 @@ func AsMockProcessor(proc processor.Processor) (*mockprocessor.MockProcessor, bo func AsMockSink(snk sink.Sink) (*mocksink.MockSink, bool) { mock, ok := snk.(*mocksink.MockSink) return mock, ok -} \ No newline at end of file +} diff --git a/internal/testfixtures/mocks_test.go b/internal/testfixtures/mocks_test.go index 4b7f38b..b3fee96 100644 --- a/internal/testfixtures/mocks_test.go +++ b/internal/testfixtures/mocks_test.go @@ -14,46 +14,46 @@ import ( func TestMockSource(t *testing.T) { // Create a mock source src := testfixtures.CreateMockSource() - + // Check if we can convert it back to a MockSource mockSrc, ok := testfixtures.AsMockSource(src) if !ok { t.Fatal("Failed to convert Source to MockSource") } - + // Create an event envelope event := source.EventEnvelope{ LedgerSeq: 123, Payload: []byte("test payload"), Cursor: "test-cursor", } - + // Create a channel and context ctx, cancel := context.WithCancel(context.Background()) defer cancel() - + eventChan := make(chan source.EventEnvelope, 1) - + // Start the source in a goroutine go func() { if err := src.Open(ctx); err != nil { t.Errorf("Failed to open source: %v", err) } - + if err := src.Events(ctx, eventChan); err != nil && err != context.Canceled { t.Errorf("Events returned error: %v", err) } }() - + // Emit an event mockSrc.EmitEvent(event) - + // Check that we received the event receivedEvent := <-eventChan if receivedEvent.LedgerSeq != event.LedgerSeq { t.Errorf("Expected ledger seq %d, got %d", event.LedgerSeq, receivedEvent.LedgerSeq) } - + // Cleanup if err := src.Close(); err != nil { t.Errorf("Failed to close source: %v", err) @@ -63,32 +63,32 @@ func TestMockSource(t *testing.T) { func TestMockProcessor(t *testing.T) { // Create a mock processor proc := testfixtures.CreateMockProcessor("test-processor") - + // Check if we can convert it back to a MockProcessor mockProc, ok := testfixtures.AsMockProcessor(proc) if !ok { t.Fatal("Failed to convert Processor to MockProcessor") } - + // Initialize the processor if err := proc.Init(map[string]any{"test": "config"}); err != nil { t.Fatalf("Failed to initialize processor: %v", err) } - + // Create an event envelope event := source.EventEnvelope{ LedgerSeq: 123, Payload: []byte("test payload"), Cursor: "test-cursor", } - + // Process the event ctx := context.Background() msgs, err := proc.Process(ctx, &event) if err != nil { t.Fatalf("Failed to process event: %v", err) } - + // Check the output if len(msgs) != 1 { t.Fatalf("Expected 1 message, got %d", len(msgs)) @@ -96,12 +96,12 @@ func TestMockProcessor(t *testing.T) { if msgs[0].Type != "mock" { t.Errorf("Expected message type 'mock', got '%s'", msgs[0].Type) } - + // Flush the processor if err := proc.Flush(ctx); err != nil { t.Fatalf("Failed to flush processor: %v", err) } - + // Check the stats stats := mockProc.Stats() if !stats["init_called"].(bool) { @@ -121,18 +121,18 @@ func TestMockProcessor(t *testing.T) { func TestMockSink(t *testing.T) { // Create a mock sink sink := testfixtures.CreateMockSink() - + // Check if we can convert it back to a MockSink mockSink, ok := testfixtures.AsMockSink(sink) if !ok { t.Fatal("Failed to convert Sink to MockSink") } - + // Connect to the sink if err := sink.Connect(map[string]any{"test": "config"}); err != nil { t.Fatalf("Failed to connect to sink: %v", err) } - + // Create some messages msgs := []*processor.Message{ { @@ -140,18 +140,18 @@ func TestMockSink(t *testing.T) { Payload: []byte("test payload"), }, } - + // Write the messages ctx := context.Background() if err := sink.Write(ctx, msgs); err != nil { t.Fatalf("Failed to write messages: %v", err) } - + // Close the sink if err := sink.Close(); err != nil { t.Fatalf("Failed to close sink: %v", err) } - + // Check the messages receivedMsgs := mockSink.Messages() if len(receivedMsgs) != 1 { @@ -163,7 +163,7 @@ func TestMockSink(t *testing.T) { if receivedMsgs[0][0].Type != "test-type" { t.Errorf("Expected message type 'test-type', got '%s'", receivedMsgs[0][0].Type) } - + // Check the stats stats := mockSink.Stats() if !stats["connect_called"].(bool) { @@ -178,4 +178,4 @@ func TestMockSink(t *testing.T) { if stats["message_count"].(int) != 1 { t.Errorf("Expected 1 batch of messages, got %d", stats["message_count"].(int)) } -} \ No newline at end of file +} diff --git a/internal/testfixtures/processor/mockprocessor.go b/internal/testfixtures/processor/mockprocessor.go index 4930284..2a13278 100644 --- a/internal/testfixtures/processor/mockprocessor.go +++ b/internal/testfixtures/processor/mockprocessor.go @@ -68,4 +68,4 @@ func (m *MockProcessor) Stats() map[string]interface{} { "flush_called": m.flushCalled, "processed_events": m.processedEvents, } -} \ No newline at end of file +} diff --git a/internal/testfixtures/production.go b/internal/testfixtures/production.go index 4c97091..0ffa169 100644 --- a/internal/testfixtures/production.go +++ b/internal/testfixtures/production.go @@ -95,4 +95,4 @@ func (p *placeholderProcessor) Flush(ctx context.Context) error { func (p *placeholderProcessor) Name() string { return p.name -} \ No newline at end of file +} diff --git a/internal/testfixtures/sink/mocksink.go b/internal/testfixtures/sink/mocksink.go index 84777e0..8c33e32 100644 --- a/internal/testfixtures/sink/mocksink.go +++ b/internal/testfixtures/sink/mocksink.go @@ -33,7 +33,7 @@ func (s *MockSink) Connect(cfg map[string]any) error { func (s *MockSink) Write(ctx context.Context, msgs []*processor.Message) error { s.mutex.Lock() defer s.mutex.Unlock() - + s.writeCalled = true s.messages = append(s.messages, msgs) return nil @@ -52,7 +52,7 @@ func (s *MockSink) Healthy() error { func (s *MockSink) Messages() [][]*processor.Message { s.mutex.Lock() defer s.mutex.Unlock() - + result := make([][]*processor.Message, len(s.messages)) copy(result, s.messages) return result @@ -62,11 +62,11 @@ func (s *MockSink) Messages() [][]*processor.Message { func (s *MockSink) Stats() map[string]interface{} { s.mutex.Lock() defer s.mutex.Unlock() - + return map[string]interface{}{ "connect_called": s.connectCalled, "write_called": s.writeCalled, "close_called": s.closeCalled, "message_count": len(s.messages), } -} \ No newline at end of file +} diff --git a/internal/testfixtures/source/mocksource.go b/internal/testfixtures/source/mocksource.go index c54231c..63eeece 100644 --- a/internal/testfixtures/source/mocksource.go +++ b/internal/testfixtures/source/mocksource.go @@ -47,4 +47,4 @@ func (m *MockSource) Healthy() error { // EmitEvent allows test code to inject events into the mock source func (m *MockSource) EmitEvent(event source.EventEnvelope) { m.ticker <- event -} \ No newline at end of file +} diff --git a/internal/translator/cue_validator.go b/internal/translator/cue_validator.go index 440259b..5fee2a8 100644 --- a/internal/translator/cue_validator.go +++ b/internal/translator/cue_validator.go @@ -81,4 +81,4 @@ func (v *CueValidator) ValidateSink(sink model.Component) error { func (v *CueValidator) ValidateConfig(cfg *model.Config) error { basicValidator := NewSchemaValidatorImpl() return basicValidator.ValidateConfig(cfg) -} \ No newline at end of file +} diff --git a/internal/translator/factory.go b/internal/translator/factory.go index 56680f5..0fdc12c 100644 --- a/internal/translator/factory.go +++ b/internal/translator/factory.go @@ -25,12 +25,12 @@ func NewDockerComposeGenerator() interfaces.Generator { func NewLocalGenerator() interfaces.Generator { // Load global config to determine which generator to use globalCfg := config.LoadGlobalConfig() - + if globalCfg.UseBashScriptGenerator() { // Use legacy bash script generator if specifically requested return generator.NewLegacyLocalGenerator() } - + // Default to Docker Compose based generator (modern approach) return generator.NewLocalGenerator() } @@ -41,4 +41,4 @@ func NewSchemaValidator() interfaces.Validator { // Note: The CUE validator has built-in fallback to the basic validator // when CUE schemas aren't available (handled in CueValidator.Validate) return NewCueValidator() -} \ No newline at end of file +} diff --git a/internal/translator/generators/docker_compose.go b/internal/translator/generators/docker_compose.go index 856deeb..cbae163 100644 --- a/internal/translator/generators/docker_compose.go +++ b/internal/translator/generators/docker_compose.go @@ -95,24 +95,24 @@ func (g *DockerComposeGenerator) Generate(pipeline *models.Pipeline, writer io.W // Process processors for _, processor := range pipeline.Spec.Processors { service := createDockerComposeService(processor, pipeline.Metadata.Name) - + // Add dependencies for _, input := range processor.Inputs { service.DependsOn = append(service.DependsOn, sanitizeServiceName(input)) } - + composeConfig.Services[sanitizeServiceName(processor.ID)] = service } // Process sinks for _, sink := range pipeline.Spec.Sinks { service := createDockerComposeService(sink, pipeline.Metadata.Name) - + // Add dependencies for _, input := range sink.Inputs { service.DependsOn = append(service.DependsOn, sanitizeServiceName(input)) } - + composeConfig.Services[sanitizeServiceName(sink.ID)] = service } @@ -144,9 +144,9 @@ func createDockerComposeService(component models.Component, pipelineName string) Networks: []string{"flow-network"}, Environment: make(map[string]string), Labels: map[string]string{ - "com.obsrvr.flow.pipeline": pipelineName, - "com.obsrvr.flow.component": component.ID, - "com.obsrvr.flow.type": component.Type, + "com.obsrvr.flow.pipeline": pipelineName, + "com.obsrvr.flow.component": component.ID, + "com.obsrvr.flow.type": component.Type, }, } @@ -177,11 +177,11 @@ func createDockerComposeService(component models.Component, pipelineName string) } else { portStr = fmt.Sprintf("%d", port.ContainerPort) } - + if port.Protocol != "" { portStr = fmt.Sprintf("%s/%s", portStr, port.Protocol) } - + service.Ports = append(service.Ports, portStr) } } @@ -211,4 +211,4 @@ func createDockerComposeService(component models.Component, pipelineName string) func sanitizeServiceName(id string) string { // Replace invalid characters with underscores return strings.ReplaceAll(id, "-", "_") -} \ No newline at end of file +} diff --git a/internal/translator/generators/generator.go b/internal/translator/generators/generator.go index 0c21b96..f3543ef 100644 --- a/internal/translator/generators/generator.go +++ b/internal/translator/generators/generator.go @@ -10,4 +10,4 @@ import ( type Generator interface { // Generate generates a deployment configuration for a pipeline Generate(pipeline *models.Pipeline, writer io.Writer) error -} \ No newline at end of file +} diff --git a/internal/translator/local.go b/internal/translator/local.go index 87b23b2..fd95166 100644 --- a/internal/translator/local.go +++ b/internal/translator/local.go @@ -381,8 +381,8 @@ func buildLocalComponent(id, name, cType string, params map[string]interface{}, // Format full health check URL if !strings.HasPrefix(component.HealthCheck, "http") { - component.HealthCheck = fmt.Sprintf("http://localhost:%d%s", - component.HealthCheckPort, + component.HealthCheck = fmt.Sprintf("http://localhost:%d%s", + component.HealthCheckPort, component.HealthCheck) } } @@ -400,4 +400,4 @@ func buildLocalComponent(id, name, cType string, params map[string]interface{}, } return component -} \ No newline at end of file +} diff --git a/internal/translator/models/pipeline.go b/internal/translator/models/pipeline.go index cd12fe8..87d0c14 100644 --- a/internal/translator/models/pipeline.go +++ b/internal/translator/models/pipeline.go @@ -18,10 +18,10 @@ type Metadata struct { // Spec contains the pipeline specification type Spec struct { - Description string `yaml:"description,omitempty"` - Sources []Component `yaml:"sources"` - Processors []Component `yaml:"processors"` - Sinks []Component `yaml:"sinks"` + Description string `yaml:"description,omitempty"` + Sources []Component `yaml:"sources"` + Processors []Component `yaml:"processors"` + Sinks []Component `yaml:"sinks"` Config map[string]interface{} `yaml:"config,omitempty"` } @@ -50,4 +50,4 @@ type Port struct { ContainerPort int `yaml:"containerPort"` HostPort int `yaml:"hostPort,omitempty"` Protocol string `yaml:"protocol,omitempty"` -} \ No newline at end of file +} diff --git a/internal/translator/parser.go b/internal/translator/parser.go index c00faa7..a005c60 100644 --- a/internal/translator/parser.go +++ b/internal/translator/parser.go @@ -66,7 +66,7 @@ func (p *YAMLParser) Parse(data []byte) (*model.Pipeline, error) { // Convert legacy config to Pipeline pipeline = convertConfigToPipeline(cfg) - + logger.Debug("Successfully converted legacy config to pipeline format", zap.String("version", cfg.Version)) @@ -145,7 +145,7 @@ func (p *JSONParser) Parse(data []byte) (*model.Pipeline, error) { // Convert legacy config to Pipeline pipeline = convertConfigToPipeline(cfg) - + logger.Debug("Successfully converted legacy config to pipeline format", zap.String("version", cfg.Version)) @@ -204,7 +204,7 @@ func convertConfigToPipeline(cfg *model.Config) model.Pipeline { Type: proc.Plugin, Inputs: []string{cfg.Source.Type}, // Default to source as input } - + // Extract common fields from params extractComponentFieldsFromParams(&processor, proc.Params) pipeline.Spec.Processors = append(pipeline.Spec.Processors, processor) @@ -215,7 +215,7 @@ func convertConfigToPipeline(cfg *model.Config) model.Pipeline { ID: cfg.Sink.Type, Type: cfg.Sink.Type, } - + // Set default input to last processor if available if len(cfg.Processors) > 0 { sink.Inputs = []string{cfg.Processors[len(cfg.Processors)-1].Name} @@ -223,7 +223,7 @@ func convertConfigToPipeline(cfg *model.Config) model.Pipeline { // Otherwise use source as input sink.Inputs = []string{cfg.Source.Type} } - + // Extract common fields from params extractComponentFieldsFromParams(&sink, cfg.Sink.Params) pipeline.Spec.Sinks = append(pipeline.Spec.Sinks, sink) @@ -237,7 +237,7 @@ func extractComponentFieldsFromParams(component *model.Component, params map[str if img, ok := params["image"].(string); ok { component.Image = img } - + // Extract command if cmd, ok := params["command"].([]string); ok { component.Command = cmd @@ -247,7 +247,7 @@ func extractComponentFieldsFromParams(component *model.Component, params map[str component.Command[i] = fmt.Sprintf("%v", c) } } - + // Extract environment variables if env, ok := params["env"].(map[string]string); ok { component.Env = env @@ -257,7 +257,7 @@ func extractComponentFieldsFromParams(component *model.Component, params map[str component.Env[k] = fmt.Sprintf("%v", v) } } - + // Extract ports if ports, ok := params["ports"].([]int); ok { component.Ports = make([]model.Port, len(ports)) @@ -287,12 +287,12 @@ func extractComponentFieldsFromParams(component *model.Component, params map[str } } } - + // Extract health check if health, ok := params["health_check"].(string); ok { component.HealthCheck = health } - + // Extract health port if port, ok := params["health_port"].(int); ok { component.HealthPort = port @@ -302,4 +302,4 @@ func extractComponentFieldsFromParams(component *model.Component, params map[str component.HealthPort = port } } -} \ No newline at end of file +} diff --git a/internal/translator/parsers/parser.go b/internal/translator/parsers/parser.go index 33fd258..601f3e2 100644 --- a/internal/translator/parsers/parser.go +++ b/internal/translator/parsers/parser.go @@ -8,4 +8,4 @@ import ( type Parser interface { // Parse parses a byte slice into a Pipeline object Parse(data []byte) (*models.Pipeline, error) -} \ No newline at end of file +} diff --git a/internal/translator/parsers/yaml_parser.go b/internal/translator/parsers/yaml_parser.go index 3d8b55f..7a58e37 100644 --- a/internal/translator/parsers/yaml_parser.go +++ b/internal/translator/parsers/yaml_parser.go @@ -3,6 +3,8 @@ package parsers import ( "fmt" + "github.com/withobsrvr/flowctl/internal/apiversion" + "github.com/withobsrvr/flowctl/internal/translator/models" "github.com/withobsrvr/flowctl/internal/utils/logger" "go.uber.org/zap" @@ -20,7 +22,7 @@ func NewYAMLParser() *YAMLParser { // Parse parses YAML data into a Pipeline object func (p *YAMLParser) Parse(data []byte) (*models.Pipeline, error) { var pipeline models.Pipeline - + err := yaml.Unmarshal(data, &pipeline) if err != nil { logger.Error("Failed to parse YAML", zap.Error(err)) @@ -45,23 +47,23 @@ func (p *YAMLParser) Parse(data []byte) (*models.Pipeline, error) { // validate checks if the parsed pipeline is valid func (p *YAMLParser) validate(pipeline *models.Pipeline) error { // Check required fields - if pipeline.APIVersion == "" { - return fmt.Errorf("apiVersion is required") + if err := apiversion.Validate(pipeline.APIVersion); err != nil { + return err } - - if pipeline.Kind != "Pipeline" { - return fmt.Errorf("kind must be 'Pipeline'") + + if err := apiversion.ValidateKind(pipeline.Kind); err != nil { + return err } - + if pipeline.Metadata.Name == "" { return fmt.Errorf("metadata.name is required") } - + // Check components if len(pipeline.Spec.Sources) == 0 { return fmt.Errorf("at least one source is required") } - + // Validate that each component has required fields for _, src := range pipeline.Spec.Sources { if src.ID == "" { @@ -71,7 +73,7 @@ func (p *YAMLParser) validate(pipeline *models.Pipeline) error { return fmt.Errorf("source image is required for %s", src.ID) } } - + for _, proc := range pipeline.Spec.Processors { if proc.ID == "" { return fmt.Errorf("processor id is required") @@ -83,7 +85,7 @@ func (p *YAMLParser) validate(pipeline *models.Pipeline) error { return fmt.Errorf("processor %s must have at least one input", proc.ID) } } - + for _, sink := range pipeline.Spec.Sinks { if sink.ID == "" { return fmt.Errorf("sink id is required") @@ -95,16 +97,16 @@ func (p *YAMLParser) validate(pipeline *models.Pipeline) error { return fmt.Errorf("sink %s must have at least one input", sink.ID) } } - + // Validate connections between components // (Check that all input references exist) componentIDs := make(map[string]bool) - + // Collect all component IDs for _, src := range pipeline.Spec.Sources { componentIDs[src.ID] = true } - + // Check processor inputs for _, proc := range pipeline.Spec.Processors { componentIDs[proc.ID] = true @@ -114,7 +116,7 @@ func (p *YAMLParser) validate(pipeline *models.Pipeline) error { } } } - + // Check sink inputs for _, sink := range pipeline.Spec.Sinks { for _, input := range sink.Inputs { @@ -123,6 +125,6 @@ func (p *YAMLParser) validate(pipeline *models.Pipeline) error { } } } - + return nil -} \ No newline at end of file +} diff --git a/internal/translator/translator.go b/internal/translator/translator.go index 961f80b..f70d193 100644 --- a/internal/translator/translator.go +++ b/internal/translator/translator.go @@ -138,4 +138,4 @@ func (t *translatorImpl) ValidFormats() []model.Format { formats = append(formats, format) } return formats -} \ No newline at end of file +} diff --git a/internal/translator/utils.go b/internal/translator/utils.go index 797769f..8b32759 100644 --- a/internal/translator/utils.go +++ b/internal/translator/utils.go @@ -1,7 +1,6 @@ package translator import ( - "fmt" "os" "path/filepath" ) @@ -29,195 +28,3 @@ func writeToFile(path string, data []byte) error { return createFile(path, data) } - -// findPort finds a port in a slice of ports that matches the format -func findPort(ports []int, basePort int) int { - if len(ports) == 0 { - return basePort - } - return ports[0] -} - -// generatePortMappings generates port mappings for Docker/K8s -func generatePortMappings(ports []int) []string { - if len(ports) == 0 { - return nil - } - - mappings := make([]string, len(ports)) - for i, p := range ports { - mappings[i] = fmt.Sprintf("%d:%d", p, p) - } - return mappings -} - -// mergeMaps merges two string maps with the second taking precedence -func mergeMaps(m1, m2 map[string]string) map[string]string { - result := make(map[string]string) - - // Copy first map - for k, v := range m1 { - result[k] = v - } - - // Merge second map - for k, v := range m2 { - result[k] = v - } - - return result -} - -// getComponentName extracts a standardized component name -func getComponentName(componentType, name string, prefix string) string { - baseName := name - if baseName == "" { - baseName = componentType - } - - if prefix != "" { - return fmt.Sprintf("%s-%s", prefix, baseName) - } - return baseName -} - -// extractParam extracts a parameter from a map as a certain type -func extractParam(params map[string]interface{}, key string, defaultValue interface{}) interface{} { - if val, ok := params[key]; ok { - return val - } - return defaultValue -} - -// extractStringParam extracts a string parameter from a map -func extractStringParam(params map[string]interface{}, key, defaultValue string) string { - val := extractParam(params, key, defaultValue) - if strVal, ok := val.(string); ok { - return strVal - } - return fmt.Sprintf("%v", val) -} - -// extractIntParam extracts an int parameter from a map -func extractIntParam(params map[string]interface{}, key string, defaultValue int) int { - val := extractParam(params, key, defaultValue) - if intVal, ok := val.(int); ok { - return intVal - } - if strVal, ok := val.(string); ok { - var intVal int - if _, err := fmt.Sscanf(strVal, "%d", &intVal); err == nil { - return intVal - } - } - return defaultValue -} - -// extractBoolParam extracts a boolean parameter from a map -func extractBoolParam(params map[string]interface{}, key string, defaultValue bool) bool { - val := extractParam(params, key, defaultValue) - if boolVal, ok := val.(bool); ok { - return boolVal - } - if strVal, ok := val.(string); ok { - if strVal == "true" { - return true - } else if strVal == "false" { - return false - } - } - return defaultValue -} - -// extractStringMapParam extracts a string map parameter from a map -func extractStringMapParam(params map[string]interface{}, key string) map[string]string { - result := make(map[string]string) - - val := extractParam(params, key, nil) - if val == nil { - return result - } - - if mapVal, ok := val.(map[string]interface{}); ok { - for k, v := range mapVal { - if strVal, ok := v.(string); ok { - result[k] = strVal - } else { - result[k] = fmt.Sprintf("%v", v) - } - } - } else if mapVal, ok := val.(map[string]string); ok { - for k, v := range mapVal { - result[k] = v - } - } - - return result -} - -// extractStringSliceParam extracts a string slice parameter from a map -func extractStringSliceParam(params map[string]interface{}, key string) []string { - val := extractParam(params, key, nil) - if val == nil { - return nil - } - - if sliceVal, ok := val.([]interface{}); ok { - result := make([]string, len(sliceVal)) - for i, v := range sliceVal { - if strVal, ok := v.(string); ok { - result[i] = strVal - } else { - result[i] = fmt.Sprintf("%v", v) - } - } - return result - } else if sliceVal, ok := val.([]string); ok { - return sliceVal - } - - return []string{fmt.Sprintf("%v", val)} -} - -// extractIntSliceParam extracts an int slice parameter from a map -func extractIntSliceParam(params map[string]interface{}, key string) []int { - val := extractParam(params, key, nil) - if val == nil { - return nil - } - - if sliceVal, ok := val.([]interface{}); ok { - result := make([]int, 0, len(sliceVal)) - for _, v := range sliceVal { - if intVal, ok := v.(int); ok { - result = append(result, intVal) - } else if strVal, ok := v.(string); ok { - var intVal int - if _, err := fmt.Sscanf(strVal, "%d", &intVal); err == nil { - result = append(result, intVal) - } - } else { - // Try to convert to int - var intVal int - if _, err := fmt.Sscanf(fmt.Sprintf("%v", v), "%d", &intVal); err == nil { - result = append(result, intVal) - } - } - } - return result - } else if sliceVal, ok := val.([]int); ok { - return sliceVal - } - - // Try to convert a single value to int - if intVal, ok := val.(int); ok { - return []int{intVal} - } else if strVal, ok := val.(string); ok { - var intVal int - if _, err := fmt.Sscanf(strVal, "%d", &intVal); err == nil { - return []int{intVal} - } - } - - return nil -} \ No newline at end of file diff --git a/internal/translator/validator.go b/internal/translator/validator.go index c596fea..f26d8ce 100644 --- a/internal/translator/validator.go +++ b/internal/translator/validator.go @@ -3,6 +3,8 @@ package translator import ( "fmt" + "github.com/withobsrvr/flowctl/internal/apiversion" + "github.com/withobsrvr/flowctl/internal/model" "github.com/withobsrvr/flowctl/internal/utils/logger" ) @@ -20,53 +22,53 @@ func (v *BasicValidator) Validate(pipeline *model.Pipeline) error { logger.Debug("Validating pipeline configuration") // Check required fields - if pipeline.APIVersion == "" { - return fmt.Errorf("apiVersion is required") + if err := apiversion.Validate(pipeline.APIVersion); err != nil { + return err } - - if pipeline.Kind != "Pipeline" { - return fmt.Errorf("kind must be 'Pipeline'") + + if err := apiversion.ValidateKind(pipeline.Kind); err != nil { + return err } - + if pipeline.Metadata.Name == "" { return fmt.Errorf("metadata.name is required") } - + // Check components - at least one source or pipeline component is required if len(pipeline.Spec.Sources) == 0 && len(pipeline.Spec.Pipelines) == 0 { return fmt.Errorf("at least one source or pipeline component is required") } - + // Validate that each component has required fields for _, source := range pipeline.Spec.Sources { if err := v.ValidateSource(source); err != nil { return err } } - + for _, processor := range pipeline.Spec.Processors { if err := v.ValidateProcessor(processor); err != nil { return err } } - + for _, sink := range pipeline.Spec.Sinks { if err := v.ValidateSink(sink); err != nil { return err } } - + // Validate pipeline components for _, pipelineComp := range pipeline.Spec.Pipelines { if err := v.ValidatePipeline(pipelineComp); err != nil { return err } } - + // Validate connections between components // (Check that all input references exist) componentIDs := make(map[string]bool) - + // Collect all component IDs for _, src := range pipeline.Spec.Sources { componentIDs[src.ID] = true @@ -74,7 +76,7 @@ func (v *BasicValidator) Validate(pipeline *model.Pipeline) error { for _, pipelineComp := range pipeline.Spec.Pipelines { componentIDs[pipelineComp.ID] = true } - + // Check processor inputs for _, proc := range pipeline.Spec.Processors { componentIDs[proc.ID] = true @@ -84,7 +86,7 @@ func (v *BasicValidator) Validate(pipeline *model.Pipeline) error { } } } - + // Check sink inputs for _, sink := range pipeline.Spec.Sinks { for _, input := range sink.Inputs { @@ -121,7 +123,7 @@ func (v *BasicValidator) ValidateSource(source model.Component) error { if source.Image == "" { return fmt.Errorf("source image is required for %s", source.ID) } - + // Source-specific validations can go here if source.HealthCheck != "" && source.HealthPort == 0 { return fmt.Errorf("source %s has health_check but no health_port specified", source.ID) @@ -178,7 +180,7 @@ func (v *BasicValidator) ValidatePipeline(pipeline model.Component) error { if pipeline.Image == "" { return fmt.Errorf("pipeline image is required for %s", pipeline.ID) } - + // Pipeline-specific validations if pipeline.Type != "" && pipeline.Type != "pipeline" { return fmt.Errorf("pipeline %s has invalid type: %s", pipeline.ID, pipeline.Type) @@ -221,4 +223,4 @@ func (v *BasicValidator) ValidateConfig(cfg *model.Config) error { } return nil -} \ No newline at end of file +} diff --git a/internal/utils/logger/logger.go b/internal/utils/logger/logger.go index 122f3d6..e6acb11 100644 --- a/internal/utils/logger/logger.go +++ b/internal/utils/logger/logger.go @@ -2,6 +2,7 @@ package logger import ( "fmt" + "time" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -19,19 +20,65 @@ func GetLogger() *zap.Logger { return log } -// Init initializes the logger with the specified level -func Init(level string) error { +// Options configure the global logger. +type Options struct { + // Color enables ANSI level colouring. Callers decide this, because only they + // know whether the destination is a terminal and whether NO_COLOR is set + // (CLI-7). The zero value is off, which is the safe default. + Color bool + // JSON forces structured JSON output. When unset, JSON is used unless Color is + // on, so an operator at a terminal gets readable console output and anything + // collecting logs gets parseable ones (OBS-1). + JSON bool + // Service and Version are attached to every line (OBS-1). + Service string + Version string +} + +// Option modifies Options. +type Option func(*Options) + +// WithColor enables or disables ANSI colour. +func WithColor(enabled bool) Option { return func(o *Options) { o.Color = enabled } } + +// WithJSON forces JSON encoding regardless of whether colour is enabled. +func WithJSON(enabled bool) Option { return func(o *Options) { o.JSON = enabled } } + +// WithService sets the service and version fields carried on every line. +func WithService(service, version string) Option { + return func(o *Options) { o.Service, o.Version = service, version } +} + +// Init initializes the logger with the specified level. +// +// Logs always go to stderr: stdout carries results only (CLI-5). +func Init(level string, opts ...Option) error { zapLevel, err := parseLevel(level) if err != nil { return err } + options := Options{Service: "flowctl"} + for _, opt := range opts { + opt(&options) + } + + encodeLevel := zapcore.CapitalLevelEncoder + encoding := "json" + switch { + case options.JSON: + // explicit JSON wins + case options.Color: + encoding = "console" + encodeLevel = zapcore.CapitalColorLevelEncoder + } + config := zap.Config{ Level: zap.NewAtomicLevelAt(zapLevel), Development: false, - Encoding: "console", + Encoding: encoding, EncoderConfig: zapcore.EncoderConfig{ - TimeKey: "ts", + TimeKey: "time", LevelKey: "level", NameKey: "logger", CallerKey: "caller", @@ -39,8 +86,8 @@ func Init(level string) error { MessageKey: "msg", StacktraceKey: "stacktrace", LineEnding: zapcore.DefaultLineEnding, - EncodeLevel: zapcore.CapitalColorLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, + EncodeLevel: encodeLevel, + EncodeTime: utcRFC3339, EncodeDuration: zapcore.StringDurationEncoder, EncodeCaller: zapcore.ShortCallerEncoder, }, @@ -53,11 +100,23 @@ func Init(level string) error { return fmt.Errorf("failed to initialize logger: %w", err) } + fields := []zap.Field{zap.String("service", options.Service)} + if options.Version != "" { + fields = append(fields, zap.String("version", options.Version)) + } + logger = logger.With(fields...) + log = logger sugar = logger.Sugar() return nil } +// utcRFC3339 encodes timestamps as RFC 3339 in UTC with a Z suffix, as DATA-3 +// requires. zap's own RFC3339 encoders emit a local offset instead. +func utcRFC3339(t time.Time, enc zapcore.PrimitiveArrayEncoder) { + enc.AppendString(t.UTC().Format(time.RFC3339Nano)) +} + // parseLevel converts a string log level to a zapcore.Level func parseLevel(level string) (zapcore.Level, error) { switch level { @@ -175,4 +234,4 @@ func Sync() error { return log.Sync() } return nil -} \ No newline at end of file +} diff --git a/internal/validator/validator_test.go b/internal/validator/validator_test.go index 5e5f8ad..c903683 100644 --- a/internal/validator/validator_test.go +++ b/internal/validator/validator_test.go @@ -11,7 +11,7 @@ func TestValidatorRejectsKnownInvalidPipeline(t *testing.T) { pipeline := &model.Pipeline{ APIVersion: "flowctl/v1", Kind: "Pipeline", - Metadata: model.Metadata{Name: "invalid-pipeline"}, + Metadata: model.Metadata{Name: "invalid-pipeline"}, Spec: model.Spec{ Driver: "unsupported-driver", Sources: []model.Component{{ @@ -54,9 +54,9 @@ func TestValidatorAcceptsQuickstartStylePipeline(t *testing.T) { pipeline := &model.Pipeline{ APIVersion: "flowctl/v1", Kind: "Pipeline", - Metadata: model.Metadata{Name: "quickstart"}, + Metadata: model.Metadata{Name: "quickstart"}, Spec: model.Spec{ - Driver: "process", + Driver: "process", Sources: []model.Component{{ID: "stellar-source", Type: "stellar-live-source@v1.0.0"}}, Processors: []model.Component{{ ID: "contract-events", diff --git a/pkg/console/heartbeat/client.go b/pkg/console/heartbeat/client.go index cd2b361..21378d7 100644 --- a/pkg/console/heartbeat/client.go +++ b/pkg/console/heartbeat/client.go @@ -21,7 +21,7 @@ type Client struct { sessionID string webhookSecret string httpClient *http.Client - ledgerCount int64 // Atomic counter + ledgerCount int64 // Atomic counter logger *log.Logger // Logger for heartbeat messages } diff --git a/pkg/runner/control_plane_client.go b/pkg/runner/control_plane_client.go index 651e9bd..55507d2 100644 --- a/pkg/runner/control_plane_client.go +++ b/pkg/runner/control_plane_client.go @@ -28,7 +28,7 @@ func NewControlPlaneClient(endpoint string, logger *zap.Logger) (*ControlPlaneCl if endpoint == "" { endpoint = "localhost:8080" } - + // Connect to control plane // Use WithBlock=false (default) to allow lazy connection, and enable waiting for ready opts := []grpc.DialOption{ @@ -39,7 +39,7 @@ func NewControlPlaneClient(endpoint string, logger *zap.Logger) (*ControlPlaneCl if err != nil { return nil, fmt.Errorf("failed to connect to control plane: %w", err) } - + return &ControlPlaneClientImpl{ conn: conn, client: pb.NewControlPlaneClient(conn), @@ -64,7 +64,7 @@ func (c *ControlPlaneClientImpl) RegisterPipeline(ctx context.Context, pipelineI MaxInflight: 100, // Default value Metadata: metadata, } - + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() @@ -73,11 +73,11 @@ func (c *ControlPlaneClientImpl) RegisterPipeline(ctx context.Context, pipelineI if err != nil { return fmt.Errorf("failed to register pipeline: %w", err) } - + c.logger.Info("Pipeline registered with control plane", zap.String("service_id", ack.ServiceId), zap.Strings("topics", ack.TopicNames)) - + return nil } @@ -91,7 +91,7 @@ func (c *ControlPlaneClientImpl) IsServiceRegistered(ctx context.Context, servic if err != nil { return false, fmt.Errorf("failed to list services: %w", err) } - + // Check if our service is in the list // Match by component_id (preferred) or service_id (legacy) for _, svc := range serviceList.Services { @@ -110,7 +110,7 @@ func (c *ControlPlaneClientImpl) SendHeartbeat(ctx context.Context, serviceID st Metrics: metrics, // Timestamp is set by the server } - + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() @@ -119,6 +119,6 @@ func (c *ControlPlaneClientImpl) SendHeartbeat(ctx context.Context, serviceID st if err != nil { return fmt.Errorf("failed to send heartbeat: %w", err) } - + return nil -} \ No newline at end of file +} diff --git a/pkg/runner/docker_cli_client.go b/pkg/runner/docker_cli_client.go index bae3091..d1ea493 100644 --- a/pkg/runner/docker_cli_client.go +++ b/pkg/runner/docker_cli_client.go @@ -71,61 +71,61 @@ func (d *DockerCLIClient) Close() error { // CreateContainer creates a new container using docker create func (d *DockerCLIClient) CreateContainer(ctx context.Context, config *ContainerConfig) (string, error) { args := []string{"create"} - + // Add name if config.Name != "" { args = append(args, "--name", config.Name) } - + // Add network mode if config.NetworkMode != "" { args = append(args, "--network", config.NetworkMode) } - + // Add restart policy if config.RestartPolicy != "" && config.RestartPolicy != "no" { args = append(args, "--restart", config.RestartPolicy) } - + // Add labels for k, v := range config.Labels { args = append(args, "--label", fmt.Sprintf("%s=%s", k, v)) } - + // Add environment variables for _, env := range config.Environment { args = append(args, "-e", env) } - + // Add volume mounts d.logger.Debug("Adding volumes to container", zap.String("name", config.Name), zap.Int("count", len(config.Volumes))) - + for _, vol := range config.Volumes { hostPath := vol.HostPath // Expand environment variables hostPath = os.ExpandEnv(hostPath) - + // Handle $HOME specially if not in environment if strings.Contains(hostPath, "$HOME") { home, _ := os.UserHomeDir() hostPath = strings.ReplaceAll(hostPath, "$HOME", home) } - + // Handle $PWD if not in environment if strings.Contains(hostPath, "$PWD") { pwd, _ := os.Getwd() hostPath = strings.ReplaceAll(hostPath, "$PWD", pwd) } - + mount := fmt.Sprintf("%s:%s", hostPath, vol.ContainerPath) if vol.ReadOnly { mount += ":ro" } args = append(args, "-v", mount) } - + // Add port mappings for _, port := range config.Ports { portMap := fmt.Sprintf("%d:%d", port.HostPort, port.ContainerPort) @@ -134,15 +134,15 @@ func (d *DockerCLIClient) CreateContainer(ctx context.Context, config *Container } args = append(args, "-p", portMap) } - + // Add image args = append(args, config.Image) - + // Add command if len(config.Command) > 0 { args = append(args, config.Command...) } - + // Run docker create cmd := exec.CommandContext(ctx, "docker", args...) d.logger.Debug("Docker create command", @@ -154,14 +154,14 @@ func (d *DockerCLIClient) CreateContainer(ctx context.Context, config *Container } return "", fmt.Errorf("docker create failed: %w", err) } - + // Extract container ID from output containerID := strings.TrimSpace(string(output)) d.logger.Info("Created container", zap.String("id", safeContainerIDPrefix(containerID)), zap.String("image", config.Image)) - + return containerID, nil } @@ -209,13 +209,13 @@ func (d *DockerCLIClient) WaitContainer(ctx context.Context, containerID string) if err != nil { return -1, fmt.Errorf("failed to wait for container: %w", err) } - + // Parse exit code var exitCode int if _, err := fmt.Sscanf(strings.TrimSpace(string(output)), "%d", &exitCode); err != nil { return -1, fmt.Errorf("failed to parse exit code: %w", err) } - + return exitCode, nil } @@ -226,7 +226,7 @@ func (d *DockerCLIClient) InspectContainer(ctx context.Context, containerID stri if err != nil { return nil, fmt.Errorf("failed to inspect container: %w", err) } - + // For now, return basic info // In a real implementation, we would parse the JSON output info := &ContainerInfo{ @@ -237,7 +237,7 @@ func (d *DockerCLIClient) InspectContainer(ctx context.Context, containerID stri Running: true, }, } - + // Check if container is actually running statusCmd := exec.CommandContext(ctx, "docker", "ps", "-q", "--filter", fmt.Sprintf("id=%s", containerID)) statusOutput, _ := statusCmd.Output() @@ -245,7 +245,7 @@ func (d *DockerCLIClient) InspectContainer(ctx context.Context, containerID stri info.State.Running = false info.State.Status = "stopped" } - + return info, nil } @@ -332,4 +332,4 @@ func (s *apiLogStream) Close() error { return fmt.Errorf("close errors: %v", errs) } return nil -} \ No newline at end of file +} diff --git a/pkg/runner/env_handler.go b/pkg/runner/env_handler.go index 9b41e5c..a3f84a7 100644 --- a/pkg/runner/env_handler.go +++ b/pkg/runner/env_handler.go @@ -24,25 +24,25 @@ func NewEnvHandler(logger *zap.Logger) *EnvHandler { func (h *EnvHandler) ProcessEnvironment(env map[string]string) ([]string, error) { // Start with system environment as base envMap := h.getSystemEnv() - + // Override with pipeline-specific environment for k, v := range env { // Expand variables in the value expanded := h.expandValue(v, envMap) envMap[k] = expanded - + h.logger.Debug("Set environment variable", zap.String("key", k), zap.String("original", v), zap.String("expanded", expanded)) } - + // Convert to slice format for Docker result := make([]string, 0, len(envMap)) for k, v := range envMap { result = append(result, fmt.Sprintf("%s=%s", k, v)) } - + return result, nil } @@ -50,22 +50,22 @@ func (h *EnvHandler) ProcessEnvironment(env map[string]string) ([]string, error) func (h *EnvHandler) expandValue(value string, env map[string]string) string { // Handle ${VAR} syntax expanded := value - + // Find all ${VAR} patterns for { start := strings.Index(expanded, "${") if start == -1 { break } - + end := strings.Index(expanded[start:], "}") if end == -1 { break } - + // Extract variable name varName := expanded[start+2 : start+end] - + // Look up value replacement := "" if val, ok := env[varName]; ok { @@ -73,11 +73,11 @@ func (h *EnvHandler) expandValue(value string, env map[string]string) string { } else if val, ok := os.LookupEnv(varName); ok { replacement = val } - + // Replace expanded = expanded[:start] + replacement + expanded[start+end+1:] } - + // Handle $VAR syntax parts := strings.Fields(expanded) for i, part := range parts { @@ -90,19 +90,19 @@ func (h *EnvHandler) expandValue(value string, env map[string]string) string { } } } - + // Rejoin if we split if len(parts) > 0 && strings.Contains(value, " ") { expanded = strings.Join(parts, " ") } - + return expanded } // getSystemEnv returns system environment as a map func (h *EnvHandler) getSystemEnv() map[string]string { env := make(map[string]string) - + // Include key system variables essentialVars := []string{ "PATH", @@ -113,13 +113,13 @@ func (h *EnvHandler) getSystemEnv() map[string]string { "TZ", "TMPDIR", } - + for _, key := range essentialVars { if val, ok := os.LookupEnv(key); ok { env[key] = val } } - + return env } @@ -137,13 +137,13 @@ func (h *EnvHandler) ValidateEnvironment(env map[string]string) error { zap.String("expanded", expanded)) } } - + // Validate key format if !isValidEnvKey(k) { return fmt.Errorf("invalid environment variable key: %s", k) } } - + return nil } @@ -152,19 +152,19 @@ func isValidEnvKey(key string) bool { if len(key) == 0 { return false } - + // Must start with letter or underscore if !isLetter(rune(key[0])) && key[0] != '_' { return false } - + // Rest must be alphanumeric or underscore for _, ch := range key[1:] { if !isAlphaNumeric(ch) && ch != '_' { return false } } - + return true } @@ -174,4 +174,4 @@ func isLetter(ch rune) bool { func isAlphaNumeric(ch rune) bool { return isLetter(ch) || (ch >= '0' && ch <= '9') -} \ No newline at end of file +} diff --git a/pkg/runner/health_monitor.go b/pkg/runner/health_monitor.go index e6f2fce..029b01c 100644 --- a/pkg/runner/health_monitor.go +++ b/pkg/runner/health_monitor.go @@ -33,11 +33,11 @@ func NewContainerHealthMonitor(containerID string, dockerClient DockerClient, co func (m *ContainerHealthMonitor) Start(ctx context.Context) { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - + m.logger.Info("Starting health monitoring for pipeline", zap.String("service_id", m.serviceID), zap.String("container_id", m.containerID)) - + for { select { case <-ctx.Done(): @@ -67,23 +67,23 @@ func (m *ContainerHealthMonitor) checkHealth(ctx context.Context) error { if err != nil { return err } - + // Build metrics metrics := map[string]float64{ "container_running": boolToFloat64(info.State.Running), - "exit_code": float64(info.State.ExitCode), + "exit_code": float64(info.State.ExitCode), } - + // Update health status m.healthy = info.State.Running && info.State.ExitCode == 0 - + // Send heartbeat to control plane if err := m.controlPlane.SendHeartbeat(ctx, m.serviceID, metrics); err != nil { m.logger.Warn("Failed to send heartbeat", zap.String("service_id", m.serviceID), zap.Error(err)) } - + return nil } @@ -93,4 +93,4 @@ func boolToFloat64(b bool) float64 { return 1.0 } return 0.0 -} \ No newline at end of file +} diff --git a/pkg/runner/interfaces.go b/pkg/runner/interfaces.go index 2003810..bc510a7 100644 --- a/pkg/runner/interfaces.go +++ b/pkg/runner/interfaces.go @@ -9,19 +9,19 @@ import ( type DockerClient interface { // CreateContainer creates a new container CreateContainer(ctx context.Context, config *ContainerConfig) (containerID string, err error) - + // StartContainer starts a created container StartContainer(ctx context.Context, containerID string) error - + // StopContainer stops a running container StopContainer(ctx context.Context, containerID string) error - + // WaitContainer waits for a container to exit WaitContainer(ctx context.Context, containerID string) (exitCode int, err error) - + // InspectContainer gets container information InspectContainer(ctx context.Context, containerID string) (*ContainerInfo, error) - + // GetLogs retrieves container logs GetLogs(ctx context.Context, containerID string, follow bool) (LogStream, error) } @@ -30,10 +30,10 @@ type DockerClient interface { type ControlPlaneClient interface { // RegisterPipeline registers a pipeline with the control plane RegisterPipeline(ctx context.Context, pipelineID string, metadata map[string]string) error - + // IsServiceRegistered checks if a service is registered IsServiceRegistered(ctx context.Context, serviceID string) (bool, error) - + // SendHeartbeat sends a heartbeat for a service SendHeartbeat(ctx context.Context, serviceID string, metrics map[string]float64) error } @@ -42,7 +42,7 @@ type ControlPlaneClient interface { type HealthChecker interface { // Start begins health monitoring Start(ctx context.Context) - + // IsHealthy checks current health status IsHealthy() bool } @@ -81,10 +81,10 @@ type PortMapping struct { // ContainerInfo represents container information type ContainerInfo struct { - ID string - State ContainerState - Config ContainerConfig - Created string + ID string + State ContainerState + Config ContainerConfig + Created string } // ContainerState represents container state @@ -95,4 +95,4 @@ type ContainerState struct { StartedAt string FinishedAt string Error string -} \ No newline at end of file +} diff --git a/pkg/runner/pipeline_runner.go b/pkg/runner/pipeline_runner.go index c4496fc..8e67086 100644 --- a/pkg/runner/pipeline_runner.go +++ b/pkg/runner/pipeline_runner.go @@ -14,14 +14,14 @@ import ( type PipelineRunner struct { // Pipeline configuration Pipeline model.Component - + // Container management - containerID string - dockerClient DockerClient - + containerID string + dockerClient DockerClient + // Control plane integration - controlPlane ControlPlaneClient - + controlPlane ControlPlaneClient + // Lifecycle management ctx context.Context cancel context.CancelFunc @@ -42,7 +42,7 @@ func NewPipelineRunner(pipeline model.Component, dockerClient DockerClient, cont // Start starts the pipeline container func (r *PipelineRunner) Start(ctx context.Context) error { r.ctx, r.cancel = context.WithCancel(ctx) - + r.logger.Info("Starting pipeline component", zap.String("id", r.Pipeline.ID), zap.String("image", r.Pipeline.Image), @@ -64,31 +64,31 @@ func (r *PipelineRunner) Start(ctx context.Context) error { if config == nil { return fmt.Errorf("failed to build container configuration") } - + // Create and start container containerID, err := r.dockerClient.CreateContainer(ctx, config) if err != nil { return fmt.Errorf("failed to create container: %w", err) } r.containerID = containerID - + if err := r.dockerClient.StartContainer(ctx, containerID); err != nil { return fmt.Errorf("failed to start container: %w", err) } - + r.logger.Info("Pipeline container started", zap.String("id", r.Pipeline.ID), zap.String("container_id", containerID)) - + // Handle pipeline registration if err := r.handleRegistration(ctx); err != nil { return fmt.Errorf("failed to handle registration: %w", err) } - + // Start health monitoring r.healthChecker = NewContainerHealthMonitor(r.containerID, r.dockerClient, r.controlPlane, r.Pipeline.ID, r.logger) go r.healthChecker.Start(r.ctx) - + return nil } @@ -97,23 +97,23 @@ func (r *PipelineRunner) Stop(ctx context.Context) error { if r.cancel != nil { r.cancel() } - + if r.containerID == "" { return nil } - + r.logger.Info("Stopping pipeline container", zap.String("id", r.Pipeline.ID), zap.String("container_id", r.containerID)) - + // Stop container with timeout stopCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - + if err := r.dockerClient.StopContainer(stopCtx, r.containerID); err != nil { return fmt.Errorf("failed to stop container: %w", err) } - + return nil } @@ -122,17 +122,17 @@ func (r *PipelineRunner) Wait(ctx context.Context) error { if r.containerID == "" { return fmt.Errorf("container not started") } - + // Wait for container to exit exitCode, err := r.dockerClient.WaitContainer(ctx, r.containerID) if err != nil { return fmt.Errorf("error waiting for container: %w", err) } - + if exitCode != 0 { return fmt.Errorf("container exited with non-zero code: %d", exitCode) } - + return nil } @@ -141,7 +141,7 @@ func (r *PipelineRunner) GetLogs(ctx context.Context, follow bool) (LogStream, e if r.containerID == "" { return nil, fmt.Errorf("container not started") } - + return r.dockerClient.GetLogs(ctx, r.containerID, follow) } @@ -155,17 +155,17 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { if len(r.Pipeline.Args) > 0 { cmd = append(cmd, r.Pipeline.Args...) } - + // Build environment variables with expansion envHandler := NewEnvHandler(r.logger) - + // Validate environment first if err := envHandler.ValidateEnvironment(r.Pipeline.Env); err != nil { r.logger.Warn("Environment validation warning", zap.String("pipeline", r.Pipeline.ID), zap.Error(err)) } - + // Process and expand environment variables env, err := envHandler.ProcessEnvironment(r.Pipeline.Env) if err != nil { @@ -175,10 +175,10 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { // Continue with empty environment env = []string{} } - + // Build volume mounts with proper path resolution volumeHandler := NewVolumeHandler(r.logger) - + // Convert model volumes to handler volumes handlerVolumes := make([]Volume, 0, len(r.Pipeline.Volumes)) for _, v := range r.Pipeline.Volumes { @@ -189,12 +189,12 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { ReadOnly: v.ReadOnly, }) } - + // Process volumes with validation and path resolution r.logger.Debug("Processing volumes", zap.String("pipeline", r.Pipeline.ID), zap.Int("count", len(handlerVolumes))) - + volumes, err := volumeHandler.ProcessVolumes(handlerVolumes) if err != nil { r.logger.Error("Failed to process volumes", @@ -202,15 +202,15 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { zap.Error(err)) return nil } - + r.logger.Debug("Processed volumes", zap.String("pipeline", r.Pipeline.ID), zap.Int("count", len(volumes))) - + // Process secrets if any if len(r.Pipeline.Secrets) > 0 { secretHandler := NewSecretHandler(r.logger) - + // Convert model secrets to handler secrets handlerSecrets := make([]SecretMount, 0, len(r.Pipeline.Secrets)) for _, s := range r.Pipeline.Secrets { @@ -225,7 +225,7 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { Labels: s.Labels, }) } - + // Process secrets to get volumes and env vars secretVolumes, secretEnvs, err := secretHandler.ProcessSecrets(handlerSecrets) if err != nil { @@ -235,16 +235,16 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { // Fail if secrets are required return nil } - + // Add secret volumes volumes = append(volumes, secretVolumes...) - + // Add secret environment variables for k, v := range secretEnvs { env = append(env, fmt.Sprintf("%s=%s", k, v)) } } - + // Build port mappings ports := make([]PortMapping, 0, len(r.Pipeline.Ports)) for _, p := range r.Pipeline.Ports { @@ -254,7 +254,7 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { Protocol: p.Protocol, }) } - + return &ContainerConfig{ Name: fmt.Sprintf("flowctl-pipeline-%s", r.Pipeline.ID), Image: r.Pipeline.Image, @@ -265,8 +265,8 @@ func (r *PipelineRunner) buildContainerConfig() *ContainerConfig { NetworkMode: "host", // Use host network for control plane connectivity RestartPolicy: r.Pipeline.RestartPolicy, Labels: map[string]string{ - "flowctl.component": "pipeline", - "flowctl.pipeline.id": r.Pipeline.ID, + "flowctl.component": "pipeline", + "flowctl.pipeline.id": r.Pipeline.ID, "flowctl.pipeline.type": r.Pipeline.Type, }, } @@ -293,10 +293,10 @@ func (r *PipelineRunner) handleRegistration(ctx context.Context) error { func (r *PipelineRunner) waitForPipelineRegistration(ctx context.Context, timeout time.Duration) error { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - + ticker := time.NewTicker(1 * time.Second) defer ticker.Stop() - + for { select { case <-ctx.Done(): @@ -321,11 +321,11 @@ func (r *PipelineRunner) registerPipelineWithControlPlane(ctx context.Context) e "pipeline_type": "complete", "managed_by": "flowctl", } - + // Add container ID to metadata if r.containerID != "" { metadata["container_id"] = r.containerID } - + return r.controlPlane.RegisterPipeline(ctx, r.Pipeline.ID, metadata) -} \ No newline at end of file +} diff --git a/pkg/runner/secret_handler.go b/pkg/runner/secret_handler.go index 52f8b26..df78779 100644 --- a/pkg/runner/secret_handler.go +++ b/pkg/runner/secret_handler.go @@ -16,11 +16,11 @@ type SecretMount struct { Name string `yaml:"name" json:"name"` HostPath string `yaml:"host_path" json:"host_path"` ContainerPath string `yaml:"container_path" json:"container_path"` - EnvVar string `yaml:"env" json:"env"` // Optional: set as environment variable - Mode string `yaml:"mode" json:"mode"` // Optional: file permissions (e.g., "0600") - Type string `yaml:"type" json:"type"` // Optional: secret type (file, dir, env) - Required bool `yaml:"required" json:"required"` // Optional: fail if secret missing - Labels map[string]string `yaml:"labels" json:"labels"` // Optional: metadata labels + EnvVar string `yaml:"env" json:"env"` // Optional: set as environment variable + Mode string `yaml:"mode" json:"mode"` // Optional: file permissions (e.g., "0600") + Type string `yaml:"type" json:"type"` // Optional: secret type (file, dir, env) + Required bool `yaml:"required" json:"required"` // Optional: fail if secret missing + Labels map[string]string `yaml:"labels" json:"labels"` // Optional: metadata labels } // SecretHandler handles secure mounting of secrets and credentials @@ -41,7 +41,7 @@ func NewSecretHandler(logger *zap.Logger) *SecretHandler { func (h *SecretHandler) ProcessSecrets(secrets []SecretMount) ([]VolumeMount, map[string]string, error) { volumes := make([]VolumeMount, 0) envVars := make(map[string]string) - + for _, secret := range secrets { // Validate secret configuration if err := h.validateSecret(secret); err != nil { @@ -53,7 +53,7 @@ func (h *SecretHandler) ProcessSecrets(secrets []SecretMount) ([]VolumeMount, ma zap.Error(err)) continue } - + // Process based on type switch secret.Type { case "env", "": @@ -71,7 +71,7 @@ func (h *SecretHandler) ProcessSecrets(secrets []SecretMount) ([]VolumeMount, ma } envVars[secret.EnvVar] = value } - + case "file", "dir": // File or directory mount volume, env, err := h.processSecretMount(secret) @@ -84,22 +84,22 @@ func (h *SecretHandler) ProcessSecrets(secrets []SecretMount) ([]VolumeMount, ma zap.Error(err)) continue } - + volumes = append(volumes, volume) - + // Add environment variable if specified if secret.EnvVar != "" { envVars[secret.EnvVar] = env } - + default: return nil, nil, fmt.Errorf("unknown secret type: %s", secret.Type) } - + // Log secret handling (without exposing values) h.logSecretHandling(secret) } - + return volumes, envVars, nil } @@ -108,11 +108,11 @@ func (h *SecretHandler) validateSecret(secret SecretMount) error { if secret.Name == "" { return fmt.Errorf("secret name is required") } - + if secret.HostPath == "" && secret.Type != "env" { return fmt.Errorf("host path is required for secret %s", secret.Name) } - + // Validate secret type validTypes := []string{"file", "dir", "env", ""} validType := false @@ -125,14 +125,14 @@ func (h *SecretHandler) validateSecret(secret SecretMount) error { if !validType { return fmt.Errorf("invalid secret type: %s", secret.Type) } - + // Validate file mode if specified if secret.Mode != "" { if _, err := parseFileMode(secret.Mode); err != nil { return fmt.Errorf("invalid file mode %s: %w", secret.Mode, err) } } - + return nil } @@ -143,32 +143,32 @@ func (h *SecretHandler) processSecretMount(secret SecretMount) (VolumeMount, str if err != nil { return VolumeMount{}, "", fmt.Errorf("failed to resolve secret path: %w", err) } - + // Check if secret exists and is readable if err := h.checkSecretAccess(hostPath); err != nil { return VolumeMount{}, "", fmt.Errorf("secret access denied: %w", err) } - + // Determine container path containerPath := secret.ContainerPath if containerPath == "" { // Default to /secrets/ containerPath = filepath.Join("/secrets", secret.Name) } - + // Create volume mount mount := VolumeMount{ HostPath: hostPath, ContainerPath: containerPath, ReadOnly: true, // Secrets are always read-only } - + // Return environment value if needed envValue := containerPath if secret.Type == "file" && secret.EnvVar != "" { envValue = containerPath } - + return mount, envValue, nil } @@ -179,21 +179,21 @@ func (h *SecretHandler) readSecretValue(path string) (string, error) { if err != nil { return "", err } - + // Check access if err := h.checkSecretAccess(resolvedPath); err != nil { return "", err } - + // Read file content, err := os.ReadFile(resolvedPath) if err != nil { return "", fmt.Errorf("failed to read secret file: %w", err) } - + // Trim whitespace value := strings.TrimSpace(string(content)) - + return value, nil } @@ -203,24 +203,24 @@ func (h *SecretHandler) checkSecretAccess(path string) error { if err != nil { return fmt.Errorf("secret not found: %w", err) } - + // Check permissions mode := info.Mode() - + // For files, check if it's not world-readable if !info.IsDir() && mode.Perm()&0004 != 0 { h.logger.Warn("Secret file is world-readable", zap.String("path", path), zap.String("mode", mode.String())) } - + // Check if we can read it file, err := os.Open(path) if err != nil { return fmt.Errorf("cannot read secret: %w", err) } file.Close() - + return nil } @@ -229,7 +229,7 @@ func (h *SecretHandler) logSecretHandling(secret SecretMount) { // Create hash of the host path for tracking hash := sha256.Sum256([]byte(secret.HostPath)) pathHash := hex.EncodeToString(hash[:8]) - + h.logger.Info("Processing secret", zap.String("name", secret.Name), zap.String("type", secret.Type), @@ -242,28 +242,30 @@ func parseFileMode(mode string) (os.FileMode, error) { if len(mode) != 4 || mode[0] != '0' { return 0, fmt.Errorf("mode must be in format '0xxx'") } - + var perm os.FileMode for i := 1; i < 4; i++ { + // mode[i] is a byte: a char below '0' underflows to >= 248, which the + // upper bound catches. digit <= 7 therefore means exactly '0'-'7'. digit := mode[i] - '0' - if digit < 0 || digit > 7 { + if digit > 7 { return 0, fmt.Errorf("invalid digit in mode: %c", mode[i]) } perm = perm*8 + os.FileMode(digit) } - + return perm, nil } // GetSecretEnvVars returns environment variables for secrets func (h *SecretHandler) GetSecretEnvVars(secrets []SecretMount) map[string]string { envVars := make(map[string]string) - + for _, secret := range secrets { if secret.EnvVar != "" && secret.ContainerPath != "" { envVars[secret.EnvVar] = secret.ContainerPath } } - + return envVars -} \ No newline at end of file +} diff --git a/pkg/runner/volume_handler.go b/pkg/runner/volume_handler.go index 666d12a..553b6d3 100644 --- a/pkg/runner/volume_handler.go +++ b/pkg/runner/volume_handler.go @@ -24,7 +24,7 @@ func NewVolumeHandler(logger *zap.Logger) *VolumeHandler { // ProcessVolumes processes and validates volume mounts func (h *VolumeHandler) ProcessVolumes(volumes []Volume) ([]VolumeMount, error) { processed := make([]VolumeMount, 0, len(volumes)) - + for _, vol := range volumes { mount, err := h.processVolume(vol) if err != nil { @@ -32,7 +32,7 @@ func (h *VolumeHandler) ProcessVolumes(volumes []Volume) ([]VolumeMount, error) } processed = append(processed, mount) } - + return processed, nil } @@ -43,7 +43,7 @@ func (h *VolumeHandler) processVolume(vol Volume) (VolumeMount, error) { if err != nil { return VolumeMount{}, fmt.Errorf("failed to resolve host path: %w", err) } - + // Determine container path containerPath := vol.ContainerPath if containerPath == "" { @@ -52,12 +52,12 @@ func (h *VolumeHandler) processVolume(vol Volume) (VolumeMount, error) { if containerPath == "" { return VolumeMount{}, fmt.Errorf("container path not specified") } - + // Validate paths if err := h.validatePaths(hostPath, containerPath); err != nil { return VolumeMount{}, err } - + // Check if host path exists and is accessible if err := h.checkHostPath(hostPath, vol.ReadOnly); err != nil { h.logger.Warn("Host path validation warning", @@ -65,19 +65,19 @@ func (h *VolumeHandler) processVolume(vol Volume) (VolumeMount, error) { zap.Error(err)) // Continue anyway - Docker might create it } - + mount := VolumeMount{ HostPath: hostPath, ContainerPath: containerPath, ReadOnly: vol.ReadOnly, } - + h.logger.Debug("Processed volume mount", zap.String("name", vol.Name), zap.String("host", mount.HostPath), zap.String("container", mount.ContainerPath), zap.Bool("readonly", mount.ReadOnly)) - + return mount, nil } @@ -86,10 +86,10 @@ func (h *VolumeHandler) resolveHostPath(path string) (string, error) { if path == "" { return "", fmt.Errorf("host path is empty") } - + // Expand environment variables expanded := h.expandEnvVars(path) - + // Handle home directory expansion if strings.HasPrefix(expanded, "~/") { home, err := os.UserHomeDir() @@ -98,7 +98,7 @@ func (h *VolumeHandler) resolveHostPath(path string) (string, error) { } expanded = filepath.Join(home, expanded[2:]) } - + // Convert to absolute path if relative if !filepath.IsAbs(expanded) { abs, err := filepath.Abs(expanded) @@ -107,10 +107,10 @@ func (h *VolumeHandler) resolveHostPath(path string) (string, error) { } expanded = abs } - + // Clean the path cleaned := filepath.Clean(expanded) - + return cleaned, nil } @@ -118,25 +118,25 @@ func (h *VolumeHandler) resolveHostPath(path string) (string, error) { func (h *VolumeHandler) expandEnvVars(path string) string { // First pass: expand $VAR and ${VAR} syntax expanded := os.ExpandEnv(path) - + // Second pass: handle $HOME specially if not set if strings.Contains(expanded, "$HOME") { home, _ := os.UserHomeDir() expanded = strings.ReplaceAll(expanded, "$HOME", home) } - + // Handle common variables that might not be in environment replacements := map[string]string{ "$PWD": mustGetwd(), "$USER": os.Getenv("USER"), } - + for placeholder, value := range replacements { if value != "" { expanded = strings.ReplaceAll(expanded, placeholder, value) } } - + return expanded } @@ -146,16 +146,16 @@ func (h *VolumeHandler) validatePaths(hostPath, containerPath string) error { if hostPath == "" { return fmt.Errorf("host path is empty after resolution") } - + // Validate container path if containerPath == "" { return fmt.Errorf("container path is empty") } - + if !filepath.IsAbs(containerPath) { return fmt.Errorf("container path must be absolute: %s", containerPath) } - + // Check for dangerous paths dangerousPaths := []string{"/", "/etc", "/bin", "/sbin", "/usr", "/lib", "/var"} for _, dangerous := range dangerousPaths { @@ -163,7 +163,7 @@ func (h *VolumeHandler) validatePaths(hostPath, containerPath string) error { return fmt.Errorf("mounting system directory %s is not allowed", dangerous) } } - + return nil } @@ -176,19 +176,19 @@ func (h *VolumeHandler) checkHostPath(hostPath string, readOnly bool) error { } return fmt.Errorf("failed to stat host path: %w", err) } - + // Check read permissions if err := h.checkReadAccess(hostPath); err != nil { return fmt.Errorf("no read access to host path: %w", err) } - + // Check write permissions if not read-only if !readOnly && !info.IsDir() { if err := h.checkWriteAccess(hostPath); err != nil { return fmt.Errorf("no write access to host path: %w", err) } } - + return nil } @@ -208,7 +208,7 @@ func (h *VolumeHandler) checkWriteAccess(path string) error { if err != nil { return err } - + if info.IsDir() { // For directories, check if we can create a temp file testFile := filepath.Join(path, ".flowctl-write-test") @@ -220,7 +220,7 @@ func (h *VolumeHandler) checkWriteAccess(path string) error { os.Remove(testFile) return nil } - + // For files, check if we can open for writing file, err := os.OpenFile(path, os.O_WRONLY, 0) if err != nil { @@ -246,4 +246,4 @@ type Volume struct { ContainerPath string HostPath string ReadOnly bool -} \ No newline at end of file +} diff --git a/proto/common.pb.go b/proto/common.pb.go index e1356cd..34af20c 100644 --- a/proto/common.pb.go +++ b/proto/common.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 -// protoc v7.34.0 +// protoc-gen-go v1.36.12 +// protoc (unknown) // source: proto/common.proto package flowctlpb diff --git a/proto/control_plane.pb.go b/proto/control_plane.pb.go index 89246cf..300db97 100644 --- a/proto/control_plane.pb.go +++ b/proto/control_plane.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.10 -// protoc v6.32.1 +// protoc-gen-go v1.36.12 +// protoc (unknown) // source: proto/control_plane.proto package flowctlpb diff --git a/proto/control_plane_grpc.pb.go b/proto/control_plane_grpc.pb.go index 2abcb2c..fdd4a7c 100644 --- a/proto/control_plane_grpc.pb.go +++ b/proto/control_plane_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.6.0 -// - protoc v6.32.1 +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) // source: proto/control_plane.proto package flowctlpb diff --git a/proto/processor.pb.go b/proto/processor.pb.go index d3f13b1..f55ef28 100644 --- a/proto/processor.pb.go +++ b/proto/processor.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 -// protoc v7.34.0 +// protoc-gen-go v1.36.12 +// protoc (unknown) // source: proto/processor.proto package flowctlpb diff --git a/proto/processor_grpc.pb.go b/proto/processor_grpc.pb.go index 44dee45..a30d837 100644 --- a/proto/processor_grpc.pb.go +++ b/proto/processor_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.6.0 -// - protoc v7.34.0 +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) // source: proto/processor.proto package flowctlpb diff --git a/proto/sink.pb.go b/proto/sink.pb.go index 77d134e..56fb914 100644 --- a/proto/sink.pb.go +++ b/proto/sink.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 -// protoc v7.34.0 +// protoc-gen-go v1.36.12 +// protoc (unknown) // source: proto/sink.proto package flowctlpb diff --git a/proto/sink_grpc.pb.go b/proto/sink_grpc.pb.go index b34de74..28b26fd 100644 --- a/proto/sink_grpc.pb.go +++ b/proto/sink_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.6.0 -// - protoc v7.34.0 +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) // source: proto/sink.proto package flowctlpb diff --git a/proto/source.pb.go b/proto/source.pb.go index 0fa7c6b..f396853 100644 --- a/proto/source.pb.go +++ b/proto/source.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 -// protoc v7.34.0 +// protoc-gen-go v1.36.12 +// protoc (unknown) // source: proto/source.proto package flowctlpb diff --git a/proto/source_grpc.pb.go b/proto/source_grpc.pb.go index a4c6c1b..d0e4bb2 100644 --- a/proto/source_grpc.pb.go +++ b/proto/source_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.6.0 -// - protoc v7.34.0 +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) // source: proto/source.proto package flowctlpb diff --git a/scripts/generate.sh b/scripts/generate.sh index 3006ca6..c7509d3 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -1,10 +1,15 @@ -#!/bin/bash +#!/usr/bin/env bash +# +# Regenerate Go code from the protobuf definitions. +# +# buf is the single generator for this repo, with plugin versions pinned in +# buf.gen.yaml so that regenerating produces identical bytes everywhere. This +# used to call a locally installed protoc, which made the output depend on +# whichever plugin versions a developer happened to have. +# +# Equivalent to `make generate`. -# Install protoc-gen-go and protoc-gen-go-grpc if not already installed -go install google.golang.org/protobuf/cmd/protoc-gen-go@latest -go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest +set -euo pipefail -# Generate Go code from proto files -protoc --go_out=. --go_opt=paths=source_relative \ - --go-grpc_out=. --go-grpc_opt=paths=source_relative \ - proto/*.proto \ No newline at end of file +cd "$(dirname "$0")/.." +exec go run github.com/bufbuild/buf/cmd/buf@v1.47.2 generate diff --git a/scripts/govulncheck.sh b/scripts/govulncheck.sh new file mode 100755 index 0000000..cdb763a --- /dev/null +++ b/scripts/govulncheck.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Run govulncheck and fail on any reachable vulnerability that is not covered by +# an OpenVEX "not_affected" statement (SEC-9, SEC-12). +# +# govulncheck has no suppression mechanism of its own, and a blanket "|| true" +# would turn the gate into decoration. Instead every exemption must be a +# statement in .vex/flowctl.openvex.json carrying a justification and an impact +# statement, so the reason is reviewable and shows up in a diff. +# +# Exits 0 if the only findings are VEXed, 1 otherwise. + +set -euo pipefail + +VEX_FILE="${VEX_FILE:-.vex/flowctl.openvex.json}" +GOVULNCHECK_VERSION="${GOVULNCHECK_VERSION:-latest}" +TARGET="${1:-./...}" + +if ! command -v jq >/dev/null 2>&1; then + echo "error: jq is required" >&2 + exit 1 +fi + +echo "Running govulncheck on ${TARGET}..." +raw=$(go run "golang.org/x/vuln/cmd/govulncheck@${GOVULNCHECK_VERSION}" \ + -format json "${TARGET}" 2>/dev/null) || true + +if [ -z "${raw}" ]; then + echo "error: govulncheck produced no output" >&2 + exit 1 +fi + +# A finding at the "symbol" level is one govulncheck believes is reachable. +# Anything less specific is informational and does not gate the merge. +found=$(printf '%s\n' "${raw}" \ + | jq -r 'select(.finding != null) + | select(.finding.trace[0].function != null) + | .finding.osv' \ + | sort -u) + +if [ -z "${found}" ]; then + echo "No reachable vulnerabilities." + exit 0 +fi + +if [ -f "${VEX_FILE}" ]; then + exempt=$(jq -r '.statements[] + | select(.status == "not_affected" or .status == "fixed") + | .vulnerability.name' "${VEX_FILE}" | sort -u) +else + exempt="" +fi + +unexempt=$(comm -23 <(printf '%s\n' "${found}") <(printf '%s\n' "${exempt}")) + +if [ -n "${exempt}" ]; then + echo + echo "Covered by ${VEX_FILE}:" + comm -12 <(printf '%s\n' "${found}") <(printf '%s\n' "${exempt}") | sed 's/^/ - /' +fi + +if [ -n "${unexempt}" ]; then + echo + echo "Reachable vulnerabilities with no VEX statement:" >&2 + printf '%s\n' "${unexempt}" | sed 's/^/ - /' >&2 + echo >&2 + echo "Upgrade the dependency, or — only if flowctl is genuinely not affected —" >&2 + echo "add a justified statement to ${VEX_FILE}." >&2 + echo >&2 + echo "Full report:" >&2 + go run "golang.org/x/vuln/cmd/govulncheck@${GOVULNCHECK_VERSION}" "${TARGET}" >&2 || true + exit 1 +fi + +echo +echo "All reachable vulnerabilities are covered by ${VEX_FILE}." diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh index 6e3d028..d9dd4ba 100755 --- a/scripts/quickstart.sh +++ b/scripts/quickstart.sh @@ -91,7 +91,7 @@ echo "==> Generating starter pipeline: ${OUTPUT}" ./bin/flowctl init --non-interactive \ --network "${NETWORK}" \ --destination "${DESTINATION}" \ - --output "${OUTPUT}" + --file "${OUTPUT}" echo "==> Validating ${OUTPUT}" ./bin/flowctl validate "${OUTPUT}" diff --git a/scripts/smoke-onboarding.sh b/scripts/smoke-onboarding.sh index bc85fe8..a7de0b4 100755 --- a/scripts/smoke-onboarding.sh +++ b/scripts/smoke-onboarding.sh @@ -208,7 +208,7 @@ if ./bin/flowctl validate schemas/cue/invalid-pipeline.yaml >/dev/null 2>&1; the fi echo "==> Generate starter pipeline" -./bin/flowctl init --preset testnet-duckdb -o "${TMP_DIR}/stellar-pipeline.yaml" >/dev/null +./bin/flowctl init --preset testnet-duckdb --file "${TMP_DIR}/stellar-pipeline.yaml" >/dev/null perl -0pi -e "s#\./stellar-pipeline\.duckdb#${TMP_DIR}/stellar-pipeline.duckdb#g" "${TMP_DIR}/stellar-pipeline.yaml" echo "==> DuckDB starter pipeline smoke" @@ -217,7 +217,7 @@ DUCKDB_PID=$! wait_for_control_plane 9090 "$DUCKDB_PID" "${TMP_DIR}/duckdb.log" 180 ./bin/flowctl processors list --endpoint 127.0.0.1:9090 >/dev/null ./bin/flowctl processors find --input stellar.ledger.v1 --endpoint 127.0.0.1:9090 >/dev/null -./bin/flowctl processors find --output stellar.contract.events.v1 --endpoint 127.0.0.1:9090 >/dev/null +./bin/flowctl processors find --output-type stellar.contract.events.v1 --endpoint 127.0.0.1:9090 >/dev/null wait_for_log_pattern "${TMP_DIR}/duckdb.log" "Stored [0-9][0-9]* contract events" "$DUCKDB_PID" 180 kill "${DUCKDB_PID}" >/dev/null 2>&1 || true wait "${DUCKDB_PID}" || true @@ -247,7 +247,7 @@ if command -v docker >/dev/null 2>&1; then done docker exec "${POSTGRES_CONTAINER}" createdb -U postgres stellar_events >/dev/null 2>&1 || true - ./bin/flowctl init --preset testnet-postgres -o "${TMP_DIR}/postgres-pipeline.yaml" >/dev/null + ./bin/flowctl init --preset testnet-postgres --file "${TMP_DIR}/postgres-pipeline.yaml" >/dev/null nohup ./bin/flowctl run --control-plane-port 9091 --no-persistence "${TMP_DIR}/postgres-pipeline.yaml" >"${TMP_DIR}/postgres.log" 2>&1 & POSTGRES_PID=$! wait_for_postgres_events "${POSTGRES_CONTAINER}" "$POSTGRES_PID" "${TMP_DIR}/postgres.log" 240 diff --git a/staticcheck.conf b/staticcheck.conf new file mode 100644 index 0000000..46bea3a --- /dev/null +++ b/staticcheck.conf @@ -0,0 +1,41 @@ +# staticcheck configuration. +# +# The §9 merge gate of the OBSRVR Engineering Standard requires `staticcheck` or +# `golangci-lint` to pass before a merge. staticcheck is the one wired into CI and +# into `make lint`. +# +# Everything is enabled except the exclusions below. Each exclusion is a backlog +# item, not a permanent opinion: they exist so the gate can be switched on now and +# catch every *new* problem, rather than waiting until the existing ones are cleared. + +checks = [ + "all", + + # -- Enabled-by-default style checks we deliberately do not want -------------- + + # ST1000: every package needs a package comment. Worth doing, but it would + # mean touching ~20 packages in the same change that turns the gate on. + # Tracked as GAP-24. + "-ST1000", + + # ST1003: naming conventions. The generated protobuf code and the existing + # config structs use names this flags, and renaming an exported config field + # is a contract change (VER-3), not a lint fix. + "-ST1003", + + # -- Backlog: real findings in code this branch does not touch --------------- + + # SA1019: use of deprecated identifiers. All current hits are + # grpc.DialContext/grpc.WithBlock, which NewClient replaces. The migration is + # not mechanical — NewClient never blocks, so the call sites that rely on + # WithBlock change behaviour — and it needs its own change with its own tests. + # Tracked as GAP-25. + "-SA1019", +] + +# Only report problems in code we own. Generated protobuf is checked by buf. +initialisms = ["API", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", + "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", + "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", + "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS", "SIP", "RTP", "AMQP", + "DB", "TS", "CUE", "DAG", "XDR", "CAP", "SEP", "NDJSON", "OSC"] diff --git a/test/delivery_process_test.go b/test/delivery_process_test.go index 4be7e54..f36b204 100644 --- a/test/delivery_process_test.go +++ b/test/delivery_process_test.go @@ -3,15 +3,18 @@ package test import ( "bytes" "context" + "errors" "fmt" - "github.com/withobsrvr/flowctl/internal/storage" - flowctlpb "github.com/withobsrvr/flowctl/proto" "os" "os/exec" "path/filepath" "strings" "testing" "time" + + "github.com/withobsrvr/flowctl/internal/exitcode" + "github.com/withobsrvr/flowctl/internal/storage" + flowctlpb "github.com/withobsrvr/flowctl/proto" ) // Exercises the CLI, process driver, control-plane registration, and gRPC @@ -128,10 +131,21 @@ spec: if ctx.Err() != nil { t.Fatalf("run timed out: %s", out) } - if tc.failed && err == nil { - t.Fatalf("expected nonzero CLI exit: %s", out) - } - if !tc.failed && err != nil { + signalled := tc.mode == "wait" || tc.mode == "drain-stop" + switch { + case tc.failed: + if err == nil { + t.Fatalf("expected nonzero CLI exit: %s", out) + } + case signalled: + // CLI-11/CLI-15: a clean shutdown on SIGINT exits 130, so a + // supervisor can tell "interrupted, resume from checkpoint" + // apart from "finished". + if got := exitCodeOf(err); got != exitcode.Interrupted { + t.Fatalf("expected exit %d after SIGINT, got %d (%v)\n%s", + exitcode.Interrupted, got, err, out) + } + case err != nil: t.Fatalf("expected successful completion: %v\n%s", err, out) } if !strings.Contains(string(out), tc.want) { @@ -191,3 +205,15 @@ spec: }) } } + +// exitCodeOf reports the process exit code behind an *exec.ExitError, or -1 if +// err is not one. A nil error means the process exited 0. +func exitCodeOf(err error) int { + if err == nil { + return 0 + } + if ee, ok := errors.AsType[*exec.ExitError](err); ok { + return ee.ExitCode() + } + return -1 +} diff --git a/test/operator_flow_test.go b/test/operator_flow_test.go index 26086f6..6d7936a 100644 --- a/test/operator_flow_test.go +++ b/test/operator_flow_test.go @@ -2,6 +2,7 @@ package test import ( "bytes" + "encoding/json" "fmt" "net" "os" @@ -137,25 +138,57 @@ spec: t.Fatalf("expected shortened run id from pipelines active, got %q", shortRunID) } + // Assert against --output json, not the table. The human layout is + // explicitly not a contract (ADR 0001); the schema is. + runInfo := runCommandJSON(t, repoRoot, 5*time.Second, flowctlBin, + "pipelines", + "run-info", shortRunID, + "--control-plane-address", "127.0.0.1", + "--control-plane-port", fmt.Sprintf("%d", controlPlanePort), + "--output", "json", + ) + if got := runInfo["schema"]; got != "flowctl.pipeline_run.v1" { + t.Errorf("run-info schema = %v", got) + } + data := runInfo["data"].(map[string]any) + if got := data["pipeline_name"]; got != "operator-smoke" { + t.Errorf("pipeline_name = %v", got) + } + if got := data["status"]; got != "running" { + t.Errorf("status = %v", got) + } + if !strings.HasPrefix(data["run_id"].(string), shortRunID) { + t.Errorf("run_id %v does not start with the short id %q", data["run_id"], shortRunID) + } + components, _ := data["component_ids"].([]any) + joined := fmt.Sprint(components...) + if !strings.Contains(joined, "test-source") || !strings.Contains(joined, "test-sink") { + t.Errorf("component_ids = %v", components) + } + + // The human path must still render; its exact spacing is not asserted. runInfoOut := runCommand(t, repoRoot, 5*time.Second, flowctlBin, "pipelines", "run-info", shortRunID, "--control-plane-address", "127.0.0.1", "--control-plane-port", fmt.Sprintf("%d", controlPlanePort), ) - assertContains(t, runInfoOut, "Run ID:") - assertContains(t, runInfoOut, "Pipeline: operator-smoke") - assertContains(t, runInfoOut, "Status: running") - assertContains(t, runInfoOut, "test-source") - assertContains(t, runInfoOut, "test-sink") + assertContains(t, runInfoOut, "operator-smoke") + assertContains(t, runInfoOut, "running") - stopOut := runCommand(t, repoRoot, 5*time.Second, flowctlBin, + stopped := runCommandJSON(t, repoRoot, 5*time.Second, flowctlBin, "pipelines", "stop", shortRunID, "--control-plane-address", "127.0.0.1", "--control-plane-port", fmt.Sprintf("%d", controlPlanePort), + "--output", "json", ) - assertContains(t, stopOut, "Pipeline stopped") + if got := stopped["schema"]; got != "flowctl.pipeline_run.v1" { + t.Errorf("stop schema = %v", got) + } + if got := stopped["data"].(map[string]any)["status"]; got == "running" { + t.Errorf("run still reports %v after stop", got) + } select { case err := <-runDone: @@ -401,3 +434,43 @@ func main() { } } ` + +// runCommandJSON runs a command capturing stdout and stderr separately, and +// returns the parsed stdout document. +// +// Keeping the streams apart is the point: it asserts CLI-5 as a side effect. +// If any progress line leaked onto stdout, the parse here would fail. +func runCommandJSON(t *testing.T, repoRoot string, timeout time.Duration, bin string, args ...string) map[string]any { + t.Helper() + + cmd := exec.Command(bin, args...) + cmd.Dir = repoRoot + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Start(); err != nil { + t.Fatalf("failed to start %s: %v", bin, err) + } + + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("command failed: %s %s\nError: %v\nStderr:\n%s", bin, strings.Join(args, " "), err, stderr.String()) + } + case <-time.After(timeout): + _ = cmd.Process.Kill() + t.Fatalf("command timed out: %s %s", bin, strings.Join(args, " ")) + } + + var doc map[string]any + if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &doc); err != nil { + t.Fatalf("stdout is not one JSON document: %v\nStdout:\n%s\nStderr:\n%s", + err, stdout.String(), stderr.String()) + } + return doc +} diff --git a/test/test_client.go b/test/test_client.go index 55093c2..4659e35 100644 --- a/test/test_client.go +++ b/test/test_client.go @@ -31,10 +31,10 @@ func main() { heartbeatInterval := flag.Duration("heartbeat", defaultHeartbeatEvery, "Heartbeat interval") statusInterval := flag.Duration("status", defaultStatusEvery, "Status check interval") flag.Parse() - - log.Printf("Starting test client (heartbeat: %s, status checks: %s)...", + + log.Printf("Starting test client (heartbeat: %s, status checks: %s)...", *heartbeatInterval, *statusInterval) - + // Connect to flowctl control plane conn, err := grpc.Dial(*serverAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { @@ -70,37 +70,37 @@ func main() { } log.Printf("Registration successful - Service ID: %s", ack.ServiceId) - + // Store the service ID for future heartbeats serviceID := ack.ServiceId // Start heartbeat loop in background heartbeatTicker := time.NewTicker(*heartbeatInterval) defer heartbeatTicker.Stop() - + // Create a signal channel for clean shutdown sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) - + log.Printf("Starting heartbeat loop (every %s) - press Ctrl+C to exit", *heartbeatInterval) - + // Periodically check service status too statusTicker := time.NewTicker(*statusInterval) defer statusTicker.Stop() - + for { select { case <-heartbeatTicker.C: // Send heartbeat heartbeat := &pb.ServiceHeartbeat{ - ServiceId: serviceID, - Timestamp: timestamppb.Now(), + ServiceId: serviceID, + Timestamp: timestamppb.Now(), Metrics: map[string]float64{ "chunks_per_second": 12.5, "last_ledger": 45678923, }, } - + hbCtx, hbCancel := context.WithTimeout(context.Background(), 2*time.Second) _, hbErr := client.Heartbeat(hbCtx, heartbeat) if hbErr != nil { @@ -109,7 +109,7 @@ func main() { log.Printf("Sent heartbeat for service %s", serviceID) } hbCancel() - + case <-statusTicker.C: // Check our status to verify health state stCtx, stCancel := context.WithTimeout(context.Background(), 2*time.Second) @@ -117,16 +117,16 @@ func main() { if stErr != nil { log.Printf("Failed to get service status: %v", stErr) } else { - log.Printf("Service status: ID=%s, Healthy=%v, Last Heartbeat=%v", - status.ServiceId, + log.Printf("Service status: ID=%s, Healthy=%v, Last Heartbeat=%v", + status.ServiceId, status.IsHealthy, status.LastHeartbeat.AsTime()) } stCancel() - + case sig := <-sigChan: log.Printf("Received signal %v, shutting down", sig) return } } -} \ No newline at end of file +} diff --git a/test/tools.go b/test/tools.go new file mode 100644 index 0000000..a0cf907 --- /dev/null +++ b/test/tools.go @@ -0,0 +1,17 @@ +//go:build tools + +// Package tools pins dependencies that are required but invisible to `go mod tidy`. +// +// test/testdata/delivery_component.go is compiled at run time by the delivery tests +// (see delivery_process_test.go and direct_sink_routing_test.go), not by the Go build. +// The toolchain ignores testdata directories entirely, so tidy cannot see that import +// and would otherwise drop the module — breaking those tests with a confusing +// "no required module provides package" failure at test time rather than build time. +// +// Build constraints do not hide imports from `go mod tidy`, which is what makes this +// file work while keeping the import out of every real build. +package tools + +import ( + _ "github.com/withObsrvr/flowctl-sdk/pkg/consumer" +)