feat(backend): add relay distributed locking, harden and instrument Horizon retry manager, add French dashboard locale

backend/docs/WEBSOCKET_RELAY_CONCURRENCY.md

@@ -0,0 +1,73 @@
+# WebSocket Relay: distributed concurrency control
+
+Issue #1455. Module: `src/lib/websocket-relay-lock.js`.
+
+## What it protects
+
+| Scope (`RelayLockScope`) | Resource | Prevents |
+|--------------------------|----------|----------|
+| `event` | relay event / payment id | Two instances delivering the same event |
+| `dlq-drain` | queue name | Concurrent DLQ replays re-sending every entry once per instance |
+| `reconnect` | upstream URL | Reconnect stampedes across instances |
+| `connection` | connection id | Concurrent mutation of one connection's state |
+
+Inside one process, `createSingleFlight()` makes concurrent callers share a
+single in-flight promise per key. Use it for reconnect loops so that many
+sockets seeing the same upstream drop start only one reconnect.
+
+`CircuitBreaker` in `websocket-relay-recovery.js` now admits exactly **one**
+probe while `HALF_OPEN`. Concurrent callers are rejected with
+`err.circuitOpen = true` and counted in
+`relay_circuit_breaker_probe_rejected_total`. Before this change every
+concurrent caller was let through to the recovering upstream.
+
+## Usage
+
+```js
+import { withRelayLock, RelayLockScope } from "./websocket-relay-lock.js";
+
+await withRelayLock(RelayLockScope.DLQ_DRAIN, "default", async ({ signal }) => {
+  while (!signal.aborted && dlq.size > 0) {
+    await deliver(dlq.shift());
+  }
+}, { ttlMs: 15_000, waitMs: 0 });
+```
+
+- `withRelayLock` throws a `409` error with `code: "RELAY_LOCK_BUSY"` (`retryable: true`) when another holder owns the lock after `waitMs` (default `0`, max 30 s, polled with ±25% jitter).
+- The lease is renewed every `ttl/3`. If a renewal fails, `signal` is aborted with `code: "RELAY_LOCK_LOST"`. Long-running work must check `signal.aborted` between units of work.
+- The lock is always released in `finally`, including when the work throws.
+
+## Safety properties
+
+- **Atomic acquire:** `SET key token NX PX ttl`.
+- **Owner-only release and renew:** compare-and-delete and compare-and-pexpire Lua scripts check the random per-lease token. A worker whose lease expired can neither delete nor revive a lock that someone else now holds.
+- **Key hygiene:** the scope must be in an allow-list. The resource is SHA-256 hashed (`lock:ws-relay:<scope>:<sha256>`), so it cannot inject key separators or CRLF, and its length is capped at 512 characters.
+- **TTL bounds:** `WS_RELAY_LOCK_TTL_MS` (default 15000) is clamped to 1 s–120 s.
+- **Degraded mode:** if Redis is down, or is the no-op client from `redis.js`, locks fall back to an in-process table. A single instance stays correct, and a warning is logged. Cross-instance exclusion returns when Redis does.
+
+## Metrics
+
+All on the relay registry (`relayMetricsRegister`):
+
+| Metric | Labels |
+|--------|--------|
+| `relay_lock_acquisitions_total` | `scope`, `result` = `acquired` / `contended` / `error` |
+| `relay_lock_wait_duration_seconds` | `scope` |
+| `relay_lock_hold_duration_seconds` | `scope` |
+| `relay_lock_lost_total` | `scope` |
+| `relay_circuit_breaker_probe_rejected_total` | — |
+
+Alerting rules: `docs/alerts/websocket-relay-locking.rules.yml`.
+
+## Configuration
+
+| Env var | Default | Notes |
+|---------|---------|-------|
+| `WS_RELAY_LOCK_TTL_MS` | `15000` | Clamped to 1000–120000 |
+| `REDIS_URL` | — | Shared with the rest of the API |
+
+## Tests
+
+```bash
+npx vitest run src/lib/websocket-relay-lock.test.js src/lib/websocket-relay-recovery.test.js
+```

backend/docs/alerts/horizon-client-retry-manager.rules.yml

@@ -0,0 +1,82 @@
+# Prometheus alerting rules for the Horizon Client Retry Manager (issue #1438).
+#
+# Load with:   rule_files: ["docs/alerts/horizon-client-retry-manager.rules.yml"]
+# Validate:    promtool check rules docs/alerts/horizon-client-retry-manager.rules.yml
+#
+# Metric reference: docs/HORIZON_CLIENT_RETRY_MANAGER.md
+
+groups:
+  - name: horizon-client-retry-manager
+    rules:
+      - alert: HorizonClientUnhealthy
+        expr: max(horizon_client_health_state) >= 2
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: Horizon client reports unhealthy
+          description: >
+            At least one Horizon operation has failed after exhausting its
+            retries on consecutive calls. Payments cannot be verified against
+            the Stellar network. Check horizon_client_consecutive_failures by
+            operation and HorizonClient.getHealth().
+
+      - alert: HorizonClientDegraded
+        expr: max(horizon_client_health_state) == 1
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: Horizon client has been degraded for 10 minutes
+          description: >
+            Some Horizon operations keep failing after retries but are below
+            the unhealthy threshold.
+
+      - alert: HorizonClientRetriesExhausted
+        expr: sum by (operation, error_type) (increase(horizon_client_retry_exhausted_total[5m])) > 5
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Horizon {{ $labels.operation }} is exhausting retries ({{ $labels.error_type }})"
+          description: >
+            {{ $value | humanize }} {{ $labels.operation }} calls failed after
+            every retry in the last 5m with {{ $labels.error_type }}.
+
+      - alert: HorizonClientHighErrorRatio
+        expr: |
+          sum(rate(horizon_client_operations_total{result="error"}[10m]))
+            /
+          clamp_min(sum(rate(horizon_client_operations_total[10m])), 1e-9)
+            > 0.25
+          and sum(increase(horizon_client_operations_total[10m])) >= 20
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: More than 25% of Horizon client operations are failing
+          description: >
+            Error ratio over the last 10m is {{ $value | humanizePercentage }}.
+
+      - alert: HorizonClientNoRecentSuccess
+        expr: (time() - max by (operation) (horizon_client_last_success_timestamp_seconds)) > 900
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "No successful Horizon {{ $labels.operation }} for 15 minutes"
+          description: >
+            The operation has succeeded before but has not succeeded for over
+            15 minutes. It may be failing or no longer being called.
+
+      - alert: HorizonClientValidationRejections
+        expr: sum(increase(horizon_client_data_validation_errors_total[10m])) > 0
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: Horizon client rejected invalid retry requests
+          description: >
+            Callers passed an invalid operation name or callback to the retry
+            manager. This is a code defect; check logs for
+            "Horizon client: rejected invalid retry request".

backend/docs/alerts/websocket-relay-locking.rules.yml

@@ -0,0 +1,47 @@
+# Prometheus alerting rules for WebSocket Relay distributed locking (issue #1455).
+#
+# Load with:   rule_files: ["docs/alerts/websocket-relay-locking.rules.yml"]
+# Validate:    promtool check rules docs/alerts/websocket-relay-locking.rules.yml
+#
+# Metric reference: docs/WEBSOCKET_RELAY_CONCURRENCY.md
+
+groups:
+  - name: websocket-relay-locking
+    rules:
+      - alert: WebSocketRelayLockLost
+        expr: sum by (scope) (increase(relay_lock_lost_total[10m])) > 0
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: "WebSocket relay lost a {{ $labels.scope }} lock mid-work"
+          description: >
+            A lease could not be renewed while work was running, so the work was
+            aborted. Usually Redis latency, a Redis failover or an event loop
+            stall longer than the lock TTL.
+
+      - alert: WebSocketRelayLockBackendErrors
+        expr: sum(increase(relay_lock_acquisitions_total{result="error"}[5m])) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: WebSocket relay is falling back to in-process locks
+          description: >
+            Redis SET NX is failing, so locks are only enforced per instance.
+            Cross-instance duplicate delivery is possible until Redis recovers.
+
+      - alert: WebSocketRelayHighLockContention
+        expr: |
+          sum by (scope) (rate(relay_lock_acquisitions_total{result="contended"}[10m]))
+            /
+          clamp_min(sum by (scope) (rate(relay_lock_acquisitions_total[10m])), 1e-9)
+            > 0.5
+        for: 15m
+        labels:
+          severity: info
+        annotations:
+          summary: "Over half of {{ $labels.scope }} lock attempts are contended"
+          description: >
+            Workers are mostly competing for the same resources. Consider
+            partitioning work or lowering the number of relay consumers.

backend/src/lib/metrics.js

@@ -890,6 +890,30 @@ export const horizonClientOperationSuccessRate = new client.Gauge({
   labelNames: ["operation"],
 });
 
+// Retry Manager alert metrics and health telemetry (Issue #1438)
+export const horizonClientRetryExhausted = new client.Counter({
+  name: "horizon_client_retry_exhausted_total",
+  help: "Horizon client operations that failed after exhausting every retry",
+  labelNames: ["operation", "error_type"],
+});
+
+export const horizonClientConsecutiveFailures = new client.Gauge({
+  name: "horizon_client_consecutive_failures",
+  help: "Consecutive infrastructure failures per Horizon client operation (reset on success)",
+  labelNames: ["operation"],
+});
+
+export const horizonClientLastSuccessTimestamp = new client.Gauge({
+  name: "horizon_client_last_success_timestamp_seconds",
+  help: "Unix timestamp of the last successful Horizon client operation",
+  labelNames: ["operation"],
+});
+
+export const horizonClientHealthState = new client.Gauge({
+  name: "horizon_client_health_state",
+  help: "Horizon client retry manager health (0=healthy, 1=degraded, 2=unhealthy)",
+});
+
 export const paymentMatchingOperations = new client.Counter({
   name: "payment_matching_operations_total",
   help: "Total number of payment matching operations",
@@ -1111,6 +1135,10 @@ register.registerMetric(horizonClientSerializationErrors);
 register.registerMetric(horizonClientRateLimitWaitTime);
 register.registerMetric(horizonClientThroughput);
 register.registerMetric(horizonClientOperationSuccessRate);
+register.registerMetric(horizonClientRetryExhausted);
+register.registerMetric(horizonClientConsecutiveFailures);
+register.registerMetric(horizonClientLastSuccessTimestamp);
+register.registerMetric(horizonClientHealthState);
 register.registerMetric(paymentMatchingOperations);
 register.registerMetric(paymentMatchingErrors);
 register.registerMetric(signatureVerificationOperations);