From a1767472b86d5af8a5615095751ee325e486aaba Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Mon, 19 Oct 2026 17:05:02 +0200 Subject: [PATCH 01/16] feat(query-engine): contract for reading Google Cloud metrics live --- packages/domain/package.json | 2 + packages/domain/src/gcp-live-metrics.ts | 39 ++++++++++++ .../domain/src/http/gcp-live-query-errors.ts | 63 +++++++++++++++++++ packages/domain/src/http/index.ts | 1 + .../query-engine/src/runtime/live-metrics.ts | 40 ++++++++++++ 5 files changed, 145 insertions(+) create mode 100644 packages/domain/src/gcp-live-metrics.ts create mode 100644 packages/domain/src/http/gcp-live-query-errors.ts create mode 100644 packages/query-engine/src/runtime/live-metrics.ts diff --git a/packages/domain/package.json b/packages/domain/package.json index cd6573f579..2d39ebf59f 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -14,10 +14,12 @@ "./format": "./src/format.ts", "./gcp": "./src/gcp.ts", "./gcp-infra": "./src/gcp-infra.ts", + "./gcp-live-metrics": "./src/gcp-live-metrics.ts", "./gcp-metrics": "./src/gcp-metrics.ts", "./gen-ai": "./src/gen-ai.ts", "./glob": "./src/glob.ts", "./http": "./src/http/index.ts", + "./http/gcp-live-query-errors": "./src/http/gcp-live-query-errors.ts", "./incident-context": "./src/incident-context.ts", "./http/v2": "./src/http/v2/index.ts", "./http/v2-worker-unavailable": "./src/http/v2/worker-unavailable.ts", diff --git a/packages/domain/src/gcp-live-metrics.ts b/packages/domain/src/gcp-live-metrics.ts new file mode 100644 index 0000000000..d3e9b154a8 --- /dev/null +++ b/packages/domain/src/gcp-live-metrics.ts @@ -0,0 +1,39 @@ +import type { MetricType } from "./query-engine" + +/** + * A Cloud Monitoring metric type: a service domain, a slash, then a path, e.g. + * `compute.googleapis.com/instance/cpu/utilization` or `kubernetes.io/container/cpu/core_usage_time`. + * The query engine reads these live from Google instead of from the warehouse. + */ +const GCP_METRIC_TYPE = /^[a-z0-9-]+(?:\.[a-z0-9-]+)+\/[A-Za-z0-9_.\-/]+$/ + +export const isGcpMetricType = (metricName: string): boolean => GCP_METRIC_TYPE.test(metricName) + +export type GcpMetricKind = "GAUGE" | "DELTA" | "CUMULATIVE" +export type GcpMetricValueType = "BOOL" | "INT64" | "DOUBLE" | "STRING" | "DISTRIBUTION" | "MONEY" + +/** A metric with data in the last ~25 hours (Cloud Monitoring `metricDescriptors?activeOnly=true`). */ +export interface GcpMetricDescriptor { + readonly type: string + readonly metricKind: GcpMetricKind + readonly valueType: GcpMetricValueType + readonly unit: string + readonly description: string + /** Resource types with recent data. Empty when Google does not list them (e.g. logging metrics). */ + readonly monitoredResourceTypes: ReadonlyArray + /** Metric label keys. Resource labels (project_id, region, ...) are not listed here. */ + readonly labels: ReadonlyArray +} + +/** How Maple's query builder treats a Cloud Monitoring metric; undefined for value types PromQL cannot chart. */ +export const gcpMetricMapleType = (descriptor: GcpMetricDescriptor): MetricType | undefined => { + if (descriptor.valueType === "STRING" || descriptor.valueType === "MONEY") return undefined + if (descriptor.valueType === "DISTRIBUTION") return "histogram" + return descriptor.metricKind === "GAUGE" ? "gauge" : "sum" +} + +/** One series of a PromQL answer. Points are `[epochSeconds, value]`, NaN and infinities dropped. */ +export interface GcpPromQlSeries { + readonly labels: Readonly> + readonly points: ReadonlyArray +} diff --git a/packages/domain/src/http/gcp-live-query-errors.ts b/packages/domain/src/http/gcp-live-query-errors.ts new file mode 100644 index 0000000000..1a71d3c370 --- /dev/null +++ b/packages/domain/src/http/gcp-live-query-errors.ts @@ -0,0 +1,63 @@ +import { Schema } from "effect" +import { HttpTaggedError } from "./error-policy" + +// Failures of a metrics query the engine answered live from Google Cloud Monitoring. +// Error-only module (no effect/http-api), so the query engine can import it. + +/** The query cannot run as asked: Google refused it, or it reaches too many projects or series. */ +export class GcpLiveQueryRejectedError extends HttpTaggedError()( + "@maple/http/errors/GcpLiveQueryRejectedError", + { + message: Schema.String, + reason: Schema.Literals(["invalid_query", "too_many_projects", "too_many_series"]), + }, + { + status: 400, + code: "gcp_live_query_rejected", + title: "Google Cloud could not run this query", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +/** No Google Cloud connection with metrics on, or this deployment has no Google identity. */ +export class GcpLiveQueryUnavailableError extends HttpTaggedError()( + "@maple/http/errors/GcpLiveQueryUnavailableError", + { + message: Schema.String, + }, + { + status: 409, + code: "gcp_live_query_unavailable", + title: "Google Cloud metrics are not connected", + retry: "never", + recovery: "none", + exposure: "public_message", + }, +) {} + +/** Google failed or refused the read on every project the query reaches. */ +export class GcpLiveQueryFailedError extends HttpTaggedError()( + "@maple/http/errors/GcpLiveQueryFailedError", + { + message: Schema.String, + failures: Schema.Array(Schema.Struct({ scope: Schema.String, message: Schema.String })), + }, + { + status: 502, + code: "gcp_live_query_failed", + title: "Google Cloud did not answer", + retry: "backoff", + recovery: "retry", + exposure: "public_message", + }, +) {} + +export const gcpLiveQueryHttpErrors = [ + GcpLiveQueryRejectedError, + GcpLiveQueryUnavailableError, + GcpLiveQueryFailedError, +] as const + +export type GcpLiveQueryError = GcpLiveQueryRejectedError | GcpLiveQueryUnavailableError | GcpLiveQueryFailedError diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index b39c5ba560..753eba710f 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -57,3 +57,4 @@ export * from "./vcs" export * from "./api-boundary" export * from "./warehouse" export * from "./widget-types" +export * from "./gcp-live-query-errors" diff --git a/packages/query-engine/src/runtime/live-metrics.ts b/packages/query-engine/src/runtime/live-metrics.ts new file mode 100644 index 0000000000..537f59f7ec --- /dev/null +++ b/packages/query-engine/src/runtime/live-metrics.ts @@ -0,0 +1,40 @@ +import type { Effect } from "effect" +import type { GcpMetricDescriptor, GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import type { GcpLiveQueryError } from "@maple/domain/http/gcp-live-query-errors" +import type { QueryTenant } from "./query-engine" + +/** + * Google Cloud Monitoring, read live through its Prometheus-compatible API. The engine compiles a + * metrics query on a Cloud Monitoring metric type (`isGcpMetricType`) to PromQL and runs it here, + * across every project the org's metrics-on connections reach. Absent in deployments without a + * Google identity (local mode, the CLI); a Google metric then fails as unavailable. + */ +export interface GcpLiveMetrics { + /** Range query, times in epoch seconds. `projects` narrows the fan-out (from a `project_id` filter). */ + readonly queryRange: ( + tenant: T, + request: { + readonly query: string + readonly startSeconds: number + readonly endSeconds: number + readonly stepSeconds: number + readonly projects?: ReadonlyArray + }, + ) => Effect.Effect, GcpLiveQueryError> + /** Instant query at `timeSeconds`; each series carries one point. */ + readonly queryInstant: ( + tenant: T, + request: { + readonly query: string + readonly timeSeconds: number + readonly projects?: ReadonlyArray + }, + ) => Effect.Effect, GcpLiveQueryError> + /** Active metrics across the org's projects (cached). */ + readonly listMetrics: (tenant: T) => Effect.Effect, GcpLiveQueryError> + /** One active metric, its resource types merged across projects; undefined when no project reports it. */ + readonly describeMetric: ( + tenant: T, + metricType: string, + ) => Effect.Effect +} From 917263074391b26ab019f02e68ed7ad2c99b55a5 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Mon, 19 Oct 2026 17:27:41 +0200 Subject: [PATCH 02/16] feat(query-engine): answer Cloud Monitoring metric queries live from Google with PromQL --- apps/ai/src/worker.ts | 2 + apps/api/src/resources/env.ts | 2 + .../GcpLiveMetricsService.test.ts | 914 ++++++++++++++++++ .../integrations/GcpLiveMetricsService.ts | 515 ++++++++++ .../integrations/GcpMetricsService.test.ts | 5 +- .../integrations/GcpMetricsService.ts | 52 +- .../src/services/integrations/gcp/api.ts | 351 ++++++- .../src/services/integrations/gcp/explain.ts | 65 ++ .../services/warehouse/GcpLiveMetricsPort.ts | 10 + .../services/warehouse/QueryEngineService.ts | 18 +- .../anticipated-error-identifiers.ts | 2 + packages/domain/src/http/warehouse-errors.ts | 7 +- .../query-engine/src/promql/compile.test.ts | 340 +++++++ packages/query-engine/src/promql/compile.ts | 273 ++++++ .../src/runtime/gcp-metrics.test.ts | 407 ++++++++ .../query-engine/src/runtime/gcp-metrics.ts | 279 ++++++ packages/query-engine/src/runtime/index.ts | 2 + .../query-engine/src/runtime/query-engine.ts | 145 ++- 18 files changed, 3291 insertions(+), 98 deletions(-) create mode 100644 packages/backend/src/services/integrations/GcpLiveMetricsService.test.ts create mode 100644 packages/backend/src/services/integrations/GcpLiveMetricsService.ts create mode 100644 packages/backend/src/services/integrations/gcp/explain.ts create mode 100644 packages/backend/src/services/warehouse/GcpLiveMetricsPort.ts create mode 100644 packages/query-engine/src/promql/compile.test.ts create mode 100644 packages/query-engine/src/promql/compile.ts create mode 100644 packages/query-engine/src/runtime/gcp-metrics.test.ts create mode 100644 packages/query-engine/src/runtime/gcp-metrics.ts diff --git a/apps/ai/src/worker.ts b/apps/ai/src/worker.ts index 9f3b00b65c..f0ca909e9e 100644 --- a/apps/ai/src/worker.ts +++ b/apps/ai/src/worker.ts @@ -62,6 +62,8 @@ const configuredEnv = ({ stage, region, domains, profile }: MapleStackContext) = // The source and sandbox tools resolve repositories through the GitHub App, so // they need its reader credentials here too, not only on api. githubAppSourceEnv, + // Tools that query Google Cloud metrics read them live; unset leaves them unavailable. + optionalSecret("MAPLE_GCP_SERVICE_ACCOUNT_KEY"), // Required wherever a sandbox Worker deploys: without it every sandbox call is refused. ...(profile.deploys.sandbox ? [requireSecretEntry("SANDBOX_INTERNAL_SERVICE_TOKEN")] : []), // Dev-only escape hatch from per-org BYO rows (see apps/api/src/resources/env.ts). diff --git a/apps/api/src/resources/env.ts b/apps/api/src/resources/env.ts index 0fff70d331..6cb1f45604 100644 --- a/apps/api/src/resources/env.ts +++ b/apps/api/src/resources/env.ts @@ -82,4 +82,6 @@ export const apiConfiguredEnv = (stage: MapleStage, region: MapleRegion, domains cloudflareOAuthEnv, planetScaleOAuthEnv, optionalPlain("MAPLE_GCP_SERVICE_ACCOUNT_EMAIL"), + // Sign-in for live Google Cloud metrics queries; unset leaves them unavailable. + optionalSecret("MAPLE_GCP_SERVICE_ACCOUNT_KEY"), ) diff --git a/packages/backend/src/services/integrations/GcpLiveMetricsService.test.ts b/packages/backend/src/services/integrations/GcpLiveMetricsService.test.ts new file mode 100644 index 0000000000..0ddd2e3b97 --- /dev/null +++ b/packages/backend/src/services/integrations/GcpLiveMetricsService.test.ts @@ -0,0 +1,914 @@ +import { afterEach, assert, beforeAll, describe, it } from "@effect/vitest" +import { EdgeCacheService, MemoryCacheBackendLive } from "@maple/cache" +import { OrgId } from "@maple/domain/primitives" +import { ConfigProvider, Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { FetchHttpClient } from "effect/http" +import { Env } from "@maple/backend/platform/Env" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { GcpLiveMetricsService } from "./GcpLiveMetricsService" + +const now = Date.UTC(2026, 9, 15, 12, 0, 0) +const ORG = Schema.decodeSync(OrgId)("org_gcp") +const OTHER_ORG = "org_other" +const TENANT = { orgId: ORG } + +// Connector ids and the reader account each one's setup script creates. +const PROJECT_CONNECTOR = "018f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8" +const FOLDER_CONNECTOR = "028f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8" +const OTHER_CONNECTOR = "038f2b3c-4d5e-4f70-8192-a3b4c5d6e7f8" +const readerOf = (connectorId: string, hostProject: string) => + `maple-${connectorId.replaceAll("-", "").slice(0, 24)}@${hostProject}.iam.gserviceaccount.com` +const PROJECT_READER = readerOf(PROJECT_CONNECTOR, "acme-prod") +const FOLDER_READER = readerOf(FOLDER_CONNECTOR, "acme-host") + +let encodedKey = "" + +beforeAll(async () => { + const pair = await crypto.subtle.generateKey( + { + name: "RSASSA-PKCS1-v1_5", + modulusLength: 2048, + publicExponent: new Uint8Array([1, 0, 1]), + hash: "SHA-256", + }, + true, + ["sign", "verify"], + ) + const body = Buffer.from(await crypto.subtle.exportKey("pkcs8", pair.privateKey)).toString("base64") + encodedKey = Buffer.from( + JSON.stringify({ + private_key_id: "key-1", + private_key: `-----BEGIN PRIVATE KEY-----\n${body.match(/.{1,64}/g)!.join("\n")}\n-----END PRIVATE KEY-----\n`, + client_email: "collector@maple-prod.iam.gserviceaccount.com", + }), + ).toString("base64") +}) + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + +const denied = () => json({ error: { code: 403, status: "PERMISSION_DENIED", message: "denied" } }, 403) + +const googleError = (status: number, code: string, reason?: string) => + json( + { + error: { + code: status, + status: code, + message: "refused", + ...(reason === undefined + ? undefined + : { details: [{ "@type": "type.googleapis.com/google.rpc.ErrorInfo", reason }] }), + }, + }, + status, + ) + +interface PromQlCall { + readonly project: string + /** `query` or `query_range`. */ + readonly path: string + readonly params: Record + /** The reader account whose token made the call. */ + readonly reader: string + readonly quotaProject: string | null +} + +interface Calls { + oauth: number + readonly impersonate: Array + readonly promql: Array + /** `project pageToken` per descriptor page read. */ + readonly descriptors: Array + readonly assets: Array + readonly scopes: Array +} + +interface StubOptions { + readonly impersonate?: (account: string) => Response | undefined + readonly promql?: (call: PromQlCall) => Response | undefined + /** Projects under `folders/123`, as `[projectId, state]`. */ + readonly folderProjects?: ReadonlyArray + readonly assets?: () => Response | undefined + /** A project's active descriptors, by page token (`""` for the first page). */ + readonly descriptors?: (project: string, pageToken: string) => Response | undefined + /** Projects a project's metrics scope reads besides itself. */ + readonly scopes?: Record> +} + +/** A stand-in project number: Google names a metrics scope's projects by number. */ +const numberOf = (projectId: string) => + String([...projectId].reduce((hash, char) => (hash * 31 + char.charCodeAt(0)) % 1_000_000_007, 7)) + +const matrix = ( + ...result: ReadonlyArray<{ metric: Record; values: Array<[number, string]> }> +) => json({ status: "success", data: { resultType: "matrix", result } }) + +const stubFetch = (calls: Calls, options: StubOptions = {}) => + (async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL( + typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url, + ) + const headers = new Headers(init?.headers) + const reader = (headers.get("authorization") ?? "").replace("Bearer reader:", "") + if (url.host === "oauth2.googleapis.com") { + calls.oauth += 1 + return json({ access_token: "ya29.maple", expires_in: 600 }) + } + if (url.host === "iamcredentials.googleapis.com") { + const account = decodeURIComponent( + url.pathname + .replace("/v1/projects/-/serviceAccounts/", "") + .replace(":generateAccessToken", ""), + ) + calls.impersonate.push(account) + return options.impersonate?.(account) ?? json({ accessToken: `reader:${account}` }) + } + if (url.host === "cloudasset.googleapis.com") { + calls.assets.push(`${url.pathname} ${url.searchParams.get("assetTypes")}`) + return ( + options.assets?.() ?? + json({ + results: (options.folderProjects ?? []).map(([projectId, state]) => ({ + name: `//cloudresourcemanager.googleapis.com/projects/${projectId.length}0001`, + assetType: "cloudresourcemanager.googleapis.com/Project", + state, + additionalAttributes: { projectId }, + })), + }) + ) + } + const promql = /^\/v1\/projects\/([^/]+)\/location\/global\/prometheus\/api\/v1\/(.+)$/.exec( + url.pathname, + ) + if (url.host === "monitoring.googleapis.com" && promql !== null) { + const call = { + project: promql[1]!, + path: promql[2]!, + params: Object.fromEntries( + new URLSearchParams(await new Response(init?.body ?? null).text()), + ), + reader, + quotaProject: headers.get("x-goog-user-project"), + } + calls.promql.push(call) + return options.promql?.(call) ?? matrix() + } + const scope = /^\/v1\/locations\/global\/metricsScopes\/(.+)$/.exec(url.pathname) + if (url.host === "monitoring.googleapis.com" && scope !== null) { + const projectId = scope[1]! + calls.scopes.push(projectId) + return json({ + name: `locations/global/metricsScopes/${numberOf(projectId)}`, + monitoredProjects: [projectId, ...(options.scopes?.[projectId] ?? [])].map((monitored) => ({ + name: `locations/global/metricsScopes/${numberOf(projectId)}/projects/${numberOf(monitored)}`, + })), + }) + } + const descriptors = /^\/v3\/projects\/([^/]+)\/metricDescriptors$/.exec(url.pathname) + if (url.host === "monitoring.googleapis.com" && descriptors !== null) { + const pageToken = url.searchParams.get("pageToken") ?? "" + calls.descriptors.push(`${descriptors[1]} ${pageToken}`.trim()) + assert.strictEqual(url.searchParams.get("activeOnly"), "true") + return options.descriptors?.(descriptors[1]!, pageToken) ?? json({}) + } + throw new Error(`unexpected request to ${url.href}`) + }) as typeof fetch + +const makeCalls = (): Calls => ({ + oauth: 0, + impersonate: [], + promql: [], + descriptors: [], + assets: [], + scopes: [], +}) + +const run = ( + testDb: TestDb, + stub: typeof fetch, + effect: Effect.Effect, + options: { readonly key?: string | null } = {}, +) => { + const key = options.key === undefined ? encodedKey : options.key + return Effect.gen(function* () { + yield* TestClock.setTime(now) + return yield* effect + }).pipe( + Effect.provide( + Layer.effect(GcpLiveMetricsService, GcpLiveMetricsService.make).pipe( + Layer.provide(FetchHttpClient.layer), + Layer.provide(Layer.succeed(FetchHttpClient.Fetch, stub)), + Layer.provide(EdgeCacheService.layer.pipe(Layer.provide(MemoryCacheBackendLive))), + Layer.provide(testDb.layer), + Layer.provide(Env.layer), + Layer.provide( + ConfigProvider.layer( + ConfigProvider.fromUnknown({ + PORT: "3472", + TINYBIRD_HOST: "https://api.tinybird.co", + TINYBIRD_TOKEN: "test-token", + MAPLE_AUTH_MODE: "self_hosted", + MAPLE_ROOT_PASSWORD: "test-root-password", + MAPLE_DEFAULT_ORG_ID: "default", + MAPLE_INGEST_KEY_ENCRYPTION_KEY: Buffer.alloc(32, 9).toString("base64"), + MAPLE_INGEST_KEY_LOOKUP_HMAC_KEY: "maple-test-lookup-secret", + ...(key === null ? undefined : { MAPLE_GCP_SERVICE_ACCOUNT_KEY: key }), + }), + ), + ), + ), + ), + ) +} + +const insertConnector = ( + testDb: TestDb, + id: string, + options: { + readonly orgId?: string + readonly projectId?: string + readonly folder?: string + /** What the setup script reported: the read-only account is in place. */ + readonly metricsOn?: boolean | null + } = {}, +) => { + const projectId = options.projectId ?? "acme-prod" + return Effect.promise(() => + executeSql( + testDb, + `INSERT INTO gcp_connectors + (id, org_id, scope_type, scope_id, project_id, secret_ciphertext, secret_iv, secret_tag, + secret_hash, applied_metrics_enabled, created_by, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, 'c', 'i', 't', $1, $6, 'user_1', now(), now())`, + [ + id, + options.orgId ?? ORG, + options.folder === undefined ? "project" : "folder", + options.folder ?? projectId, + projectId, + options.metricsOn === undefined ? true : options.metricsOn, + ], + ), + ) +} + +/** The org's two connections: project `acme-prod`, and folder 123 hosted in `acme-host`. */ +const insertConnectors = (testDb: TestDb) => + Effect.all([ + insertConnector(testDb, PROJECT_CONNECTOR), + insertConnector(testDb, FOLDER_CONNECTOR, { projectId: "acme-host", folder: "123" }), + insertConnector(testDb, OTHER_CONNECTOR, { orgId: OTHER_ORG, projectId: "rival-prod" }), + ]) + +const FOLDER_PROJECTS = [ + ["acme-shop", "ACTIVE"], + ["acme-data", "ACTIVE"], + ["acme-gone", "DELETE_REQUESTED"], +] as const + +const T0 = Date.UTC(2026, 9, 15, 11, 0) / 1000 + +const range = (query: string, projects?: ReadonlyArray) => + GcpLiveMetricsService.use((gcp) => + gcp.queryRange(TENANT, { + query, + startSeconds: T0, + endSeconds: T0 + 3600, + stepSeconds: 600, + ...(projects === undefined ? undefined : { projects }), + }), + ) + +const failureScopes = (error: { readonly _tag: string }) => + "failures" in error && Array.isArray(error.failures) + ? error.failures.map((failure: { readonly scope: string }) => failure.scope) + : [] + +const firstFailure = (error: { readonly _tag: string }) => + "failures" in error && Array.isArray(error.failures) + ? (error.failures[0] as { readonly message: string }).message + : error._tag + +describe("GcpLiveMetricsService.queryRange", () => { + it.effect("reads every project of the org's connections, each with its connection's reader", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + promql: (call) => + call.project === "acme-prod" + ? // Google left out `project_id`, and two points are not numbers. + matrix({ + metric: { service: "checkout" }, + values: [ + [T0, "3"], + [T0 + 600, "NaN"], + [T0 + 1200, "+Inf"], + [T0 + 1800, "1.5"], + ], + }) + : // Google's own `project_id` stays. + matrix({ + metric: { + service: "checkout", + project_id: + call.project === "acme-shop" ? "acme-monitored" : call.project, + }, + values: [[T0, "7"]], + }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const series = yield* range("sum by (service) (rate(x[5m]))") + + assert.sameDeepMembers( + [...series], + [ + { labels: { service: "checkout", project_id: "acme-data" }, points: [[T0, 7]] }, + { + labels: { service: "checkout", project_id: "acme-prod" }, + points: [ + [T0, 3], + [T0 + 1800, 1.5], + ], + }, + { labels: { service: "checkout", project_id: "acme-monitored" }, points: [[T0, 7]] }, + ], + ) + // Only the folder is searched, for projects only; the other org's connection is never read. + assert.deepStrictEqual(calls.assets, [ + "/v1/folders/123:searchAllResources cloudresourcemanager.googleapis.com/Project", + ]) + assert.sameDeepMembers( + calls.promql.map(({ project, reader, quotaProject }) => ({ + project, + reader, + quotaProject, + })), + [ + { project: "acme-prod", reader: PROJECT_READER, quotaProject: "acme-prod" }, + { project: "acme-shop", reader: FOLDER_READER, quotaProject: "acme-host" }, + { project: "acme-data", reader: FOLDER_READER, quotaProject: "acme-host" }, + ], + ) + assert.strictEqual(calls.promql[0]!.path, "query_range") + assert.deepInclude(calls.promql[0]!.params, { + query: "sum by (service) (rate(x[5m]))", + start: String(T0), + end: String(T0 + 3600), + step: "600", + }) + }), + ) + }) + + it.effect("reads only the projects asked for, and refuses one the connections do not reach", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { folderProjects: FOLDER_PROJECTS }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + yield* range("up", ["acme-shop"]) + assert.deepStrictEqual( + calls.promql.map((call) => call.project), + ["acme-shop"], + ) + + for (const outside of ["rival-prod", "acme-gone"]) { + const error = yield* Effect.flip(range("up", ["acme-shop", outside])) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryRejectedError") + assert.strictEqual("reason" in error ? error.reason : undefined, "invalid_query") + assert.include(error.message, outside) + } + assert.lengthOf(calls.promql, 1) + }), + ) + }) + + it.effect("refuses to read more than 20 projects unless the query filters on project_id", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const many = Array.from({ length: 20 }, (_, index) => [`acme-team-${index + 10}`, "ACTIVE"] as const) + return run( + testDb, + stubFetch(calls, { folderProjects: many }), + Effect.gen(function* () { + yield* insertConnector(testDb, FOLDER_CONNECTOR, { projectId: "acme-host", folder: "123" }) + yield* range("up") + assert.lengthOf(calls.promql, 20) + + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(range("down")) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryRejectedError") + assert.strictEqual("reason" in error ? error.reason : undefined, "too_many_projects") + assert.include(error.message, "21 projects") + assert.include(error.message, "project_id") + assert.include(error.message, "acme-prod") + assert.lengthOf(calls.promql, 20) + + yield* range("up", ["acme-team-10", "acme-prod"]) + assert.sameMembers( + calls.promql.slice(20).map((call) => call.project), + ["acme-prod", "acme-team-10"], + ) + }), + ) + }) + + it.effect("returns what answered when part fails, and fails only when nothing answered", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + promql: (call) => + call.project === "acme-prod" + ? matrix({ metric: { project_id: "acme-prod" }, values: [[T0, "1"]] }) + : denied(), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const series = yield* range("up") + assert.deepStrictEqual(series, [{ labels: { project_id: "acme-prod" }, points: [[T0, 1]] }]) + + const error = yield* Effect.flip(range("up", ["acme-data", "acme-shop"])) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryFailedError") + assert.sameMembers(failureScopes(error), ["projects/acme-data", "projects/acme-shop"]) + assert.strictEqual( + firstFailure(error), + "Google denied Maple's read of folder 123: the read-only roles are missing. Grant them with the setup script, the console steps or your Terraform. A new grant can take a few minutes. (Cloud Monitoring returned 403 PERMISSION_DENIED)", + ) + }), + ) + }) + + it.effect("reads the rest when a connection cannot sign in or its projects cannot be listed", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + assets: denied, + promql: (call) => matrix({ metric: { project_id: call.project }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const series = yield* range("up") + assert.deepStrictEqual( + series.map((one) => one.labels.project_id), + ["acme-prod"], + ) + + // A project the failed listing may hold is not refused as unknown. + const error = yield* Effect.flip(range("up", ["acme-shop"])) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryFailedError") + assert.deepStrictEqual(failureScopes(error), ["folders/123"]) + assert.include(firstFailure(error), "Cloud Asset Viewer is missing") + }), + ) + }) + + it.effect("passes Google's refusal of the query on as an invalid query", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const refusal = `invalid parameter "query": ${"x".repeat(2_000)}` + return run( + testDb, + stubFetch(calls, { + promql: (call) => + call.params.query === "slow" + ? json( + { + status: "error", + errorType: "execution", + error: "query processing would load too many samples", + }, + 422, + ) + : call.params.query === "busy" + ? // Google's front end answers a rate limit with a small HTML page. + new Response("Too Many Requests", { + status: 429, + headers: { "content-type": "text/html" }, + }) + : json({ status: "error", errorType: "bad_data", error: refusal }, 400), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(range('{"serviceruntime.googleapis.com/api/request_count"}')) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryRejectedError") + assert.strictEqual("reason" in error ? error.reason : undefined, "invalid_query") + assert.strictEqual(error.message, refusal.slice(0, 1_000)) + + const slow = yield* Effect.flip(range("slow")) + assert.strictEqual( + firstFailure(slow), + "Google could not finish the query on this project. Narrow the query or widen the step. (Cloud Monitoring returned 422 execution: query processing would load too many samples)", + ) + const busy = yield* Effect.flip(range("busy")) + assert.strictEqual(busy._tag, "@maple/http/errors/GcpLiveQueryFailedError") + assert.strictEqual( + firstFailure(busy), + "Google refused the read for the project acme-prod: a rate limit, or an answer past Google's size limit. Narrow the query, or try again in a few minutes. (Cloud Monitoring returned 429)", + ) + }), + ) + }) + + it.effect("refuses an answer of more than 500 series across its projects, never cut short", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + // 250 series from each project: 750 in all, 500 from the folder alone. + promql: (call) => + matrix( + ...Array.from({ length: 250 }, (_, index) => ({ + metric: { instance: `vm-${index}`, project_id: call.project }, + values: [[T0, "1"]] as Array<[number, string]>, + })), + ), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const error = yield* Effect.flip(range("up")) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryRejectedError") + assert.strictEqual("reason" in error ? error.reason : undefined, "too_many_series") + assert.include(error.message, "group by fewer labels") + + const atTheCap = yield* range("up", ["acme-shop", "acme-data"]) + assert.lengthOf(atTheCap, 500) + }), + ) + }) + + it.effect("refuses a project's answer past 8 MB as too many series", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + promql: () => matrix({ metric: { pod: "x".repeat(9 * 1024 * 1024) }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(range("up")) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryRejectedError") + assert.strictEqual("reason" in error ? error.reason : undefined, "too_many_series") + assert.include(error.message, "larger than 8 MB") + }), + ) + }) + + it.effect("reads a project another target's metrics scope holds through that scope, once", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + scopes: { "acme-data": ["acme-shop"] }, + // What Google answers for `sum by (service)`: no `project_id`. + promql: (call) => + matrix({ + metric: { service: "checkout" }, + values: [[T0, call.project === "acme-data" ? "9" : "3"]], + }), + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const series = yield* range("sum by (service) (rate(x[5m]))") + assert.sameMembers( + calls.promql.map((call) => call.project), + ["acme-data", "acme-prod"], + ) + // acme-data's sum covers acme-shop too, so it names the scope, not a project. + assert.sameDeepMembers( + series.map((one) => one.labels), + [ + { service: "checkout", metrics_scope: "acme-data" }, + { service: "checkout", project_id: "acme-prod" }, + ], + ) + }), + ) + }) + + it.effect("reads only connections with metrics on, and is unavailable without one", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const other = Schema.decodeSync(OrgId)(OTHER_ORG) + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + promql: (call) => matrix({ metric: { project_id: call.project }, values: [[T0, "1"]] }), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + yield* insertConnector(testDb, FOLDER_CONNECTOR, { + projectId: "acme-host", + folder: "123", + metricsOn: null, + }) + yield* insertConnector(testDb, OTHER_CONNECTOR, { + orgId: OTHER_ORG, + projectId: "rival-prod", + metricsOn: false, + }) + const series = yield* range("up") + assert.deepStrictEqual( + series.map((one) => one.labels.project_id), + ["acme-prod"], + ) + assert.deepStrictEqual(calls.assets, []) + assert.deepStrictEqual(calls.impersonate, [PROJECT_READER]) + + const error = yield* Effect.flip( + GcpLiveMetricsService.use((gcp) => + gcp.queryRange( + { orgId: other }, + { query: "up", startSeconds: T0, endSeconds: T0, stepSeconds: 60 }, + ), + ), + ) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryUnavailableError") + assert.strictEqual( + error.message, + "Connect Google Cloud with metrics on to query Google Cloud metrics.", + ) + }), + ) + }) + + it.effect("is unavailable without Maple's Google key, before any call", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(range("up")) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryUnavailableError") + assert.deepStrictEqual(calls, makeCalls()) + }), + { key: null }, + ) + }) + + it.effect("says a refusal of Maple's own project is Maple's, not the connection's setup", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { + impersonate: () => googleError(403, "PERMISSION_DENIED", "SERVICE_DISABLED"), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(range("up")) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryFailedError") + assert.strictEqual( + firstFailure(error), + "Maple's request to Google's IAM Credentials API failed. Try again in a few minutes. If this repeats, write to support@maple.dev. (IAM Credentials returned 403 PERMISSION_DENIED, SERVICE_DISABLED)", + ) + }), + ) + }) + + it.effect("tells a connection whose grant is missing what to do", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { impersonate: denied }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(range("up")) + assert.deepStrictEqual(failureScopes(error), ["projects/acme-prod"]) + assert.include(firstFailure(error), "run the setup script again") + }), + ) + }) + + it.effect("signs in once and reuses the tokens for later queries", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { folderProjects: FOLDER_PROJECTS }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + yield* range("up") + yield* range("down") + assert.strictEqual(calls.oauth, 1) + assert.sameMembers(calls.impersonate, [PROJECT_READER, FOLDER_READER]) + // The folder's projects and each project's metrics scope are read once an hour, not per query. + assert.lengthOf(calls.assets, 1) + assert.lengthOf(calls.scopes, 3) + // Answers are not cached: each query reads Google. + assert.lengthOf(calls.promql, 6) + + // Ten minutes on, both tokens have expired. + yield* TestClock.adjust("10 minutes") + yield* range("sideways") + assert.strictEqual(calls.oauth, 2) + assert.lengthOf(calls.impersonate, 4) + }), + ) + }) +}) + +describe("GcpLiveMetricsService.queryInstant", () => { + it.effect("reads one point per series, a scalar as one series, and refuses a string", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const instant = (query: string) => + GcpLiveMetricsService.use((gcp) => gcp.queryInstant(TENANT, { query, timeSeconds: T0 })) + return run( + testDb, + stubFetch(calls, { + promql: (call) => + call.params.query === "up" + ? json({ + status: "success", + data: { + resultType: "vector", + result: [ + { metric: { project_id: "acme-prod", zone: "a" }, value: [T0, "4"] }, + ], + }, + }) + : call.params.query === '"text"' + ? json({ + status: "success", + data: { resultType: "string", result: [T0, "text"] }, + }) + : json({ status: "success", data: { resultType: "scalar", result: [T0, "2"] } }), + }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + assert.deepStrictEqual(yield* instant("up"), [ + { labels: { project_id: "acme-prod", zone: "a" }, points: [[T0, 4]] }, + ]) + assert.strictEqual(calls.promql[0]!.path, "query") + assert.deepInclude(calls.promql[0]!.params, { query: "up", time: String(T0) }) + + assert.deepStrictEqual(yield* instant("1 + 1"), [ + { labels: { project_id: "acme-prod" }, points: [[T0, 2]] }, + ]) + + const text = yield* Effect.flip(instant('"text"')) + assert.strictEqual(text._tag, "@maple/http/errors/GcpLiveQueryRejectedError") + assert.include(text.message, "String results are not supported") + }), + ) + }) +}) + +const descriptor = (type: string, extra: Record = {}) => ({ + type, + metricKind: "GAUGE", + valueType: "DOUBLE", + unit: "1", + description: `${type} description`, + ...extra, +}) + +describe("GcpLiveMetricsService discovery", () => { + it.effect("lists the active metrics of every project, merged per type, and caches them", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + const CPU = "cloudsql.googleapis.com/database/cpu/utilization" + return run( + testDb, + stubFetch(calls, { + folderProjects: FOLDER_PROJECTS, + descriptors: (project, pageToken) => { + if (project === "acme-prod") { + return pageToken === "" + ? json({ + metricDescriptors: [ + descriptor(CPU, { + monitoredResourceTypes: ["cloudsql_database"], + labels: [{ key: "database", description: "The database." }], + }), + // A logging metric: Google lists no resource types. + descriptor("logging.googleapis.com/user/checkout_errors", { + metricKind: "DELTA", + valueType: "INT64", + labels: [{ key: "severity" }], + }), + ], + nextPageToken: "page-2", + }) + : json({ + metricDescriptors: [ + descriptor("run.googleapis.com/request_latencies", { + metricKind: "DELTA", + valueType: "DISTRIBUTION", + monitoredResourceTypes: ["cloud_run_revision"], + }), + ], + }) + } + if (project === "acme-shop") { + return json({ + metricDescriptors: [ + descriptor(CPU, { + monitoredResourceTypes: [ + "cloudsql_instance_database", + "cloudsql_database", + ], + labels: [{ key: "replica" }, { key: "database" }], + }), + // A kind Google leaves unspecified is not a metric Maple can query. + descriptor("custom.googleapis.com/odd", { + metricKind: "METRIC_KIND_UNSPECIFIED", + }), + ], + }) + } + return denied() + }, + }), + Effect.gen(function* () { + yield* insertConnectors(testDb) + const metrics = yield* GcpLiveMetricsService.use((gcp) => gcp.listMetrics(TENANT)) + assert.deepStrictEqual(metrics, [ + { + type: CPU, + metricKind: "GAUGE", + valueType: "DOUBLE", + unit: "1", + description: `${CPU} description`, + monitoredResourceTypes: ["cloudsql_database", "cloudsql_instance_database"], + labels: ["database", "replica"], + }, + { + type: "logging.googleapis.com/user/checkout_errors", + metricKind: "DELTA", + valueType: "INT64", + unit: "1", + description: "logging.googleapis.com/user/checkout_errors description", + monitoredResourceTypes: [], + labels: ["severity"], + }, + { + type: "run.googleapis.com/request_latencies", + metricKind: "DELTA", + valueType: "DISTRIBUTION", + unit: "1", + description: "run.googleapis.com/request_latencies description", + monitoredResourceTypes: ["cloud_run_revision"], + labels: [], + }, + ]) + assert.sameMembers(calls.descriptors, [ + "acme-prod", + "acme-prod page-2", + "acme-shop", + "acme-data", + ]) + + // From the cache: only the project that failed is read again. + const described = yield* GcpLiveMetricsService.use((gcp) => gcp.describeMetric(TENANT, CPU)) + assert.deepStrictEqual(described?.labels, ["database", "replica"]) + assert.lengthOf(calls.descriptors, 5) + assert.isUndefined( + yield* GcpLiveMetricsService.use((gcp) => + gcp.describeMetric(TENANT, "custom.googleapis.com/odd"), + ), + ) + }), + ) + }) + + it.effect("fails the listing only when no project answered", () => { + const testDb = createTestDb(trackedDbs) + const calls = makeCalls() + return run( + testDb, + stubFetch(calls, { descriptors: denied }), + Effect.gen(function* () { + yield* insertConnector(testDb, PROJECT_CONNECTOR) + const error = yield* Effect.flip(GcpLiveMetricsService.use((gcp) => gcp.listMetrics(TENANT))) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryFailedError") + assert.deepStrictEqual(failureScopes(error), ["projects/acme-prod"]) + }), + ) + }) +}) diff --git a/packages/backend/src/services/integrations/GcpLiveMetricsService.ts b/packages/backend/src/services/integrations/GcpLiveMetricsService.ts new file mode 100644 index 0000000000..632a32a843 --- /dev/null +++ b/packages/backend/src/services/integrations/GcpLiveMetricsService.ts @@ -0,0 +1,515 @@ +/** + * Google Cloud Monitoring metrics, read live through Google's Prometheus-compatible API: the query + * engine's `GcpLiveMetrics` port. A connection with metrics on (its read-only account in place) + * is a target: a project scope is its project, a folder or organization the active projects + * Cloud Asset finds under it. Each project is read with its connection's read-only account, one + * call per metrics scope. What answered is returned; only when nothing did is it a failure. + */ +import * as PG from "@maple-dev/effect-orm/postgres" +import { EdgeCacheService } from "@maple/cache" +import { GcpConnectors, type GcpConnectorRow } from "@maple/db/tables" +import type { GcpMetricDescriptor, GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import { + GcpLiveQueryFailedError, + GcpLiveQueryRejectedError, + GcpLiveQueryUnavailableError, +} from "@maple/domain/http/gcp-live-query-errors" +import { GcpProjectId, type OrgId } from "@maple/domain/primitives" +import type { GcpLiveMetrics, QueryTenant } from "@maple/query-engine/runtime" +import { Clock, Context, Effect, Layer, Option, Redacted, Schema } from "effect" +import { FetchHttpClient, HttpClient } from "effect/http" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { Env } from "@maple/backend/platform/Env" +import { + ActiveMetricDescriptor, + fetchMapleAccessToken, + getMetricsScope, + type GcpApiError, + type GcpPromQlParams, + type GcpReader, + importServiceAccountKey, + impersonateReader, + listActiveMetricDescriptorsPage, + promQlQuery, + searchProjectsPage, +} from "./gcp/api" +import { describeGcpApiError } from "./gcp/explain" +import { mapGcpResource } from "./gcp/mapping" + +/** Google's tokens live 600 s. Three minutes to spare outlast the slowest fan-out. */ +const TOKEN_REUSE_MS = 7 * 60_000 +const CONCURRENCY = 4 +/** The engine has no project picker: past this a query must filter on `project_id`. */ +const MAX_PROJECTS = 20 +const MAX_SERIES = 500 +/** Projects a folder or organization search reads, 500 a page. */ +const MAX_PROJECT_PAGES = 4 +/** 10,000 descriptors a page; a project rarely fills one. */ +const MAX_DESCRIPTOR_PAGES = 5 +const HOUR_SECONDS = 3_600 +/** Above the default: a miss here costs a Google call, not a row lookup. */ +const CACHE_READ_TIMEOUT_MS = 250 +const API_ERROR = "@maple/api/integrations/GcpApiError" + +const isProjectId = Schema.is(GcpProjectId) + +type Connector = Pick +type Failure = GcpLiveQueryFailedError["failures"][number] +type Port = GcpLiveMetrics +interface Target { + readonly projectId: GcpProjectId + readonly connector: Connector +} + +const scopeName = (connector: Connector) => `${connector.scopeType}s/${connector.scopeId}` + +/** A project's metrics scope: its number, and the numbers of the projects it reads, itself included. */ +const MetricsScope = Schema.Struct({ number: Schema.String, monitored: Schema.Array(Schema.String) }) + +const nothingRead = (failures: ReadonlyArray) => + new GcpLiveQueryFailedError({ + message: `Google Cloud could not be read. ${failures.map((failure) => `${failure.scope}: ${failure.message}`).join(" ")}`, + failures, + }) + +const mapleSignInFailed = () => + new GcpLiveQueryFailedError({ + message: + "Maple could not sign in to Google Cloud. Try again in a few minutes. If this repeats, write to support@maple.dev.", + failures: [], + }) + +/** Every project's descriptors as one list: a type's resource types and labels are the union. */ +const mergeDescriptors = (lists: ReadonlyArray>) => { + const merged = new Map< + string, + { first: GcpMetricDescriptor; resources: Set; labels: Set } + >() + for (const descriptor of lists.flat()) { + const entry = merged.get(descriptor.type) + if (entry === undefined) { + merged.set(descriptor.type, { + first: descriptor, + resources: new Set(descriptor.monitoredResourceTypes), + labels: new Set(descriptor.labels), + }) + continue + } + for (const resource of descriptor.monitoredResourceTypes) entry.resources.add(resource) + for (const label of descriptor.labels) entry.labels.add(label) + } + return [...merged.values()] + .map(({ first, resources, labels }): GcpMetricDescriptor => ({ + ...first, + monitoredResourceTypes: [...resources].sort(), + labels: [...labels].sort(), + })) + .sort((a, b) => a.type.localeCompare(b.type)) +} + +export class GcpLiveMetricsService extends Context.Service()( + "@maple/api/services/GcpLiveMetricsService", + { + make: Effect.gen(function* () { + const database = yield* Database + const env = yield* Env + const httpClient = yield* HttpClient.HttpClient + const edgeCache = yield* EdgeCacheService + // A failed connector read is a defect: the port has no persistence error. + const dbExecute = makeDbExecute(database, "GcpLiveMetricsService", (error) => error) + + const signingKey = Option.match(env.MAPLE_GCP_SERVICE_ACCOUNT_KEY, { + onNone: () => + Effect.fail( + new GcpLiveQueryUnavailableError({ + message: + "Google Cloud metrics are not available on this Maple deployment: it has no Google service account key (MAPLE_GCP_SERVICE_ACCOUNT_KEY).", + }), + ), + onSome: Effect.succeed, + }) + + // Tokens are kept for the isolate's lifetime. Only settled values are kept: a pending + // sign-in shared across requests would resume in another request's I/O context. + let mapleToken: { readonly token: Redacted.Redacted; readonly until: number } | undefined + const readers = new Map() + + /** A connector's reader. Its own failure is a `GcpApiError`; Maple's sign-in fails the query. */ + const readerFor = Effect.fnUntraced(function* ( + key: Redacted.Redacted, + connector: Connector, + ) { + const now = yield* Clock.currentTimeMillis + const cached = readers.get(connector.id) + if (cached !== undefined && cached.until > now) return cached.reader + if (mapleToken === undefined || mapleToken.until <= now) { + const token = yield* importServiceAccountKey(key).pipe( + Effect.flatMap((signer) => fetchMapleAccessToken(httpClient, signer, now)), + Effect.tapError((error) => + Effect.logError("gcp sign-in as Maple failed", { error: error.message }), + ), + Effect.mapError(mapleSignInFailed), + ) + mapleToken = { token, until: now + TOKEN_REUSE_MS } + } + const reader = yield* impersonateReader(httpClient, mapleToken.token, connector) + readers.set(connector.id, { reader, until: now + TOKEN_REUSE_MS }) + return reader + }) + + const searchProjects = Effect.fnUntraced(function* ( + key: Redacted.Redacted, + connector: Connector, + ) { + const reader = yield* readerFor(key, connector) + // The id is validated when the connector is created; encoding keeps it inside the path. + const scope = `${connector.scopeType}s/${encodeURIComponent(connector.scopeId)}` + const projects: Array = [] + let pageToken: string | undefined + for (let page = 0; page < MAX_PROJECT_PAGES; page++) { + const body = yield* searchProjectsPage(httpClient, reader, scope, pageToken) + for (const result of body.results ?? []) { + const projectId = mapGcpResource(result)?.projectId + // A project pending deletion answers nothing but errors. + if ((result.state ?? "ACTIVE") === "ACTIVE" && isProjectId(projectId)) + projects.push(projectId) + } + pageToken = body.nextPageToken + if (pageToken === undefined) break + } + return projects + }) + + /** + * Every project the org's metrics-on connections reach, each with the connection that + * reads it, and the connections whose projects could not be listed. + */ + const targetsOf = Effect.fnUntraced(function* (orgId: OrgId, key: Redacted.Redacted) { + const connectors = yield* dbExecute((db) => + db.run( + PG.from(GcpConnectors) + .select("id", "scopeType", "scopeId", "projectId", "lastMetricsReceivedAt") + .where(($) => [$.orgId.eq(orgId), $.appliedMetricsEnabled.eq(true)]) + .orderBy(($) => [ + [$.createdAt, "asc"], + [$.id, "asc"], + ]), + ), + ).pipe(Effect.orDie) + if (connectors.length === 0) { + return yield* new GcpLiveQueryUnavailableError({ + message: "Connect Google Cloud with metrics on to query Google Cloud metrics.", + }) + } + // One connection at a time, so a cold isolate signs in as Maple once. + const listed = yield* Effect.forEach(connectors, (connector) => + connector.scopeType === "project" + ? Effect.succeed({ connector, projects: [connector.projectId] }) + : edgeCache + .getOrCompute( + { + bucket: "gcp-projects", + key: `${orgId}:${connector.id}`, + ttlSeconds: HOUR_SECONDS, + readTimeoutMs: CACHE_READ_TIMEOUT_MS, + schema: Schema.Array(GcpProjectId), + }, + searchProjects(key, connector), + ) + .pipe( + Effect.map(({ value }) => ({ connector, projects: value })), + Effect.catchTag(API_ERROR, (error) => + Effect.succeed({ + connector, + projects: [], + failure: { + scope: scopeName(connector), + message: describeGcpApiError(error, connector, "live"), + }, + }), + ), + ), + ) + // Overlapping connections read a project through the oldest. + const projects = new Map() + for (const { connector, projects: ids } of listed) { + for (const projectId of ids) { + if (!projects.has(projectId)) projects.set(projectId, { projectId, connector }) + } + } + return { + projects, + failures: listed.flatMap((entry) => ("failure" in entry ? [entry.failure] : [])), + } + }) + + /** Signs each connection in once, so one that cannot fails once, not once per project. */ + const signIn = Effect.fnUntraced(function* ( + key: Redacted.Redacted, + targets: ReadonlyArray, + ) { + const signedIn = new Map() + const failures: Array = [] + for (const connector of new Set(targets.map((target) => target.connector))) { + const reader = yield* readerFor(key, connector).pipe( + Effect.catchTag(API_ERROR, (error) => { + failures.push({ + scope: scopeName(connector), + message: describeGcpApiError(error, connector, "live"), + }) + return Effect.succeed(undefined) + }), + ) + if (reader !== undefined) signedIn.set(connector, reader) + } + return { + reads: targets.flatMap((target) => { + const reader = signedIn.get(target.connector) + return reader === undefined ? [] : [{ ...target, reader }] + }), + failures, + } + }) + + const projectFailure = (target: Target, error: GcpApiError): Failure => ({ + scope: `projects/${target.projectId}`, + message: describeGcpApiError(error, target.connector, "live"), + }) + + /** Partial answers have no warnings channel: the failures go to the span and the log. */ + const recordFailures = (failures: ReadonlyArray) => { + const [first] = failures + return first === undefined + ? Effect.void + : Effect.annotateCurrentSpan({ + "maple.gcp.failed_projects": failures.length, + "maple.gcp.failure": first.message, + }).pipe( + Effect.andThen(Effect.logWarning("gcp live read failed in part", { failures })), + ) + } + + /** The projects a query reads: the ones it names, else all, at most `MAX_PROJECTS`. */ + const selectTargets = Effect.fnUntraced(function* ( + targets: Effect.Success>, + requested: ReadonlyArray | undefined, + ) { + const all = [...targets.projects.values()].sort((a, b) => + a.projectId.localeCompare(b.projectId), + ) + if (requested !== undefined) { + const unknown = requested.filter((projectId) => !targets.projects.has(projectId)) + if (unknown.length > 0) { + // A connection whose projects could not be listed may hold them. + if (targets.failures.length > 0) return yield* nothingRead(targets.failures) + return yield* new GcpLiveQueryRejectedError({ + message: `Not among the projects this organization's Google Cloud connections can read: ${unknown.join(", ")}.`, + reason: "invalid_query", + }) + } + return all.filter((target) => requested.includes(target.projectId)) + } + if (all.length > MAX_PROJECTS) { + const named = all.slice(0, 5).map((target) => target.projectId) + return yield* new GcpLiveQueryRejectedError({ + message: `The Google Cloud connections reach ${all.length} projects, and one query reads at most ${MAX_PROJECTS}. Filter on project_id to choose them, e.g. ${named.join(", ")}.`, + reason: "too_many_projects", + }) + } + return all + }) + + const runQuery = Effect.fnUntraced(function* ( + orgId: OrgId, + requested: ReadonlyArray | undefined, + params: GcpPromQlParams, + ) { + const key = yield* signingKey + const targets = yield* targetsOf(orgId, key) + const selected = yield* selectTargets(targets, requested) + yield* Effect.annotateCurrentSpan({ "maple.gcp.project_count": selected.length }) + const signedIn = yield* signIn(key, selected) + + // A query on a project reads its whole metrics scope, so a target that another target's + // scope holds is answered through that one, not read twice. + const scoped = yield* Effect.forEach( + signedIn.reads, + (target) => + edgeCache + .getOrCompute( + { + bucket: "gcp-metrics-scopes", + key: `${orgId}:${target.projectId}`, + ttlSeconds: HOUR_SECONDS, + readTimeoutMs: CACHE_READ_TIMEOUT_MS, + schema: MetricsScope, + }, + getMetricsScope(httpClient, target.reader, target.projectId), + ) + .pipe( + Effect.map(({ value }) => ({ ...target, scope: value })), + Effect.catchTag(API_ERROR, (error) => + Effect.succeed({ ...target, failure: projectFailure(target, error) }), + ), + ), + { concurrency: CONCURRENCY }, + ) + const withScope = scoped.flatMap((target) => ("scope" in target ? [target] : [])) + const covered = new Set() + const reads: typeof withScope = [] + // Widest scopes first. + for (const target of withScope.sort( + (a, b) => b.scope.monitored.length - a.scope.monitored.length, + )) { + if (covered.has(target.scope.number)) continue + for (const number of target.scope.monitored) covered.add(number) + reads.push(target) + } + + let seriesCount = 0 + const answers = yield* Effect.forEach( + reads, + (read) => + promQlQuery(httpClient, read.reader, read.projectId, params).pipe( + // Checked as each project answers, so a broad query stops early. + Effect.flatMap((series) => { + seriesCount += series.length + return seriesCount > MAX_SERIES + ? Effect.fail( + new GcpLiveQueryRejectedError({ + message: `The query returned more than ${MAX_SERIES} series across its projects. Narrow the query or group by fewer labels.`, + reason: "too_many_series", + }), + ) + : Effect.succeed({ read, series }) + }), + Effect.catchTag(API_ERROR, (error) => + Effect.succeed({ read, failure: projectFailure(read, error) }), + ), + ), + { concurrency: CONCURRENCY }, + ) + + const failures = [ + ...targets.failures, + ...signedIn.failures, + ...scoped.flatMap((target) => ("failure" in target ? [target.failure] : [])), + ...answers.flatMap((answer) => ("failure" in answer ? [answer.failure] : [])), + ] + yield* recordFailures(failures) + const answered = answers.flatMap((answer) => ("series" in answer ? [answer] : [])) + if (answered.length === 0 && failures.length > 0) return yield* nothingRead(failures) + const series = answered.flatMap(({ read, series }) => + series.map((one): GcpPromQlSeries => ({ + // Identical label sets from two projects stay apart. A scope of several projects + // names itself, not a project its series may not come from. + labels: + one.labels.project_id === undefined + ? { + ...one.labels, + [read.scope.monitored.length > 1 ? "metrics_scope" : "project_id"]: + read.projectId, + } + : one.labels, + points: one.points, + })), + ) + yield* Effect.annotateCurrentSpan({ "maple.gcp.series_count": series.length }) + return series + }) + + const listDescriptors = Effect.fnUntraced(function* (reader: GcpReader, projectId: GcpProjectId) { + const descriptors: Array = [] + let pageToken: string | undefined + for (let page = 0; page < MAX_DESCRIPTOR_PAGES; page++) { + const body = yield* listActiveMetricDescriptorsPage( + httpClient, + reader, + projectId, + pageToken, + ) + descriptors.push(...body.descriptors) + pageToken = body.nextPageToken + if (pageToken === undefined) break + } + return descriptors + }) + + const listMetrics = Effect.fn("GcpLiveMetrics.listMetrics")(function* (tenant: QueryTenant) { + const key = yield* signingKey + const targets = yield* targetsOf(tenant.orgId, key) + // Discovery has no `project_id` filter to narrow it: past the cap, the first projects speak. + const selected = [...targets.projects.values()] + .sort((a, b) => a.projectId.localeCompare(b.projectId)) + .slice(0, MAX_PROJECTS) + yield* Effect.annotateCurrentSpan({ "maple.gcp.project_count": selected.length }) + const signedIn = yield* signIn(key, selected) + const listed = yield* Effect.forEach( + signedIn.reads, + (target) => + edgeCache + .getOrCompute( + { + bucket: "gcp-metric-descriptors", + key: `${tenant.orgId}:${target.projectId}`, + ttlSeconds: HOUR_SECONDS, + readTimeoutMs: CACHE_READ_TIMEOUT_MS, + schema: Schema.Array(ActiveMetricDescriptor), + }, + listDescriptors(target.reader, target.projectId), + ) + .pipe( + Effect.map(({ value }) => ({ descriptors: value })), + Effect.catchTag(API_ERROR, (error) => + Effect.succeed({ failure: projectFailure(target, error) }), + ), + ), + { concurrency: CONCURRENCY }, + ) + const failures = [ + ...targets.failures, + ...signedIn.failures, + ...listed.flatMap((entry) => ("failure" in entry ? [entry.failure] : [])), + ] + yield* recordFailures(failures) + const lists = listed.flatMap((entry) => ("descriptors" in entry ? [entry.descriptors] : [])) + if (lists.length === 0 && failures.length > 0) return yield* nothingRead(failures) + return mergeDescriptors(lists) + }) + + return { + queryRange: Effect.fn("GcpLiveMetrics.queryRange")(function* ( + tenant: QueryTenant, + request: Parameters[1], + ) { + return yield* runQuery(tenant.orgId, request.projects, { + query: request.query, + start: request.startSeconds, + end: request.endSeconds, + step: request.stepSeconds, + }) + }), + queryInstant: Effect.fn("GcpLiveMetrics.queryInstant")(function* ( + tenant: QueryTenant, + request: Parameters[1], + ) { + return yield* runQuery(tenant.orgId, request.projects, { + query: request.query, + time: request.timeSeconds, + }) + }), + listMetrics, + describeMetric: Effect.fn("GcpLiveMetrics.describeMetric")(function* ( + tenant: QueryTenant, + metricType: string, + ) { + const metrics = yield* listMetrics(tenant) + return metrics.find((metric) => metric.type === metricType) + }), + } satisfies Port + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(FetchHttpClient.layer)) +} diff --git a/packages/backend/src/services/integrations/GcpMetricsService.test.ts b/packages/backend/src/services/integrations/GcpMetricsService.test.ts index b0dcabde10..ccf3ac4e34 100644 --- a/packages/backend/src/services/integrations/GcpMetricsService.test.ts +++ b/packages/backend/src/services/integrations/GcpMetricsService.test.ts @@ -1104,13 +1104,14 @@ describe("GcpMetricsService", () => { yield* pollAll assert.strictEqual(yield* metricsError, READ_DENIED) - // Signing in as the reader does not go through the host project. + // Signing in as the reader does not go through the host project: a refusal for + // Maple's own project is Maple's to fix, not the connection's setup. impersonate = () => googleError(403, "PERMISSION_DENIED", [errorInfo("SERVICE_DISABLED")]) yield* TestClock.setTime(now + 15 * minute) yield* pollAll assert.strictEqual( yield* metricsError, - CANNOT_SIGN_IN.replace("DENIED)", "DENIED, SERVICE_DISABLED)"), + "Maple's request to Google's IAM Credentials API failed. Maple retries in 5 minutes. If this repeats, write to support@maple.dev. (IAM Credentials returned 403 PERMISSION_DENIED, SERVICE_DISABLED)", ) // Details of another form cost neither the status code nor the poll. diff --git a/packages/backend/src/services/integrations/GcpMetricsService.ts b/packages/backend/src/services/integrations/GcpMetricsService.ts index 286202cb08..718bff51cb 100644 --- a/packages/backend/src/services/integrations/GcpMetricsService.ts +++ b/packages/backend/src/services/integrations/GcpMetricsService.ts @@ -39,6 +39,7 @@ import { type GcpTimeSeriesQuery, } from "./gcp/api" import { gcpSqlSettings } from "./gcp/cloud-sql" +import { describeGcpApiError, RETRIES } from "./gcp/explain" import { gcpTopKey, gcpTopSeries, @@ -111,7 +112,6 @@ const halts = (outcome: ReadOutcome) => const SYSTEM_USER_ID = Schema.decodeUnknownSync(UserId)("system-gcp-metrics") -const RETRIES = "Maple retries in 5 minutes." const NOT_STORED = `${RETRIES} Nothing to do.` /** What helps a scope too large for one read. A project has no parts to connect instead. */ @@ -120,50 +120,6 @@ const whenTooLarge = (connector: GcpConnectorRow) => ? "write to support@maple.dev" : "connect its folders or projects separately" -/** Why a failed call failed and what to do about it, in the customer's terms. */ -const explainApiError = ( - error: GcpApiError, - connector: GcpConnectorRow, - reading: "metrics" | "resources", -) => { - // The host project is the one Google checks for billing, enabled APIs and quota. A project - // scope is its own host, and reads as "the project". - const host = `${connector.scopeType === "project" ? "project" : "host project"} ${connector.projectId}` - const scope = `${connector.scopeType} ${connector.scopeId}` - if (error.reason === "BILLING_DISABLED") { - return `The ${host} has no active billing account, and ${error.api} only answers for projects that have one. Link one: https://console.cloud.google.com/billing/linkedaccount?project=${connector.projectId} Maple retries every 5 minutes.` - } - if (error.reason === "SERVICE_DISABLED") { - return `The ${error.api} API is switched off in the ${host}. Switch it on in the console, or run the setup script again.` - } - if (error.kind === "denied" || error.kind === "not_found") { - if (error.api === "IAM Credentials") { - // A connection that has been read before lost its account or its grant. - if (connector.lastMetricsReceivedAt !== null) { - return "Maple can no longer sign in as this connection's read-only service account. The account was deleted, or the grant to Maple was removed or is blocked by an organization policy. Run the setup script again or redo the read-only steps: either restores both." - } - // Shown both before any setup and for the minutes after one in which Google does not - // accept the new grant yet, so it has to read right in both. - return "Maple can't sign in as this connection's read-only service account yet. Google needs a few minutes to accept a new grant, and Maple retries every 5 minutes. If this stays, the account does not exist, or the grant to Maple is missing or blocked by an organization policy: finish the read-only account steps of a manual setup, or run the setup script again." - } - return reading === "metrics" - ? `Google denied Maple's read of ${scope}: the read-only roles are missing. Grant them with the setup script, the console steps or your Terraform. A new grant can take a few minutes.` - : `Google denied the resource listing for ${scope}. Cloud Asset Viewer is missing. Grant it with the setup script, the console steps or your Terraform.` - } - // Signing in is counted against Maple's own project, not the host project. - if (error.kind === "rate_limited" && error.api !== "IAM Credentials") { - return `Google rate-limited the ${error.api} API for the ${host}. ${RETRIES} If this repeats, raise that API's quota on the project.` - } - return `Maple's request to Google's ${error.api} API failed. ${RETRIES} If this repeats, write to support@maple.dev.` -} - -/** - * What the connection shows for a failed call: one sentence of cause, one of what to do, and - * Google's status at the end. - */ -const describeApiError = (error: GcpApiError, connector: GcpConnectorRow, reading: "metrics" | "resources") => - `${explainApiError(error, connector, reading)} (${error.message})` - const toPersistenceError = makePersistenceErrorMapper( IntegrationsPersistenceError, "Google Cloud metrics database error", @@ -435,7 +391,7 @@ export class GcpMetricsService extends Context.Service Effect.succeed({ - lastResourcesError: describeApiError(error, connector, "resources"), + lastResourcesError: describeGcpApiError(error, connector, "poll"), }), "@maple/http/errors/IntegrationsPersistenceError": () => Effect.succeed({ @@ -626,7 +582,7 @@ export class GcpMetricsService extends Context.Service - recordFailure(connector, describeApiError(error, connector, "metrics")), + recordFailure(connector, describeGcpApiError(error, connector, "poll")), "@maple/api/integrations/GcpMetricsIngestError": (error) => error.status === 402 ? recordFailure( diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index df22b4c964..42c97fd397 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -4,19 +4,23 @@ * here may leak a credential: tokens stay `Redacted`, and a failure carries the HTTP status and * Google's error code and reason only, never a response body, transport error or schema issue. * The one sentence of Google's that is read, to tell a missing billing account from a missing - * role, chooses a reason and is dropped. + * role, chooses a reason and is dropped. The exception is PromQL: Google's explanation of why + * the caller's own query was refused or did not finish is passed on. * Untraced on purpose: the HTTP client records a span per request, and "not set up yet" is not an * exception. */ import { gcpConnectorResourceNames } from "@maple/domain/gcp" import { GCP_ASSET_TYPES, + GCP_PROJECT_ASSET_TYPE, gcpMetricAligner, type GcpMetric, type GcpMetricGroup, } from "@maple/domain/gcp-metrics" +import type { GcpMetricDescriptor, GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import { GcpLiveQueryRejectedError } from "@maple/domain/http/gcp-live-query-errors" import type { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" -import { Duration, Effect, Option, Redacted, Schema } from "effect" +import { Duration, Effect, Option, Redacted, Schema, Stream } from "effect" import { HttpClient, HttpClientRequest } from "effect/http" const TOKEN_URL = "https://oauth2.googleapis.com/token" @@ -116,36 +120,80 @@ const errorReason = (body: unknown, kind: GcpApiError["kind"]): string | undefin .flatMap((detail) => Option.toArray(decodeErrorInfo(detail)))[0]?.reason ?? (kind === "denied" ? reasonFromMessage(body) : undefined) +/** + * The largest answer a live query reads: four in flight stay far below the Worker's 128 MB. + * The poller's calls are not capped. + */ +export const MAX_BODY_BYTES = 8 * 1024 * 1024 +const parseJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown)) + +interface CallOptions { + /** Turns a failed answer into the caller's own error. */ + readonly refuse?: (status: number, body: unknown) => E | undefined + /** Reads at most `MAX_BODY_BYTES` of a successful answer; `tooLarge` is the caller's error past it. */ + readonly capped?: { readonly tooLarge?: () => E } +} + /** One JSON call. `api` names the Google service in the error. */ -const call = ( +const call = ( httpClient: HttpClient.HttpClient, api: GcpApiError["api"], request: HttpClientRequest.HttpClientRequest, decode: (body: unknown) => Effect.Effect, -): Effect.Effect => { - const failed = (what: string) => () => - Effect.fail(new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" })) + options: CallOptions = {}, +): Effect.Effect => { + const failed = (what: string) => new GcpApiError({ message: `${api} ${what}`, api, kind: "upstream" }) return Effect.gen(function* () { const response = yield* httpClient.execute(request) - const body = yield* response.json.pipe(Effect.orElseSucceed(() => null)) - if (response.status >= 300) { - const kind = errorKind(response.status) - const reason = errorReason(body, kind) - return yield* new GcpApiError({ - message: `${api} returned ${response.status}${errorCode(body)}${reason === undefined ? "" : `, ${reason}`}`, - api, - kind, - ...(reason === undefined ? undefined : { reason }), - }) + if (options.capped === undefined) { + const body: unknown = yield* response.json.pipe(Effect.orElseSucceed(() => null)) + return { status: response.status, body, overflow: false } } - return yield* decode(body) + // Read as it arrives, never trusting Content-Length: past the cap the rest is left unread. + const chunks: Array = [] + let size = 0 + yield* Stream.runForEachWhile(response.stream, (chunk) => + Effect.sync(() => { + size += chunk.byteLength + if (size > MAX_BODY_BYTES) return false + chunks.push(chunk) + return true + }), + ) + const overflow = size > MAX_BODY_BYTES + // A small non-JSON body (an HTML error page) is no body, never an overflow. + const body = overflow ? null : Option.getOrNull(parseJson(Buffer.concat(chunks).toString("utf8"))) + return { status: response.status, body, overflow } }).pipe( Effect.timeout(REQUEST_TIMEOUT), Effect.catchTags({ - HttpClientError: failed("request failed"), - TimeoutError: failed("request timed out"), - SchemaError: failed("answered with an unexpected body"), + HttpClientError: () => Effect.fail(failed("request failed")), + TimeoutError: () => Effect.fail(failed("request timed out")), }), + Effect.flatMap(({ status, body, overflow }) => + Effect.gen(function* () { + if (status >= 300) { + const refused = options.refuse?.(status, body) + if (refused !== undefined) return yield* Effect.fail(refused) + const kind = errorKind(status) + const reason = errorReason(body, kind) + return yield* new GcpApiError({ + message: `${api} returned ${status}${errorCode(body)}${reason === undefined ? "" : `, ${reason}`}`, + api, + kind, + ...(reason === undefined ? undefined : { reason }), + }) + } + if (overflow) { + return yield* Effect.fail( + options.capped?.tooLarge?.() ?? failed("answered with more than 8 MB"), + ) + } + return yield* decode(body).pipe( + Effect.mapError(() => failed("answered with an unexpected body")), + ) + }), + ), Effect.annotateSpans("peer.service", "google-cloud"), ) } @@ -241,6 +289,8 @@ const decodeImpersonatedToken = Schema.decodeUnknownEffect( Schema.Struct({ accessToken: Schema.RedactedFromValue(Schema.String) }), ) +const MAPLE_SIDE_REASONS = new Set(["SERVICE_DISABLED", "BILLING_DISABLED", "USER_PROJECT_DENIED"]) + /** A connector's reader account, signed in. Its project pays the quota for what it reads. */ export interface GcpReader { readonly accessToken: Redacted.Redacted @@ -279,8 +329,16 @@ export const impersonateReader = Effect.fnUntraced(function* ( ), decodeImpersonatedToken, ).pipe( - // Not made through the host project, so its reason says nothing about that project. - Effect.mapError(({ message, api, kind }) => new GcpApiError({ message, api, kind })), + // Not made through the host project, so its reason says nothing about that project: a + // refusal for Maple's own (API off, no billing, no quota grant) is Maple's to fix. + Effect.mapError( + ({ message, api, kind, reason }) => + new GcpApiError({ + message, + api, + kind: reason !== undefined && MAPLE_SIDE_REASONS.has(reason) ? "upstream" : kind, + }), + ), ) return { accessToken: response.accessToken, quotaProject: connector.projectId } satisfies GcpReader }) @@ -390,6 +448,232 @@ export const listTimeSeriesPage = ( decodeTimeSeriesPage, ) +// Cloud Monitoring PromQL + +/** Google's own limit is 120 s; this one answers before `REQUEST_TIMEOUT` gives up. */ +const PROMQL_TIMEOUT = "18s" + +const Labels = Schema.Record(Schema.String, Schema.String) +/** Prometheus sends values as strings: `"1.5"`, `"NaN"`, `"+Inf"`. */ +const Sample = Schema.Tuple([Schema.Number, Schema.String]) + +const decodePromQlResult = Schema.decodeUnknownEffect( + Schema.Struct({ + data: Schema.Union([ + Schema.Struct({ + resultType: Schema.Literal("matrix"), + result: Schema.Array(Schema.Struct({ metric: Labels, values: Schema.Array(Sample) })), + }), + Schema.Struct({ + resultType: Schema.Literal("vector"), + result: Schema.Array(Schema.Struct({ metric: Labels, value: Sample })), + }), + // `1 + 1`, `time()`: answered as one series without labels. + Schema.Struct({ resultType: Schema.Literal("scalar"), result: Sample }), + Schema.Struct({ resultType: Schema.Literal("string"), result: Schema.Unknown }), + ]), + }), +) + +const toPoints = (samples: ReadonlyArray): Array => + samples.flatMap(([time, raw]) => { + const value = Number(raw) + return Number.isFinite(value) ? [[time, value] as const] : [] + }) + +const decodePromQlError = Schema.decodeUnknownOption( + Schema.Struct({ status: Schema.Literal("error"), errorType: Schema.String, error: Schema.String }), +) + +/** + * Google's own error for a query: 400 `bad_data` refuses the query itself. 422 `execution` and + * 503 `timeout` (too many samples, out of time) are the project's alone and stay a + * `GcpApiError`, carrying Google's text. + */ +const refuseQuery = (status: number, body: unknown) => + Option.match(decodePromQlError(body), { + onNone: () => undefined, + onSome: ({ errorType, error }) => + status === 400 + ? new GcpLiveQueryRejectedError({ message: error.slice(0, 1_000), reason: "invalid_query" }) + : status === 422 || status === 503 + ? new GcpApiError({ + message: `Cloud Monitoring returned ${status} ${errorType}: ${error.slice(0, 500)}`, + api: "Cloud Monitoring", + kind: status === 422 ? "invalid" : "upstream", + reason: "QUERY_EXECUTION", + }) + : undefined, + }) + +export type GcpPromQlParams = + | { readonly query: string; readonly time: number } + | { readonly query: string; readonly start: number; readonly end: number; readonly step: number } + +/** + * One instant or range query against one project, times in epoch seconds. A scalar answers as + * one series without labels. + */ +export const promQlQuery = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, + params: GcpPromQlParams, +): Effect.Effect, GcpApiError | GcpLiveQueryRejectedError> => + call( + httpClient, + "Cloud Monitoring", + // A form body: a 4,000-character query would make a long URL. + HttpClientRequest.post( + `https://monitoring.googleapis.com/v1/projects/${projectId}/location/global/prometheus/api/v1/${"time" in params ? "query" : "query_range"}`, + ).pipe(asReader(reader), HttpClientRequest.bodyUrlParams({ ...params, timeout: PROMQL_TIMEOUT })), + decodePromQlResult, + { + refuse: refuseQuery, + capped: { + tooLarge: () => + new GcpLiveQueryRejectedError({ + message: `Google's answer for project ${projectId} is larger than 8 MB: too many series or points. Narrow the query or group by fewer labels.`, + reason: "too_many_series", + }), + }, + }, + ).pipe( + Effect.flatMap(({ data }) => + Effect.gen(function* () { + switch (data.resultType) { + case "matrix": + return data.result.map((series): GcpPromQlSeries => ({ + labels: series.metric, + points: toPoints(series.values), + })) + case "vector": + return data.result.map((series): GcpPromQlSeries => ({ + labels: series.metric, + points: toPoints([series.value]), + })) + case "scalar": + return [{ labels: {}, points: toPoints([data.result]) } satisfies GcpPromQlSeries] + case "string": + return yield* new GcpLiveQueryRejectedError({ + message: + "String results are not supported: query a vector, a range vector or a scalar.", + reason: "invalid_query", + }) + } + }), + ), + ) + +const decodeMetricsScope = Schema.decodeUnknownEffect( + Schema.Struct({ + name: Schema.String, + monitoredProjects: Schema.optionalKey(Schema.Array(Schema.Struct({ name: Schema.String }))), + }), +) + +const lastSegment = (name: string) => name.slice(name.lastIndexOf("/") + 1) + +/** + * The metrics scope a query on the project reads: its own project number, and the numbers of + * the projects it monitors, itself included. Needs `resourcemanager.projects.get`, which + * Monitoring Viewer holds. + */ +export const getMetricsScope = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get( + `https://monitoring.googleapis.com/v1/locations/global/metricsScopes/${projectId}`, + ).pipe(asReader(reader)), + decodeMetricsScope, + { capped: {} }, + ).pipe( + Effect.map((scope) => ({ + number: lastSegment(scope.name), + monitored: (scope.monitoredProjects ?? []).map((project) => lastSegment(project.name)), + })), + ) + +/** A `GcpMetricDescriptor`, as the live reader caches it. */ +export const ActiveMetricDescriptor = Schema.Struct({ + type: Schema.String, + metricKind: Schema.Literals(["GAUGE", "DELTA", "CUMULATIVE"]), + valueType: Schema.Literals(["BOOL", "INT64", "DOUBLE", "STRING", "DISTRIBUTION", "MONEY"]), + unit: Schema.String, + description: Schema.String, + monitoredResourceTypes: Schema.Array(Schema.String), + labels: Schema.Array(Schema.String), +}) +// A kind or value type Google leaves unspecified is neither. +const isMetricKind = Schema.is(ActiveMetricDescriptor.fields.metricKind) +const isValueType = Schema.is(ActiveMetricDescriptor.fields.valueType) + +const decodeMetricDescriptorsPage = Schema.decodeUnknownEffect( + Schema.Struct({ + metricDescriptors: Schema.optionalKey( + Schema.Array( + Schema.Struct({ + type: Schema.String, + metricKind: Schema.optionalKey(Schema.String), + valueType: Schema.optionalKey(Schema.String), + unit: Schema.optionalKey(Schema.String), + description: Schema.optionalKey(Schema.String), + monitoredResourceTypes: Schema.optionalKey(Schema.Array(Schema.String)), + labels: Schema.optionalKey(Schema.Array(Schema.Struct({ key: Schema.String }))), + }), + ), + ), + nextPageToken: Schema.optionalKey(Schema.String), + }), +) + +/** + * One page of the metrics with data in the project's last ~25 hours. A descriptor of a kind or + * value type Google leaves unspecified is dropped. + */ +export const listActiveMetricDescriptorsPage = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, + pageToken: string | undefined, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get( + `https://monitoring.googleapis.com/v3/projects/${projectId}/metricDescriptors`, + ).pipe( + asReader(reader), + HttpClientRequest.setUrlParams({ activeOnly: true, pageSize: 10_000, pageToken }), + ), + decodeMetricDescriptorsPage, + { capped: {} }, + ).pipe( + Effect.map((page) => ({ + descriptors: (page.metricDescriptors ?? []).flatMap((descriptor): Array => { + const { metricKind = "", valueType = "" } = descriptor + if (!isMetricKind(metricKind) || !isValueType(valueType)) return [] + return [ + { + type: descriptor.type, + metricKind, + valueType, + unit: descriptor.unit ?? "", + description: descriptor.description ?? "", + monitoredResourceTypes: descriptor.monitoredResourceTypes ?? [], + labels: (descriptor.labels ?? []).map((label) => label.key), + }, + ] + }), + nextPageToken: page.nextPageToken, + })), + ) + // Cloud Asset Inventory const ResourceSearchResult = Schema.Struct({ @@ -414,11 +698,13 @@ const decodeResourcePage = Schema.decodeUnknownEffect( }), ) -export const searchResourcesPage = ( +const resourcePage = ( httpClient: HttpClient.HttpClient, reader: GcpReader, scope: string, pageToken: string | undefined, + assetTypes: ReadonlyArray, + options: CallOptions = {}, ) => call( httpClient, @@ -426,15 +712,32 @@ export const searchResourcesPage = ( HttpClientRequest.get(`https://cloudasset.googleapis.com/v1/${scope}:searchAllResources`).pipe( asReader(reader), HttpClientRequest.setUrlParams({ - assetTypes: GCP_ASSET_TYPES, + assetTypes, readMask: Object.keys(ResourceSearchResult.fields).join(","), pageSize: RESOURCE_PAGE_SIZE, pageToken, }), ), decodeResourcePage, + options, ) +/** One page of the resource inventory sync. */ +export const searchResourcesPage = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + scope: string, + pageToken: string | undefined, +) => resourcePage(httpClient, reader, scope, pageToken, GCP_ASSET_TYPES) + +/** One page of the projects in a folder or organization, for a live query. */ +export const searchProjectsPage = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + scope: string, + pageToken: string | undefined, +) => resourcePage(httpClient, reader, scope, pageToken, [GCP_PROJECT_ASSET_TYPE], { capped: {} }) + /** What Maple reads of a Cloud SQL instance: the Cloud SQL Admin API's `DatabaseInstance`. */ const SqlInstanceData = Schema.Struct({ name: Schema.String, diff --git a/packages/backend/src/services/integrations/gcp/explain.ts b/packages/backend/src/services/integrations/gcp/explain.ts new file mode 100644 index 0000000000..f05ed014b5 --- /dev/null +++ b/packages/backend/src/services/integrations/gcp/explain.ts @@ -0,0 +1,65 @@ +import type { GcpConnectorRow } from "@maple/db/tables" +import type { GcpApiError } from "./api" + +export const RETRIES = "Maple retries in 5 minutes." + +type ExplainedConnector = Pick< + GcpConnectorRow, + "scopeType" | "scopeId" | "projectId" | "lastMetricsReceivedAt" +> + +/** Whose call failed: the poller's, or a live query's that a person is waiting on and nothing retries. */ +export type GcpCaller = "poll" | "live" + +/** Why a failed call failed and what to do about it, in the customer's terms. */ +const explainGcpApiError = (error: GcpApiError, connector: ExplainedConnector, caller: GcpCaller) => { + // The host project is the one Google checks for billing, enabled APIs and quota. A project + // scope is its own host, and reads as "the project". + const host = `${connector.scopeType === "project" ? "project" : "host project"} ${connector.projectId}` + const scope = `${connector.scopeType} ${connector.scopeId}` + const live = caller === "live" + const retries = live ? "Try again in a few minutes." : RETRIES + if (error.reason === "BILLING_DISABLED") { + return `The ${host} has no active billing account, and ${error.api} only answers for projects that have one. Link one: https://console.cloud.google.com/billing/linkedaccount?project=${connector.projectId}${live ? "" : " Maple retries every 5 minutes."}` + } + // A PromQL query that ran out of samples or time; Google's text, in the status, says which. + if (error.reason === "QUERY_EXECUTION") { + return "Google could not finish the query on this project. Narrow the query or widen the step." + } + if (error.reason === "SERVICE_DISABLED") { + return `The ${error.api} API is switched off in the ${host}. Switch it on in the console, or run the setup script again.` + } + if (error.kind === "denied" || error.kind === "not_found") { + if (error.api === "IAM Credentials") { + if (live) { + return "Maple can't sign in as this connection's read-only service account, which live queries read with. Google needs a few minutes to accept a new grant. If this stays, the account does not exist, or the grant to Maple is missing or blocked by an organization policy: finish the read-only account steps of a manual setup, or run the setup script again." + } + // A connection that has been read before lost its account or its grant. + if (connector.lastMetricsReceivedAt !== null) { + return "Maple can no longer sign in as this connection's read-only service account. The account was deleted, or the grant to Maple was removed or is blocked by an organization policy. Run the setup script again or redo the read-only steps: either restores both." + } + // Shown both before any setup and for the minutes after one in which Google does not + // accept the new grant yet, so it has to read right in both. + return "Maple can't sign in as this connection's read-only service account yet. Google needs a few minutes to accept a new grant, and Maple retries every 5 minutes. If this stays, the account does not exist, or the grant to Maple is missing or blocked by an organization policy: finish the read-only account steps of a manual setup, or run the setup script again." + } + return error.api === "Cloud Asset Inventory" + ? `Google denied the resource listing for ${scope}. Cloud Asset Viewer is missing. Grant it with the setup script, the console steps or your Terraform.` + : `Google denied Maple's read of ${scope}: the read-only roles are missing. Grant them with the setup script, the console steps or your Terraform. A new grant can take a few minutes.` + } + // Signing in is counted against Maple's own project, not the host project. + if (error.kind === "rate_limited" && error.api !== "IAM Credentials") { + // Google answers a query past its size limit the same way as a spent quota. + if (live) { + return `Google refused the read for the ${host}: a rate limit, or an answer past Google's size limit. Narrow the query, or try again in a few minutes.` + } + return `Google rate-limited the ${error.api} API for the ${host}. ${RETRIES} If this repeats, raise that API's quota on the project.` + } + return `Maple's request to Google's ${error.api} API failed. ${retries} If this repeats, write to support@maple.dev.` +} + +/** + * What the customer sees for a failed call: one sentence of cause, one of what to do, and + * Google's status at the end. + */ +export const describeGcpApiError = (error: GcpApiError, connector: ExplainedConnector, caller: GcpCaller) => + `${explainGcpApiError(error, connector, caller)} (${error.message})` diff --git a/packages/backend/src/services/warehouse/GcpLiveMetricsPort.ts b/packages/backend/src/services/warehouse/GcpLiveMetricsPort.ts new file mode 100644 index 0000000000..c11a8b4f82 --- /dev/null +++ b/packages/backend/src/services/warehouse/GcpLiveMetricsPort.ts @@ -0,0 +1,10 @@ +import { Context } from "effect" +import type { GcpLiveMetrics, QueryTenant } from "@maple/query-engine/runtime" + +/** + * Optional port `QueryEngineService` reads Cloud Monitoring metrics through. Deployments without + * a Google identity leave it unprovided, and those metrics then fail as unavailable. + */ +export class GcpLiveMetricsPort extends Context.Service>()( + "@maple/backend/services/warehouse/GcpLiveMetricsPort", +) {} diff --git a/packages/backend/src/services/warehouse/QueryEngineService.ts b/packages/backend/src/services/warehouse/QueryEngineService.ts index a51e13056b..292bcaa8e3 100644 --- a/packages/backend/src/services/warehouse/QueryEngineService.ts +++ b/packages/backend/src/services/warehouse/QueryEngineService.ts @@ -1,5 +1,6 @@ // BOUNDARY: This module owns unparsed external values and narrows them before domain use. -import { Clock, Context, Effect, Layer, Metric, type Schema } from "effect" +import { Clock, Context, Effect, Layer, Metric, Option, type Schema } from "effect" +import { isGcpMetricType } from "@maple/domain/gcp-live-metrics" import { QueryEngineExecuteResponse, type QueryEngineExecuteRequest } from "@maple/query-engine" import type { QueryEngineTimeoutError } from "@maple/domain/http" import { @@ -41,6 +42,7 @@ import { import { EdgeCacheService } from "@maple/cache" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import * as QueryEngineMetrics from "@maple/backend/observability/QueryEngineMetrics" +import { GcpLiveMetricsPort } from "@maple/backend/services/warehouse/GcpLiveMetricsPort" // QueryEngineService — caching + orchestration. The pure lowering (validation, // QuerySpec → CH, evaluate/raw-SQL) lives in `@maple/query-engine/runtime`; this @@ -97,9 +99,10 @@ export class QueryEngineService extends Context.Service = [ "@maple/http/errors/ErrorIssuePullRequestNotFoundError", "@maple/http/errors/ErrorIssueTransitionError", "@maple/http/errors/ErrorValidationError", + "@maple/http/errors/GcpLiveQueryRejectedError", + "@maple/http/errors/GcpLiveQueryUnavailableError", "@maple/http/errors/GcpMetricsUnavailableError", "@maple/http/errors/GcpScopeAlreadyConnectedError", "@maple/http/errors/IncidentTriageUnauthorizedError", diff --git a/packages/domain/src/http/warehouse-errors.ts b/packages/domain/src/http/warehouse-errors.ts index 534b25e84e..de3ca70bef 100644 --- a/packages/domain/src/http/warehouse-errors.ts +++ b/packages/domain/src/http/warehouse-errors.ts @@ -1,5 +1,6 @@ import { Schema } from "effect" import { HttpTaggedError, publicHttpErrorDefinitionFor } from "./error-policy" +import { gcpLiveQueryHttpErrors } from "./gcp-live-query-errors" import { OrgClickHouseSettingsEncryptionError, OrgClickHouseSettingsPersistenceError, @@ -330,10 +331,14 @@ export const warehouseSettingsRouteHttpErrors = [ /** Errors unique to minting an org-scoped token for user-authored raw SQL. */ export const warehouseTokenRouteHttpErrors = [TinybirdOrgTokenConfigError, TinybirdOrgTokenMintError] as const -/** Normal reads resolve saved settings but never mint a raw-SQL token. */ +/** + * Normal reads resolve saved settings but never mint a raw-SQL token. A metrics read on a + * Cloud Monitoring metric type is answered live from Google, so its failures ride here too. + */ export const warehouseReadHttpErrors = [ ...managedWarehouseHttpErrors, ...warehouseSettingsRouteHttpErrors, + ...gcpLiveQueryHttpErrors, ] as const /** Failures that can escape compiled or raw execution into public v2 query routes. */ diff --git a/packages/query-engine/src/promql/compile.test.ts b/packages/query-engine/src/promql/compile.test.ts new file mode 100644 index 0000000000..5b0ce642e9 --- /dev/null +++ b/packages/query-engine/src/promql/compile.test.ts @@ -0,0 +1,340 @@ +import { assert, describe, it } from "@effect/vitest" +import { DeploymentEnvironment, MetricName, ServiceName } from "@maple/domain" +import type { GcpMetricDescriptor } from "@maple/domain/gcp-live-metrics" +import type { AttributeFilter, MetricsFilters, MetricsMetric } from "@maple/domain/query-engine" +import { Effect } from "effect" +import { + compileGcpLabelSample, + compileGcpLabelValues, + compileGcpMetricQuery, + gcpPointBucket, + gcpRangeWindow, +} from "./compile" + +const descriptor = (overrides: Partial & Pick) => + ({ + metricKind: "GAUGE", + valueType: "DOUBLE", + unit: "1", + description: "", + monitoredResourceTypes: ["cloudsql_database"], + labels: [], + ...overrides, + }) satisfies GcpMetricDescriptor + +const cpu = descriptor({ type: "cloudsql.googleapis.com/database/cpu/utilization" }) +const transactions = descriptor({ + type: "cloudsql.googleapis.com/database/postgresql/transaction_count", + metricKind: "DELTA", + valueType: "INT64", +}) +const coreUsage = descriptor({ + type: "kubernetes.io/container/cpu/core_usage_time", + metricKind: "CUMULATIVE", + monitoredResourceTypes: ["k8s_container"], +}) +const latencies = descriptor({ + type: "serviceruntime.googleapis.com/api/request_latencies", + metricKind: "DELTA", + valueType: "DISTRIBUTION", + monitoredResourceTypes: ["consumed_api"], +}) + +interface CompileOptions { + readonly filters?: Partial + readonly groupLabel?: string + readonly windowSeconds?: number +} + +const request = (d: GcpMetricDescriptor, metric: MetricsMetric, options: CompileOptions = {}) => ({ + metric, + filters: { metricName: MetricName.make(d.type), metricType: "gauge" as const, ...options.filters }, + groupLabel: options.groupLabel, + windowSeconds: options.windowSeconds ?? 60, +}) + +const compile = (d: GcpMetricDescriptor, metric: MetricsMetric, options?: CompileOptions) => + Effect.runSync(compileGcpMetricQuery(request(d, metric, options), d)) + +const query = (d: GcpMetricDescriptor, metric: MetricsMetric, options?: CompileOptions) => + compile(d, metric, options).query + +const rejection = (d: GcpMetricDescriptor, metric: MetricsMetric, options?: CompileOptions) => + Effect.runSync(Effect.flip(compileGcpMetricQuery(request(d, metric, options), d))).details.join(" ") + +const withFilters = (attributeFilters: ReadonlyArray) => ({ filters: { attributeFilters } }) + +const CPU = `{"cloudsql.googleapis.com/database/cpu/utilization", monitored_resource="cloudsql_database"}` +const TX = `{"cloudsql.googleapis.com/database/postgresql/transaction_count", monitored_resource="cloudsql_database"}` +const CORE = `{"kubernetes.io/container/cpu/core_usage_time", monitored_resource="k8s_container"}` +const LAT = (suffix: string) => + `{"serviceruntime.googleapis.com/api/request_latencies_${suffix}", monitored_resource="consumed_api"}` + +describe("compileGcpMetricQuery", () => { + describe("GAUGE", () => { + for (const [metric, expected] of [ + ["avg", `avg (avg_over_time(${CPU}[60s]))`], + ["min", `min (min_over_time(${CPU}[60s]))`], + ["max", `max (max_over_time(${CPU}[60s]))`], + ["sum", `sum (sum_over_time(${CPU}[60s]))`], + ["count", `sum (count_over_time(${CPU}[60s]))`], + ["rate", `sum (rate(${CPU}[60s]))`], + ["increase", `sum (increase(${CPU}[60s]))`], + ] as const) { + it(metric, () => assert.strictEqual(query(cpu, metric), expected)) + } + }) + + describe("DELTA", () => { + for (const [metric, expected] of [ + // Summing the deltas is exact; `increase` would extrapolate. + ["sum", `sum (sum_over_time(${TX}[300s]))`], + ["increase", `sum (sum_over_time(${TX}[300s]))`], + ["rate", `sum (sum_over_time(${TX}[300s])) / 300`], + ["count", `sum (count_over_time(${TX}[300s]))`], + ["avg", `avg (avg_over_time(${TX}[300s]))`], + ["min", `min (min_over_time(${TX}[300s]))`], + ["max", `max (max_over_time(${TX}[300s]))`], + ] as const) { + it(metric, () => + assert.strictEqual(query(transactions, metric, { windowSeconds: 300 }), expected), + ) + } + }) + + describe("CUMULATIVE", () => { + for (const [metric, expected] of [ + ["rate", `sum (rate(${CORE}[60s]))`], + ["increase", `sum (increase(${CORE}[60s]))`], + ["sum", `sum (increase(${CORE}[60s]))`], + ["count", `sum (count_over_time(${CORE}[60s]))`], + ["avg", `avg (avg_over_time(${CORE}[60s]))`], + ["max", `max (max_over_time(${CORE}[60s]))`], + ] as const) { + it(metric, () => assert.strictEqual(query(coreUsage, metric), expected)) + } + }) + + describe("DISTRIBUTION", () => { + it("averages as sum over count per group", () => + assert.strictEqual( + query(latencies, "avg", { groupLabel: "method" }), + `sum by (method) (sum_over_time(${LAT("sum")}[60s])) / sum by (method) (sum_over_time(${LAT("count")}[60s]))`, + )) + it("sums the _sum series", () => + assert.strictEqual(query(latencies, "sum"), `sum (sum_over_time(${LAT("sum")}[60s]))`)) + it("counts the _count series", () => + assert.strictEqual(query(latencies, "count"), `sum (sum_over_time(${LAT("count")}[60s]))`)) + it("rates a delta _count per second of the window", () => + assert.strictEqual(query(latencies, "rate"), `sum (sum_over_time(${LAT("count")}[60s])) / 60`)) + it("rates a cumulative _count with rate and sums it with increase", () => { + const cumulative = { ...latencies, metricKind: "CUMULATIVE" as const } + assert.strictEqual(query(cumulative, "rate"), `sum (rate(${LAT("count")}[60s]))`) + assert.strictEqual(query(cumulative, "sum"), `sum (increase(${LAT("sum")}[60s]))`) + }) + for (const metric of ["min", "max"] as const) { + it(`refuses ${metric}`, () => + assert.include(rejection(latencies, metric), `${metric} is not available`)) + } + }) + + it("groups by the flat label, quoting names PromQL cannot spell bare", () => { + assert.strictEqual( + query(transactions, "sum", { groupLabel: "database_id" }), + `sum by (database_id) (sum_over_time(${TX}[60s]))`, + ) + assert.strictEqual( + query(cpu, "max", { groupLabel: "k8s.pod.name" }), + `max by ("k8s.pod.name") (max_over_time(${CPU}[60s]))`, + ) + }) + + it("refuses values PromQL cannot chart", () => + assert.include(rejection({ ...cpu, valueType: "STRING" }, "avg"), "cannot be charted")) + + it("refuses service.name and deployment.environment filters", () => { + assert.include( + rejection(cpu, "avg", { filters: { serviceName: ServiceName.make("api") } }), + "no service.name", + ) + assert.include( + rejection(cpu, "avg", { filters: { environments: [DeploymentEnvironment.make("production")] } }), + "no deployment.environment", + ) + }) + + describe("monitored_resource", () => { + const entries = descriptor({ + type: "logging.googleapis.com/log_entry_count", + metricKind: "DELTA", + valueType: "INT64", + monitoredResourceTypes: ["gce_instance", "k8s_container"], + }) + const entriesIn = (type: string) => + `sum_over_time({"logging.googleapis.com/log_entry_count", monitored_resource="${type}"}[60s])` + + it("unions one exact selector per listed resource type inside the aggregation", () => + assert.strictEqual( + query(entries, "sum", { groupLabel: "severity" }), + `sum by (severity) (${entriesIn("gce_instance")} or ${entriesIn("k8s_container")})`, + )) + + it("uses a monitored_resource equals filter instead of the listed types", () => + assert.strictEqual( + query(entries, "sum", { + filters: { + resourceAttributeFilters: [ + { key: "monitored_resource", mode: "equals", value: "k8s_container" }, + ], + }, + }), + `sum (${entriesIn("k8s_container")})`, + )) + + it("asks for a monitored_resource filter when Google lists no type", () => { + const unlisted = { ...entries, monitoredResourceTypes: [] } + assert.include(rejection(unlisted, "sum"), "filter on monitored_resource") + assert.strictEqual( + query(unlisted, "sum", { + filters: { + resourceAttributeFilters: [ + { key: "monitored_resource", mode: "equals", value: "gce_instance" }, + ], + }, + }), + `sum (${entriesIn("gce_instance")})`, + ) + }) + }) + + describe("filters", () => { + const matchers = (filters: ReadonlyArray) => + query(cpu, "avg", withFilters(filters)).slice( + `avg (avg_over_time({"cloudsql.googleapis.com/database/cpu/utilization", monitored_resource="cloudsql_database", ` + .length, + -"}[60s]))".length, + ) + + it("lowers equals, in, contains and exists", () => { + assert.strictEqual( + matchers([{ key: "database_id", mode: "equals", value: "p:db" }]), + `database_id="p:db"`, + ) + assert.strictEqual( + matchers([{ key: "database_id", mode: "equals", value: "p:db", negated: true }]), + `database_id!="p:db"`, + ) + assert.strictEqual( + matchers([{ key: "region", mode: "in", values: ["us-east1", "eu.west"] }]), + `region=~"(?:us-east1|eu\\\\.west)"`, + ) + assert.strictEqual( + matchers([{ key: "region", mode: "in", values: ["a"], negated: true }]), + `region!~"(?:a)"`, + ) + assert.strictEqual( + matchers([{ key: "region", mode: "contains", value: "us" }]), + `region=~".*us.*"`, + ) + assert.strictEqual(matchers([{ key: "region", mode: "exists" }]), `region=~".+"`) + assert.strictEqual(matchers([{ key: "region", mode: "exists", negated: true }]), `region=""`) + }) + + it("escapes string values and regex metacharacters", () => { + assert.strictEqual( + matchers([{ key: "query", mode: "equals", value: 'say "hi"\\\nbye' }]), + `query="say \\"hi\\"\\\\\\nbye"`, + ) + assert.strictEqual( + matchers([{ key: "query", mode: "contains", value: "a+b(c)" }]), + `query=~".*a\\\\+b\\\\(c\\\\).*"`, + ) + assert.strictEqual( + matchers([{ key: "cloud.region", mode: "equals", value: "x" }]), + `"cloud.region"="x"`, + ) + }) + + it("folds an OR group of equals/in on one label into one regex", () => + assert.strictEqual( + matchers([ + { + key: "state", + mode: "equals", + value: "idle", + or: [{ key: "state", mode: "in", values: ["active", "waiting"] }], + }, + ]), + `state=~"(?:idle|active|waiting)"`, + )) + + it("refuses OR groups across labels and numeric comparisons", () => { + assert.include( + rejection( + cpu, + "avg", + withFilters([ + { + key: "a", + mode: "equals", + value: "1", + or: [{ key: "b", mode: "equals", value: "2" }], + }, + ]), + ), + "OR filters", + ) + assert.include( + rejection(cpu, "avg", withFilters([{ key: "a", mode: "gt", value: "1" }])), + "Google Cloud labels are strings", + ) + }) + + it("narrows the project fan-out on a project_id filter and keeps the matcher", () => { + const compiled = compile( + cpu, + "avg", + withFilters([{ key: "project_id", mode: "in", values: ["alpha", "beta"] }]), + ) + assert.deepStrictEqual(compiled.projects, ["alpha", "beta"]) + assert.include(compiled.query, `project_id=~"(?:alpha|beta)"`) + assert.isUndefined(compile(cpu, "avg").projects) + }) + }) +}) + +describe("discovery", () => { + it("samples recent series of a distribution through its _count series", () => + assert.strictEqual( + Effect.runSync(compileGcpLabelSample(latencies)), + `topk(5, last_over_time(${LAT("count")}[1h]))`, + )) + + it("counts series per label value", () => + assert.strictEqual( + Effect.runSync(compileGcpLabelValues(cpu, "database_id", 86400)), + `count by (database_id) (last_over_time(${CPU}[86400s]))`, + )) +}) + +describe("bucket alignment", () => { + const at = (iso: string) => Date.parse(iso) + + it("evaluates from the first bucket's end to the last bucket's end", () => { + const window = gcpRangeWindow(at("2026-10-19T10:02:30Z"), at("2026-10-19T10:30:10Z"), 300) + assert.deepStrictEqual(window, { + stepSeconds: 300, + startSeconds: at("2026-10-19T10:05:00Z") / 1000, + endSeconds: at("2026-10-19T10:35:00Z") / 1000, + }) + }) + + it("labels a point by the bucket it closes", () => + assert.strictEqual( + gcpPointBucket(at("2026-10-19T10:05:00Z") / 1000, 300), + "2026-10-19T10:00:00.000Z", + )) + + it("never steps below Google's one-minute sample period", () => + assert.strictEqual(gcpRangeWindow(0, 600_000, 15).stepSeconds, 60)) +}) diff --git a/packages/query-engine/src/promql/compile.ts b/packages/query-engine/src/promql/compile.ts new file mode 100644 index 0000000000..3a2d4c9818 --- /dev/null +++ b/packages/query-engine/src/promql/compile.ts @@ -0,0 +1,273 @@ +// Metrics spec → PromQL for Google Cloud Monitoring's Prometheus-compatible API. +// +// Google flattens metric and resource labels onto one label set, so `attr.X` and `resource.X` +// both read the label `X`. A metric written for several monitored resource types only answers a +// selector with an exact `monitored_resource` matcher, so each type gets its own range function +// and the outer aggregation unions them with `or`. + +import { DateTime, Effect } from "effect" +import type { GcpMetricDescriptor, GcpMetricKind } from "@maple/domain/gcp-live-metrics" +import { gcpMetricMapleType } from "@maple/domain/gcp-live-metrics" +import { QueryEngineValidationError } from "@maple/domain/http" +import type { + AttributeFilter, + AttributeFilterLeaf, + MetricsFilters, + MetricsMetric, +} from "@maple/domain/query-engine" + +const reject = (detail: string) => + new QueryEngineValidationError({ message: "Unsupported Google Cloud metrics query", details: [detail] }) + +/** A double-quoted PromQL string literal. */ +export const promQlString = (value: string): string => + `"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n")}"` + +const PLAIN_LABEL = /^[a-zA-Z_][a-zA-Z0-9_]*$/ +const labelName = (name: string): string => (PLAIN_LABEL.test(name) ? name : promQlString(name)) + +const escapeRegex = (value: string): string => value.replace(/[\\.+*?()|[\]{}^$]/g, "\\$&") +const anyOf = (values: ReadonlyArray): string => `(?:${values.map(escapeRegex).join("|")})` + +interface Matcher { + readonly label: string + readonly op: "=" | "!=" | "=~" | "!~" + readonly value: string +} + +const renderSelector = (metricName: string, matchers: ReadonlyArray): string => + `{${[promQlString(metricName), ...matchers.map((m) => `${labelName(m.label)}${m.op}${promQlString(m.value)}`)].join(", ")}}` + +/** The values of a plain positive `equals` / `in` filter, else undefined. */ +const positiveValues = (filter: AttributeFilterLeaf): ReadonlyArray | undefined => { + if (filter.negated) return undefined + if (filter.mode === "equals") return [filter.value ?? ""] + if (filter.mode === "in") return filter.values ?? [] + return undefined +} + +const leafMatcher = (filter: AttributeFilterLeaf): Effect.Effect => { + const label = filter.key + const negated = filter.negated === true + switch (filter.mode) { + case "equals": + return Effect.succeed({ label, op: negated ? "!=" : "=", value: filter.value ?? "" }) + case "in": + return Effect.succeed({ label, op: negated ? "!~" : "=~", value: anyOf(filter.values ?? []) }) + case "contains": + return Effect.succeed({ + label, + op: negated ? "!~" : "=~", + value: `.*${escapeRegex(filter.value ?? "")}.*`, + }) + case "exists": + return Effect.succeed(negated ? { label, op: "=", value: "" } : { label, op: "=~", value: ".+" }) + default: + return Effect.fail( + reject( + `Label ${label} cannot be compared with ${filter.mode}: Google Cloud labels are strings`, + ), + ) + } +} + +const filterMatcher = (filter: AttributeFilter): Effect.Effect => { + if (!filter.or?.length) return leafMatcher(filter) + const alternatives = [filter, ...filter.or] + const values = alternatives.map((alternative) => + alternative.key === filter.key ? positiveValues(alternative) : undefined, + ) + if (values.some((v) => v === undefined)) { + return Effect.fail( + reject(`OR filters on Google Cloud metrics must all be equals/in on one label (${filter.key})`), + ) + } + return Effect.succeed({ label: filter.key, op: "=~", value: anyOf(values.flatMap((v) => v ?? [])) }) +} + +interface Scope { + /** One selector per monitored resource type, for the given series name. */ + readonly selectors: (metricName: string) => ReadonlyArray + readonly projects?: ReadonlyArray +} + +const compileScope = Effect.fnUntraced(function* ( + descriptor: GcpMetricDescriptor, + filters: ReadonlyArray, +) { + const matchers: Matcher[] = [] + let resourceTypes: ReadonlyArray | undefined + let projects: ReadonlyArray | undefined + for (const filter of filters) { + const values = filter.or?.length ? undefined : positiveValues(filter) + if (filter.key === "monitored_resource" && values) { + resourceTypes = values + continue + } + if (filter.key === "project_id" && values) projects = values + matchers.push(yield* filterMatcher(filter)) + } + const types = resourceTypes ?? descriptor.monitoredResourceTypes + if (types.length === 0) { + return yield* reject( + `Google Cloud lists no monitored resource for ${descriptor.type}; filter on monitored_resource (e.g. resource.monitored_resource = cloudsql_database)`, + ) + } + return { + selectors: (metricName) => + types.map((type) => + renderSelector(metricName, [ + { label: "monitored_resource", op: "=", value: type }, + ...matchers, + ]), + ), + ...(projects === undefined ? undefined : { projects }), + } satisfies Scope +}) + +const rangeFunction = (kind: GcpMetricKind, metric: MetricsMetric): string => { + switch (metric) { + case "avg": + case "min": + case "max": + return `${metric}_over_time` + case "count": + return "count_over_time" + case "sum": + return kind === "CUMULATIVE" ? "increase" : "sum_over_time" + case "increase": + return kind === "DELTA" ? "sum_over_time" : "increase" + case "rate": + return kind === "DELTA" ? "sum_over_time" : "rate" + } +} + +const outerAggregation = (metric: MetricsMetric): string => + metric === "avg" || metric === "min" || metric === "max" ? metric : "sum" + +export interface GcpPromQlRequest { + readonly metric: MetricsMetric + readonly filters: MetricsFilters + /** Flat label the result is grouped by; undefined is one ungrouped series. */ + readonly groupLabel: string | undefined + /** Range-function window, in seconds. */ + readonly windowSeconds: number +} + +export interface GcpPromQl { + readonly query: string + /** Projects a `project_id` filter narrows the fan-out to. */ + readonly projects?: ReadonlyArray +} + +/** One value per group per evaluation step, matching the ClickHouse metrics semantics. */ +export const compileGcpMetricQuery = Effect.fnUntraced(function* ( + request: GcpPromQlRequest, + descriptor: GcpMetricDescriptor, +) { + const { filters, metric } = request + if (filters.serviceName !== undefined) { + return yield* reject( + "Google Cloud metrics have no service.name; filter on a label such as service_name or project_id", + ) + } + if (filters.environments?.length) { + return yield* reject( + "Google Cloud metrics have no deployment.environment; filter on a label such as project_id", + ) + } + if (gcpMetricMapleType(descriptor) === undefined) { + return yield* reject(`${descriptor.type} has ${descriptor.valueType} values, which cannot be charted`) + } + + const scope = yield* compileScope(descriptor, [ + ...(filters.attributeFilters ?? []), + ...(filters.resourceAttributeFilters ?? []), + ]) + const w = request.windowSeconds + const by = request.groupLabel === undefined ? "" : ` by (${labelName(request.groupLabel)})` + const aggregate = (aggregation: string, metricName: string, fn: string) => + `${aggregation}${by} (${scope + .selectors(metricName) + .map((selector) => `${fn}(${selector}[${w}s])`) + .join(" or ")})` + + const type = descriptor.type + const compiled = (query: string): GcpPromQl => + scope.projects === undefined ? { query } : { query, projects: scope.projects } + + if (descriptor.valueType === "DISTRIBUTION") { + // Distributions answer as `_count` / `_sum` counters; the bare name returns nothing. + const counter = descriptor.metricKind === "CUMULATIVE" ? "increase" : "sum_over_time" + switch (metric) { + case "min": + case "max": + return yield* reject( + `${metric} is not available for the distribution ${type}; use avg, sum, count or rate`, + ) + case "avg": + return compiled( + `${aggregate("sum", `${type}_sum`, counter)} / ${aggregate("sum", `${type}_count`, counter)}`, + ) + case "sum": + return compiled(aggregate("sum", `${type}_sum`, counter)) + case "count": + case "increase": + return compiled(aggregate("sum", `${type}_count`, counter)) + case "rate": + return compiled( + descriptor.metricKind === "CUMULATIVE" + ? aggregate("sum", `${type}_count`, "rate") + : `${aggregate("sum", `${type}_count`, counter)} / ${w}`, + ) + } + } + + const expression = aggregate(outerAggregation(metric), type, rangeFunction(descriptor.metricKind, metric)) + // A delta counter's samples are already increments: sum them (exact, unlike `increase`). + return compiled( + descriptor.metricKind === "DELTA" && metric === "rate" ? `${expression} / ${w}` : expression, + ) +}) + +const discoverySeries = (descriptor: GcpMetricDescriptor): string => + descriptor.valueType === "DISTRIBUTION" ? `${descriptor.type}_count` : descriptor.type + +/** A few recent series of the metric, to read which labels it carries. */ +export const compileGcpLabelSample = Effect.fnUntraced(function* (descriptor: GcpMetricDescriptor) { + const scope = yield* compileScope(descriptor, []) + return `topk(5, ${scope + .selectors(discoverySeries(descriptor)) + .map((selector) => `last_over_time(${selector}[1h])`) + .join(" or ")})` +}) + +/** Series per value of one label over the window. */ +export const compileGcpLabelValues = Effect.fnUntraced(function* ( + descriptor: GcpMetricDescriptor, + label: string, + windowSeconds: number, +) { + const scope = yield* compileScope(descriptor, []) + return `count by (${labelName(label)}) (${scope + .selectors(discoverySeries(descriptor)) + .map((selector) => `last_over_time(${selector}[${windowSeconds}s])`) + .join(" or ")})` +}) + +/** + * Range-query bounds for a bucketed timeseries. Google evaluates a point at `t` over + * `(t - step, t]` while Maple labels a bucket by its start, so the query runs from the first + * bucket's end to the last bucket's end and a point at `t` belongs to the bucket at `t - step`. + * The last bucket's end may lie in the future; Google answers it with what has arrived so far. + */ +export const gcpRangeWindow = (startMs: number, endMs: number, bucketSeconds: number) => { + const stepSeconds = Math.max(60, bucketSeconds) + const firstBucket = Math.floor(startMs / 1000 / stepSeconds) * stepSeconds + const lastBucket = Math.floor(endMs / 1000 / stepSeconds) * stepSeconds + return { stepSeconds, startSeconds: firstBucket + stepSeconds, endSeconds: lastBucket + stepSeconds } +} + +/** The ISO bucket a range-query point at `pointSeconds` belongs to. */ +export const gcpPointBucket = (pointSeconds: number, stepSeconds: number): string => + DateTime.formatIso(DateTime.makeUnsafe((pointSeconds - stepSeconds) * 1000)) diff --git a/packages/query-engine/src/runtime/gcp-metrics.test.ts b/packages/query-engine/src/runtime/gcp-metrics.test.ts new file mode 100644 index 0000000000..5c52a87b83 --- /dev/null +++ b/packages/query-engine/src/runtime/gcp-metrics.test.ts @@ -0,0 +1,407 @@ +import { assert, describe, it } from "@effect/vitest" +import { MetricName, OrgId } from "@maple/domain" +import type { GcpMetricDescriptor, GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import { QueryEngineExecuteRequest, type QueryEngineResult, type QuerySpec } from "@maple/domain/query-engine" +import { Effect } from "effect" +import type { GcpLiveMetrics } from "./live-metrics" +import { + makeQueryEngineEvaluate, + makeQueryEngineEvaluateSeries, + makeQueryEngineExecute, + type AlertEvaluateRequest, + type QueryEngineWarehouse, +} from "./query-engine" + +const tenant = { orgId: OrgId.make("org_gcp_live_test") } + +const TX = "cloudsql.googleapis.com/database/postgresql/transaction_count" +const transactions: GcpMetricDescriptor = { + type: TX, + metricKind: "DELTA", + valueType: "INT64", + unit: "1", + description: "", + monitoredResourceTypes: ["cloudsql_database"], + labels: ["transaction_type"], +} + +const seconds = (iso: string) => Date.parse(iso) / 1000 + +interface Calls { + readonly range: Array[1]> + readonly instant: Array[1]> + warehouse: number +} + +const makeFakes = (answer: (query: string) => ReadonlyArray = () => []) => { + const calls: Calls = { range: [], instant: [], warehouse: 0 } + const gcp: GcpLiveMetrics = { + queryRange: (_tenant, request) => { + calls.range.push(request) + return Effect.succeed(answer(request.query)) + }, + queryInstant: (_tenant, request) => { + calls.instant.push(request) + return Effect.succeed(answer(request.query)) + }, + listMetrics: () => Effect.succeed([transactions]), + describeMetric: (_tenant, type) => Effect.succeed(type === TX ? transactions : undefined), + } + const warehouse: QueryEngineWarehouse = { + rawSqlQuery: () => Effect.die("unexpected rawSqlQuery"), + compiledQuery: () => { + calls.warehouse++ + return Effect.succeed([]) + }, + compiledQueryWithCapabilities: () => Effect.die("unexpected capability-aware query"), + } + return { calls, gcp, warehouse } +} + +const series = (labels: Record, points: ReadonlyArray) => ({ + labels, + points: points.map(([iso, value]) => [seconds(iso), value] as const), +}) + +const execute = ( + fakes: { readonly warehouse: QueryEngineWarehouse; readonly gcp?: GcpLiveMetrics }, + query: QuerySpec, +) => + makeQueryEngineExecute(fakes.warehouse, fakes.gcp)( + tenant, + new QueryEngineExecuteRequest({ + startTime: "2026-10-19 10:00:00", + endTime: "2026-10-19 10:15:00", + query, + }), + ) + +const dataOf = ( + response: { readonly result: QueryEngineResult }, + kind: Kind, +): Extract["data"] => + response.result.kind === kind + ? (response.result as Extract).data + : assert.fail(`expected a ${kind} result, got ${response.result.kind}`) + +const metricFilters = (metricName: string) => ({ + metricName: MetricName.make(metricName), + metricType: "sum" as const, +}) + +const timeseries = (overrides: Partial> = {}) => + ({ + kind: "timeseries", + source: "metrics", + metric: "sum", + filters: metricFilters(TX), + bucketSeconds: 300, + ...overrides, + }) satisfies QuerySpec + +describe("Google Cloud metrics routing", () => { + it.effect( + "answers a Cloud Monitoring metric from Google and labels points by the bucket they close", + () => + Effect.gen(function* () { + const fakes = makeFakes(() => [ + series({ database_id: "p:main" }, [ + ["2026-10-19T10:05:00Z", 4], + ["2026-10-19T10:10:00Z", 7], + ]), + series({}, [["2026-10-19T10:10:00Z", 1]]), + ]) + + const response = yield* execute( + fakes, + timeseries({ + groupBy: ["attribute"], + filters: { ...metricFilters(TX), groupByAttributeKey: "database_id" }, + }), + ) + + assert.strictEqual(fakes.calls.warehouse, 0) + assert.deepStrictEqual(fakes.calls.range, [ + { + query: `sum by (database_id) (sum_over_time({"${TX}", monitored_resource="cloudsql_database"}[300s]))`, + startSeconds: seconds("2026-10-19T10:05:00Z"), + endSeconds: seconds("2026-10-19T10:20:00Z"), + stepSeconds: 300, + }, + ]) + assert.deepStrictEqual(dataOf(response, "timeseries"), [ + { bucket: "2026-10-19T10:00:00.000Z", series: { "p:main": 4 } }, + { bucket: "2026-10-19T10:05:00.000Z", series: { "p:main": 7, "(empty)": 1 } }, + { bucket: "2026-10-19T10:10:00.000Z", series: {} }, + { bucket: "2026-10-19T10:15:00.000Z", series: {} }, + ]) + }), + ) + + it.effect("keeps every other metric on the warehouse", () => + Effect.gen(function* () { + const fakes = makeFakes() + yield* execute(fakes, timeseries({ filters: metricFilters("http.server.duration") })) + assert.strictEqual(fakes.calls.warehouse, 1) + assert.strictEqual(fakes.calls.range.length, 0) + }), + ) + + it.effect("returns one ungrouped series, ignoring a service group-by", () => + Effect.gen(function* () { + const fakes = makeFakes(() => [series({}, [["2026-10-19T10:05:00Z", 3]])]) + const response = yield* execute(fakes, timeseries({ groupBy: ["service"] })) + assert.deepStrictEqual(dataOf(response, "timeseries")[0], { + bucket: "2026-10-19T10:00:00.000Z", + series: { all: 3 }, + }) + assert.notInclude(fakes.calls.range[0]?.query ?? "", " by ") + }), + ) + + it.effect("caps a grouped series to the highest peaks", () => + Effect.gen(function* () { + const fakes = makeFakes(() => [ + series({ database_id: "low" }, [["2026-10-19T10:05:00Z", 1]]), + series({ database_id: "high" }, [["2026-10-19T10:05:00Z", 9]]), + series({ database_id: "mid" }, [["2026-10-19T10:10:00Z", 5]]), + ]) + const response = yield* execute( + fakes, + timeseries({ + groupBy: ["attribute"], + filters: { ...metricFilters(TX), groupByAttributeKey: "database_id" }, + seriesLimit: 2, + }), + ) + const names = new Set( + dataOf(response, "timeseries").flatMap((point) => Object.keys(point.series)), + ) + assert.deepStrictEqual([...names].sort(), ["high", "mid"]) + }), + ) + + it.effect("reads an unknown metric as no data, with every bucket still present", () => + Effect.gen(function* () { + const fakes = makeFakes() + const response = yield* execute( + fakes, + timeseries({ filters: metricFilters("cloudsql.googleapis.com/database/never_reported") }), + ) + assert.strictEqual(fakes.calls.range.length, 0) + const data = dataOf(response, "timeseries") + assert.strictEqual(data.length, 4) + assert.isTrue(data.every((point) => Object.keys(point.series).length === 0)) + }), + ) + + it.effect("fails as unavailable when the deployment has no Google port", () => + Effect.gen(function* () { + const error = yield* Effect.flip(execute({ warehouse: makeFakes().warehouse }, timeseries())) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryUnavailableError") + }), + ) + + it.effect("narrows the fan-out to the filtered projects", () => + Effect.gen(function* () { + const fakes = makeFakes() + yield* execute( + fakes, + timeseries({ + filters: { + ...metricFilters(TX), + resourceAttributeFilters: [{ key: "project_id", mode: "equals", value: "alpha" }], + }, + }), + ) + assert.deepStrictEqual(fakes.calls.range[0]?.projects, ["alpha"]) + }), + ) +}) + +describe("Google Cloud metrics breakdown", () => { + const breakdown = ( + overrides: Partial> = {}, + ) => + ({ + kind: "breakdown", + source: "metrics", + metric: "rate", + groupBy: "attribute", + filters: { ...metricFilters(TX), groupByAttributeKey: "database_id" }, + ...overrides, + }) satisfies QuerySpec + + it.effect("ranks groups from one instant query over the whole range", () => + Effect.gen(function* () { + const fakes = makeFakes(() => [ + series({ database_id: "a" }, [["2026-10-19T10:15:00Z", 2]]), + series({ database_id: "b" }, [["2026-10-19T10:15:00Z", 5]]), + series({ database_id: "c" }, [["2026-10-19T10:15:00Z", 1]]), + ]) + const response = yield* execute(fakes, breakdown({ limit: 2 })) + assert.deepStrictEqual(fakes.calls.instant, [ + { + query: `sum by (database_id) (sum_over_time({"${TX}", monitored_resource="cloudsql_database"}[900s])) / 900`, + timeSeconds: seconds("2026-10-19T10:15:00Z"), + }, + ]) + assert.deepStrictEqual(dataOf(response, "breakdown"), [ + { name: "b", value: 5 }, + { name: "a", value: 2 }, + ]) + }), + ) + + it.effect("asks for a label group instead of service", () => + Effect.gen(function* () { + const error = yield* Effect.flip(execute(makeFakes(), breakdown({ groupBy: "service" }))) + assert.strictEqual(error._tag, "@maple/http/errors/QueryEngineValidationError") + }), + ) +}) + +describe("Google Cloud metric discovery", () => { + it.effect("lists the descriptor's labels and the labels seen on recent series", () => + Effect.gen(function* () { + const fakes = makeFakes(() => [ + series( + { __name__: TX, project_id: "alpha", database_id: "a", transaction_type: "commit" }, + [], + ), + series({ __name__: TX, project_id: "alpha", database_id: "b" }, []), + ]) + const all = yield* execute(fakes, { kind: "attributeKeys", source: "metrics", metricName: TX }) + assert.deepStrictEqual(dataOf(all, "attributeKeys"), [ + { key: "database_id", count: 2 }, + { key: "project_id", count: 2 }, + { key: "transaction_type", count: 1 }, + ]) + const resource = yield* execute(fakes, { + kind: "attributeKeys", + source: "metrics", + metricName: TX, + scope: "resource", + }) + assert.deepStrictEqual( + dataOf(resource, "attributeKeys").map((item) => item.key), + ["database_id", "project_id"], + ) + }), + ) + + it.effect("lists a label's values with their series counts", () => + Effect.gen(function* () { + const fakes = makeFakes(() => [ + series({ database_id: "a" }, [["2026-10-19T10:15:00Z", 1]]), + series({ database_id: "b" }, [["2026-10-19T10:15:00Z", 3]]), + series({}, [["2026-10-19T10:15:00Z", 9]]), + ]) + const response = yield* execute(fakes, { + kind: "attributeValues", + source: "metrics", + scope: "metric", + attributeKey: "database_id", + metricName: TX, + }) + assert.strictEqual( + fakes.calls.instant[0]?.query, + `count by (database_id) (last_over_time({"${TX}", monitored_resource="cloudsql_database"}[900s]))`, + ) + assert.deepStrictEqual(dataOf(response, "attributeValues"), [ + { value: "b", count: 3 }, + { value: "a", count: 1 }, + ]) + }), + ) + + it.effect("leaves Google Cloud metrics out of the warehouse sparklines read", () => + Effect.gen(function* () { + const fakes = makeFakes() + const response = yield* execute(fakes, { + kind: "sparklines", + source: "metrics", + metricType: "sum", + metricNames: [MetricName.make(TX)], + }) + assert.strictEqual(fakes.calls.warehouse, 0) + assert.deepStrictEqual(dataOf(response, "sparklines"), [{ metricName: TX, points: [] }]) + }), + ) +}) + +describe("Google Cloud metric alerts", () => { + const alertRequest = (query: QuerySpec): AlertEvaluateRequest => ({ + startTime: "2026-10-19 10:00:00", + endTime: "2026-10-19 10:05:00", + source: { kind: "spec", query }, + reducer: "identity", + sampleCountStrategy: "metric_data_points", + }) + const answer = (query: string) => + query.includes("count_over_time") + ? [series({ database_id: "a" }, [["2026-10-19T10:05:00Z", 5]])] + : [ + series({ database_id: "a" }, [["2026-10-19T10:05:00Z", 42]]), + series({ database_id: "b" }, [["2026-10-19T10:05:00Z", 7]]), + ] + const grouped = timeseries({ + groupBy: ["attribute"], + filters: { ...metricFilters(TX), groupByAttributeKey: "database_id" }, + bucketSeconds: 60, + }) + + it.effect("evaluates the window as one instant query per group, with sample counts", () => + Effect.gen(function* () { + const fakes = makeFakes(answer) + const result = yield* makeQueryEngineEvaluate(fakes.warehouse, fakes.gcp)( + tenant, + alertRequest(grouped), + ) + assert.deepStrictEqual( + fakes.calls.instant.map((call) => [call.query, call.timeSeconds]), + [ + [ + `sum by (database_id) (sum_over_time({"${TX}", monitored_resource="cloudsql_database"}[300s]))`, + seconds("2026-10-19T10:05:00Z"), + ], + [ + `sum by (database_id) (count_over_time({"${TX}", monitored_resource="cloudsql_database"}[300s]))`, + seconds("2026-10-19T10:05:00Z"), + ], + ], + ) + assert.deepStrictEqual(result, [ + { groupKey: "a", value: 42, sampleCount: 5, hasData: true }, + { groupKey: "b", value: null, sampleCount: 0, hasData: false }, + ]) + }), + ) + + it.effect("previews the alert as bucketed range queries", () => + Effect.gen(function* () { + const fakes = makeFakes(answer) + const result = yield* makeQueryEngineEvaluateSeries(fakes.warehouse, fakes.gcp)( + tenant, + alertRequest(grouped), + ) + assert.strictEqual(fakes.calls.range.length, 2) + assert.deepStrictEqual(result[0], { + bucket: "2026-10-19T10:04:00.000Z", + groupKey: "a", + value: 42, + sampleCount: 5, + }) + }), + ) + + it.effect("fails as unavailable without a Google port", () => + Effect.gen(function* () { + const fakes = makeFakes(answer) + const error = yield* Effect.flip( + makeQueryEngineEvaluate(fakes.warehouse)(tenant, alertRequest(grouped)), + ) + assert.strictEqual(error._tag, "@maple/http/errors/GcpLiveQueryUnavailableError") + }), + ) +}) diff --git a/packages/query-engine/src/runtime/gcp-metrics.ts b/packages/query-engine/src/runtime/gcp-metrics.ts new file mode 100644 index 0000000000..1ea043cb39 --- /dev/null +++ b/packages/query-engine/src/runtime/gcp-metrics.ts @@ -0,0 +1,279 @@ +// Metrics queries on a Cloud Monitoring metric type, answered live from Google (see +// `../promql/compile`). Each returns rows the engine shapes exactly like its ClickHouse results. + +import { DateTime, Effect } from "effect" +import type { GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import { QueryEngineValidationError } from "@maple/domain/http" +import { GcpLiveQueryUnavailableError } from "@maple/domain/http/gcp-live-query-errors" +import type { MetricsFilters, MetricsMetric, QuerySpec } from "@maple/domain/query-engine" +import { ENGINE_UNGROUPED_GROUP_KEY } from "../group-key" +import { + compileGcpLabelSample, + compileGcpLabelValues, + compileGcpMetricQuery, + gcpPointBucket, + gcpRangeWindow, + type GcpPromQl, +} from "../promql/compile" +import type { GcpLiveMetrics } from "./live-metrics" +import type { BucketGroupObs, QueryTenant } from "./query-engine" + +type MetricsTimeseriesSpec = Extract +type MetricsBreakdownSpec = Extract + +export interface GcpTimeseriesRow { + readonly bucket: string + readonly groupName: string + readonly value: number +} + +const describe = Effect.fnUntraced(function* ( + gcp: GcpLiveMetrics | undefined, + tenant: T, + metricName: string, +) { + if (gcp === undefined) { + return yield* new GcpLiveQueryUnavailableError({ + message: "Google Cloud metrics are not available in this deployment", + }) + } + return { port: gcp, descriptor: yield* gcp.describeMetric(tenant, metricName) } +}) + +const annotate = (promQl: string) => + Effect.annotateCurrentSpan({ "maple.query.backend": "gcp", "maple.gcp.promql": promQl }) + +const projectsOf = (compiled: GcpPromQl) => + compiled.projects === undefined ? undefined : { projects: compiled.projects } + +const groupLabelOf = (groupBy: ReadonlyArray, filters: MetricsFilters): string | undefined => { + if (groupBy.includes("none")) return undefined + if (groupBy.includes("attribute")) return filters.groupByAttributeKey + if (groupBy.includes("resource_attribute")) return filters.groupByResourceAttributeKey + return undefined +} + +/** A whole-range window: whole seconds, at least one Google sample period. */ +const wholeWindowSeconds = (rangeSeconds: number): number => Math.max(60, Math.ceil(rangeSeconds)) + +const groupValue = (series: GcpPromQlSeries, groupLabel: string | undefined): string | undefined => + groupLabel === undefined ? undefined : series.labels[groupLabel] + +/** Keep the `limit` groups with the highest peak, ties by name, like `finalizeTimeseries`. */ +const capSeries = (rows: ReadonlyArray, limit: number): ReadonlyArray => { + const peaks = new Map() + for (const row of rows) + peaks.set(row.groupName, Math.max(peaks.get(row.groupName) ?? -Infinity, row.value)) + const kept = new Set( + [...peaks] + .sort(([leftName, left], [rightName, right]) => right - left || leftName.localeCompare(rightName)) + .slice(0, limit) + .map(([name]) => name), + ) + return rows.filter((row) => kept.has(row.groupName)) +} + +/** Bucketed series; `stepSeconds` is the bucket width the points sit on. */ +export const gcpMetricsTimeseries = Effect.fnUntraced(function* ( + gcp: GcpLiveMetrics | undefined, + tenant: T, + query: MetricsTimeseriesSpec, + range: { readonly startMs: number; readonly endMs: number; readonly bucketSeconds: number }, +) { + const window = gcpRangeWindow(range.startMs, range.endMs, range.bucketSeconds) + const { port, descriptor } = yield* describe(gcp, tenant, query.filters.metricName) + if (descriptor === undefined || window.startSeconds > window.endSeconds) { + return { stepSeconds: window.stepSeconds, rows: [] } + } + const groupLabel = groupLabelOf(query.groupBy ?? [], query.filters) + const compiled = yield* compileGcpMetricQuery( + { metric: query.metric, filters: query.filters, groupLabel, windowSeconds: window.stepSeconds }, + descriptor, + ) + yield* annotate(compiled.query) + const series = yield* port.queryRange(tenant, { + query: compiled.query, + ...window, + ...projectsOf(compiled), + }) + const rows = series.flatMap((s) => + s.points.map(([seconds, value]) => ({ + bucket: gcpPointBucket(seconds, window.stepSeconds), + groupName: + groupLabel === undefined + ? ENGINE_UNGROUPED_GROUP_KEY + : groupValue(s, groupLabel) || "(empty)", + value, + })), + ) + return { + stepSeconds: window.stepSeconds, + rows: groupLabel !== undefined && query.seriesLimit ? capSeries(rows, query.seriesLimit) : rows, + } +}) + +/** One value per group over the whole range, ranked descending. */ +export const gcpMetricsBreakdown = Effect.fnUntraced(function* ( + gcp: GcpLiveMetrics | undefined, + tenant: T, + query: MetricsBreakdownSpec, + range: { readonly endMs: number; readonly rangeSeconds: number }, +) { + if (query.groupBy === "service") { + return yield* new QueryEngineValidationError({ + message: "Unsupported Google Cloud metrics query", + details: [ + "Google Cloud metrics have no service.name; break down by a label (groupBy attribute or resource_attribute)", + ], + }) + } + const { port, descriptor } = yield* describe(gcp, tenant, query.filters.metricName) + if (descriptor === undefined) return [] + const groupLabel = groupLabelOf([query.groupBy], query.filters) + const compiled = yield* compileGcpMetricQuery( + { + metric: query.metric, + filters: query.filters, + groupLabel, + windowSeconds: wholeWindowSeconds(range.rangeSeconds), + }, + descriptor, + ) + yield* annotate(compiled.query) + const series = yield* port.queryInstant(tenant, { + query: compiled.query, + timeSeconds: Math.floor(range.endMs / 1000), + ...projectsOf(compiled), + }) + return series + .flatMap((s) => + s.points.map(([, value]) => ({ name: groupValue(s, groupLabel) || "(empty)", value })), + ) + .sort((a, b) => b.value - a.value) + .slice(0, query.limit ?? 10) +}) + +/** + * Alert observations: the value and its sample count (the `count` aggregation, as the ClickHouse + * path's `dataPointCount`) per group. `wholeRange` is one instant query over the window at its end. + */ +export const gcpMetricsAlertBuckets = Effect.fnUntraced(function* ( + gcp: GcpLiveMetrics | undefined, + tenant: T, + query: MetricsTimeseriesSpec, + range: { + readonly startMs: number + readonly endMs: number + readonly bucketSeconds: number + readonly wholeRange?: boolean + }, +) { + const none: ReadonlyArray = [] + const { port, descriptor } = yield* describe(gcp, tenant, query.filters.metricName) + const window = range.wholeRange + ? undefined + : gcpRangeWindow(range.startMs, range.endMs, range.bucketSeconds) + if (descriptor === undefined || (window && window.startSeconds > window.endSeconds)) return none + const groupLabel = groupLabelOf(query.groupBy ?? [], query.filters) + const windowSeconds = window?.stepSeconds ?? wholeWindowSeconds((range.endMs - range.startMs) / 1000) + const compile = (metric: MetricsMetric) => + compileGcpMetricQuery({ metric, filters: query.filters, groupLabel, windowSeconds }, descriptor) + const value = yield* compile(query.metric) + const samples = yield* compile("count") + yield* annotate(value.query) + + const run = (compiled: GcpPromQl) => + window + ? port.queryRange(tenant, { query: compiled.query, ...window, ...projectsOf(compiled) }) + : port.queryInstant(tenant, { + query: compiled.query, + timeSeconds: Math.floor(range.endMs / 1000), + ...projectsOf(compiled), + }) + const [values, counts] = yield* Effect.all([run(value), run(samples)], { concurrency: 2 }) + + const rangeStart = DateTime.formatIso(DateTime.makeUnsafe(range.startMs)) + const bucketOf = (seconds: number) => (window ? gcpPointBucket(seconds, window.stepSeconds) : rangeStart) + const groupKeyOf = (s: GcpPromQlSeries) => groupValue(s, groupLabel) || ENGINE_UNGROUPED_GROUP_KEY + const sampleCounts = new Map() + for (const s of counts) { + for (const [seconds, count] of s.points) + sampleCounts.set(`${bucketOf(seconds)} ${groupKeyOf(s)}`, count) + } + return values.flatMap((s) => + s.points.map(([seconds, observed]): BucketGroupObs => { + const bucket = bucketOf(seconds) + const groupKey = groupKeyOf(s) + const sampleCount = sampleCounts.get(`${bucket} ${groupKey}`) ?? 0 + return { bucket, groupKey, value: sampleCount > 0 ? observed : null, sampleCount } + }), + ) +}) + +/** Label keys: the descriptor's metric labels plus those seen on a few recent series. */ +export const gcpMetricAttributeKeys = Effect.fnUntraced(function* ( + gcp: GcpLiveMetrics | undefined, + tenant: T, + request: { + readonly metricName: string + readonly scope?: "span" | "resource" + readonly limit?: number + readonly endMs: number + }, +) { + const { port, descriptor } = yield* describe(gcp, tenant, request.metricName) + if (descriptor === undefined) return [] + const sample = yield* compileGcpLabelSample(descriptor) + yield* annotate(sample) + const series = yield* port.queryInstant(tenant, { + query: sample, + timeSeconds: Math.floor(request.endMs / 1000), + }) + // `scope: "resource"` asks for the labels Google adds beyond the metric's own. + const resourceOnly = request.scope === "resource" + const counts = new Map(resourceOnly ? [] : descriptor.labels.map((label) => [label, 0])) + for (const s of series) { + for (const key of Object.keys(s.labels)) { + if (key === "__name__" || (resourceOnly && descriptor.labels.includes(key))) continue + counts.set(key, (counts.get(key) ?? 0) + 1) + } + } + return [...counts] + .map(([key, count]) => ({ key, count })) + .sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)) + .slice(0, request.limit ?? 200) +}) + +/** Values of one label with their series counts over the range. */ +export const gcpMetricAttributeValues = Effect.fnUntraced(function* ( + gcp: GcpLiveMetrics | undefined, + tenant: T, + request: { + readonly metricName: string + readonly attributeKey: string + readonly limit?: number + readonly endMs: number + readonly rangeSeconds: number + }, +) { + const { port, descriptor } = yield* describe(gcp, tenant, request.metricName) + if (descriptor === undefined) return [] + const promQl = yield* compileGcpLabelValues( + descriptor, + request.attributeKey, + wholeWindowSeconds(request.rangeSeconds), + ) + yield* annotate(promQl) + const series = yield* port.queryInstant(tenant, { + query: promQl, + timeSeconds: Math.floor(request.endMs / 1000), + }) + return series + .flatMap((s) => { + const value = s.labels[request.attributeKey] + const point = s.points[0] + return value && point ? [{ value, count: point[1] }] : [] + }) + .sort((a, b) => b.count - a.count) + .slice(0, request.limit ?? 50) +}) diff --git a/packages/query-engine/src/runtime/index.ts b/packages/query-engine/src/runtime/index.ts index 4ab4f69d5d..e8bd438865 100644 --- a/packages/query-engine/src/runtime/index.ts +++ b/packages/query-engine/src/runtime/index.ts @@ -1,5 +1,7 @@ export * from "./query-engine" +export * from "./live-metrics" export * from "./raw-sql" export * from "./cache-policy" export * from "./query-definition-runner" export { withAlertEvaluationScope } from "./alert-evaluation-scope" +export type { GcpLiveMetrics } from "./live-metrics" diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index f3de0c3a2c..87af8ef156 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -71,6 +71,15 @@ import { } from "../registry/product-events" import { runQueryDefinition } from "./query-definition-runner" import { resolveDirectRouteCachePolicy, type DirectRouteCachePolicyInput } from "./cache-policy" +import { isGcpMetricType } from "@maple/domain/gcp-live-metrics" +import type { GcpLiveMetrics } from "./live-metrics" +import { + gcpMetricAttributeKeys, + gcpMetricAttributeValues, + gcpMetricsAlertBuckets, + gcpMetricsBreakdown, + gcpMetricsTimeseries, +} from "./gcp-metrics" export { makeDirectRouteCachePolicy, @@ -1448,7 +1457,11 @@ const withTraceFacetsFallback = ( ) : run(true) -export const makeQueryEngineExecute = (warehouse: QueryEngineWarehouse) => +/** `gcp` answers metrics on a Cloud Monitoring metric type; absent, those fail as unavailable. */ +export const makeQueryEngineExecute = ( + warehouse: QueryEngineWarehouse, + gcp?: GcpLiveMetrics, +) => Effect.fn("QueryEngineService.execute")(function* ( tenant: T, request: QueryEngineExecuteRequest, @@ -1639,6 +1652,29 @@ export const makeQueryEngineExecute = (warehouse: QueryEn }) } + if ( + request.query.source === "metrics" && + request.query.kind === "timeseries" && + isGcpMetricType(request.query.filters.metricName) + ) { + const { rows, stepSeconds } = yield* gcpMetricsTimeseries(gcp, tenant, request.query, { + startMs: range.startMs, + endMs: range.endMs, + bucketSeconds, + }) + return new QueryEngineExecuteResponse({ + result: { + kind: "timeseries", + source: "metrics", + data: groupTimeSeriesRows(rows, (row) => row.value, { + startMs: range.startMs, + endMs: range.endMs, + bucketSeconds: stepSeconds, + }), + }, + }) + } + if (request.query.source === "metrics" && request.query.kind === "timeseries") { const execution = yield* executeMetricsTimeseriesRows( warehouse, @@ -1720,27 +1756,32 @@ export const makeQueryEngineExecute = (warehouse: QueryEn result: { kind: "sparklines", source: "metrics", data: [] }, }) } + // Google Cloud metrics are not read for previews; they keep empty points. + const warehouseMetricNames = metricNames.filter((name) => !isGcpMetricType(name)) const sparklineBucketSeconds = request.query.bucketSeconds ?? computeBucketSeconds(range.startMs, range.endMs) yield* Effect.annotateCurrentSpan("query.bucketSeconds", sparklineBucketSeconds) yield* Effect.annotateCurrentSpan("query.metricCount", metricNames.length) - const rows = yield* executeCHQuery( - warehouse, - tenant, - CH.metricsSparklinesQuery({ - metricType: request.query.metricType, - metricNames, - }), - { - orgId: tenant.orgId, - startTime: request.startTime, - endTime: request.endTime, - bucketSeconds: sparklineBucketSeconds, - }, - "metricsSparklines", - ) + const rows = + warehouseMetricNames.length === 0 + ? [] + : yield* executeCHQuery( + warehouse, + tenant, + CH.metricsSparklinesQuery({ + metricType: request.query.metricType, + metricNames: warehouseMetricNames, + }), + { + orgId: tenant.orgId, + startTime: request.startTime, + endTime: request.endTime, + bucketSeconds: sparklineBucketSeconds, + }, + "metricsSparklines", + ) // Numbers are coerced here (not via a rowSchema) because BYO-ClickHouse // serializes 64-bit aggregates as JSON strings. @@ -1859,6 +1900,20 @@ export const makeQueryEngineExecute = (warehouse: QueryEn }) } + if ( + request.query.source === "metrics" && + request.query.kind === "breakdown" && + isGcpMetricType(request.query.filters.metricName) + ) { + const data = yield* gcpMetricsBreakdown(gcp, tenant, request.query, { + endMs: range.endMs, + rangeSeconds: range.rangeSeconds, + }) + return new QueryEngineExecuteResponse({ + result: { kind: "breakdown", source: "metrics", data }, + }) + } + if (request.query.source === "metrics" && request.query.kind === "breakdown") { const data = yield* executeMetricsBreakdownRows(warehouse, tenant, request.query, { startTime: request.startTime, @@ -2095,6 +2150,23 @@ export const makeQueryEngineExecute = (warehouse: QueryEn }) } + if ( + request.query.source === "metrics" && + request.query.kind === "attributeKeys" && + request.query.metricName !== undefined && + isGcpMetricType(request.query.metricName) + ) { + const data = yield* gcpMetricAttributeKeys(gcp, tenant, { + metricName: request.query.metricName, + scope: request.query.scope, + limit: request.query.limit, + endMs: range.endMs, + }) + return new QueryEngineExecuteResponse({ + result: { kind: "attributeKeys", source: "metrics", data }, + }) + } + if (request.query.kind === "attributeKeys") { const scope = resolveAttributeScope(request.query.source, request.query.scope) // Per-metric scoping reads the raw metric table (the hourly rollup has no @@ -2318,6 +2390,24 @@ export const makeQueryEngineExecute = (warehouse: QueryEn }) } + if ( + request.query.source === "metrics" && + request.query.kind === "attributeValues" && + request.query.metricName !== undefined && + isGcpMetricType(request.query.metricName) + ) { + const data = yield* gcpMetricAttributeValues(gcp, tenant, { + metricName: request.query.metricName, + attributeKey: request.query.attributeKey, + limit: request.query.limit, + endMs: range.endMs, + rangeSeconds: range.rangeSeconds, + }) + return new QueryEngineExecuteResponse({ + result: { kind: "attributeValues", source: "metrics", data }, + }) + } + if (request.query.kind === "attributeValues") { // Per-metric scoping reads the raw metric table (see attributeKeys above). const { metricName, metricNames, metricType } = request.query @@ -2487,6 +2577,7 @@ export const computeAlertBuckets = Effect.fnUntraced(function* , ) { if (request.source.kind === "raw_sql") { return yield* computeRawSqlBuckets(warehouse, tenant, request.source, request) @@ -2591,6 +2682,15 @@ export const computeAlertBuckets = Effect.fnUntraced(function* (warehouse: QueryEngineWarehouse) => +export const makeQueryEngineEvaluate = ( + warehouse: QueryEngineWarehouse, + gcp?: GcpLiveMetrics, +) => Effect.fn("QueryEngineService.evaluate")(function* ( tenant: T, request: AlertEvaluateRequest, @@ -2852,7 +2955,7 @@ export const makeQueryEngineEvaluate = (warehouse: QueryE endTime: request.endTime, wholeRange: true, } - const load = computeAlertBuckets(warehouse, tenant, bucketRequest, bucketSeconds) + const load = computeAlertBuckets(warehouse, tenant, bucketRequest, bucketSeconds, gcp) // Raw SQL may contain volatile functions. Keep its existing whole-result // cache policy; only structured queries share buckets across reducers. const obs = yield* request.source.kind === "spec" @@ -2876,7 +2979,10 @@ export const makeQueryEngineEvaluate = (warehouse: QueryE * * Preview requests are ad-hoc form states and execute directly. */ -export const makeQueryEngineEvaluateSeries = (warehouse: QueryEngineWarehouse) => +export const makeQueryEngineEvaluateSeries = ( + warehouse: QueryEngineWarehouse, + gcp?: GcpLiveMetrics, +) => Effect.fn("QueryEngineService.evaluateSeries")(function* ( tenant: T, request: AlertEvaluateRequest, @@ -2889,6 +2995,7 @@ export const makeQueryEngineEvaluateSeries = (warehouse: tenant, { source: request.source, startTime: request.startTime, endTime: request.endTime }, bucketSeconds, + gcp, ) yield* Effect.annotateCurrentSpan("result.pointCount", series.length) return series From e6bc68140e1d389cb7e811a0afc162f6ea158d16 Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Mon, 19 Oct 2026 17:52:36 +0200 Subject: [PATCH 03/16] feat(gcp): list Google Cloud metrics with Maple's, and read them live in api, MCP and alerts --- apps/ai/src/mcp/dispatcher.ts | 3 + .../src/mcp/lib/map-warehouse-error.test.ts | 40 +++++ apps/ai/src/mcp/lib/map-warehouse-error.ts | 68 +++++++-- apps/ai/src/mcp/tools/list-metrics.ts | 26 +++- apps/ai/src/mcp/tools/query-data.ts | 4 +- .../src/routes/internal/query-engine.http.ts | 10 +- apps/api/src/routes/v2/telemetry.http.test.ts | 43 +++++- apps/api/src/routes/v2/telemetry.http.ts | 75 +++++---- apps/api/src/routes/v2/v2-test-support.ts | 1 + .../src/routes/v2/widget-summary.http.test.ts | 1 + apps/api/src/runtime/query-http-graph.ts | 3 +- apps/api/src/runtime/service-graph.ts | 3 + .../src/content/docs/integrations/gcp.md | 12 ++ .../metrics/metric-metadata-panel.tsx | 5 +- .../metrics/metric-preview-card.tsx | 13 +- .../src/components/metrics/metrics-table.tsx | 6 +- apps/web/src/hooks/use-widget-builder-data.ts | 7 +- .../src/services/alerts/AlertsService.ts | 3 + .../services/warehouse/GcpLiveMetricsPort.ts | 10 +- .../services/warehouse/QueryEngineService.ts | 11 ++ .../warehouse/gcp-metric-catalog.test.ts | 143 ++++++++++++++++++ .../services/warehouse/gcp-metric-catalog.ts | 84 ++++++++++ 22 files changed, 509 insertions(+), 62 deletions(-) create mode 100644 packages/backend/src/services/warehouse/gcp-metric-catalog.test.ts create mode 100644 packages/backend/src/services/warehouse/gcp-metric-catalog.ts diff --git a/apps/ai/src/mcp/dispatcher.ts b/apps/ai/src/mcp/dispatcher.ts index 50e6b668ca..62991cf270 100644 --- a/apps/ai/src/mcp/dispatcher.ts +++ b/apps/ai/src/mcp/dispatcher.ts @@ -30,6 +30,7 @@ import { VcsCommitService } from "@maple/backend/services/integrations/vcs/VcsCo import { IntegrationInventoryService } from "@maple/backend/services/integrations/IntegrationInventoryService" import { AgentFeedbackService } from "@maple/backend/services/feedback/AgentFeedbackService" import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" +import { GcpLiveMetricsPort } from "@maple/backend/services/warehouse/GcpLiveMetricsPort" // BOUNDARY: This module owns unparsed external values and narrows them before domain use. import type { McpToolNotFoundError } from "@maple/domain/mcp-tool-contract" @@ -278,6 +279,8 @@ const McpRuntimeServicesLive = Layer.mergeAll( ), ), Layer.provide(PullRequestLookupLive), + // Graph-wide, so every `QueryEngineService.layer` build (it is memoized) reads Google metrics. + Layer.provide(GcpLiveMetricsPort.layer), Layer.provide(FetchHttpClient.layer), ) diff --git a/apps/ai/src/mcp/lib/map-warehouse-error.test.ts b/apps/ai/src/mcp/lib/map-warehouse-error.test.ts index ff56febe66..c2da9cbbe8 100644 --- a/apps/ai/src/mcp/lib/map-warehouse-error.test.ts +++ b/apps/ai/src/mcp/lib/map-warehouse-error.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest" import { WarehouseQueryError, WarehouseResultDecodeError, WarehouseSchemaDriftError } from "@maple/domain" +import { + GcpLiveQueryFailedError, + GcpLiveQueryRejectedError, + GcpLiveQueryUnavailableError, +} from "@maple/domain/http/gcp-live-query-errors" import { toMcpQueryError } from "./map-warehouse-error" describe("toMcpQueryError", () => { @@ -37,4 +42,39 @@ describe("toMcpQueryError", () => { expect(mcp.message).toBe("row did not decode") expect(mcp.message).not.toContain("schema apply") }) + + it("makes a query Google refused the caller's to fix", () => { + const err = new GcpLiveQueryRejectedError({ + message: "The query reaches 34 projects; filter on project_id", + reason: "too_many_projects", + }) + const mcp = toMcpQueryError("query_data")(err) + expect(mcp._tag).toBe("@maple/mcp/errors/McpInvalidInputError") + expect(mcp.message).toBe("The query reaches 34 projects; filter on project_id") + }) + + it("reports Google Cloud metrics as unavailable when no connection reads them", () => { + const err = new GcpLiveQueryUnavailableError({ + message: "No Google Cloud connection has metrics on.", + }) + const mcp = toMcpQueryError("query_data")(err) + expect(mcp).toMatchObject({ + _tag: "@maple/mcp/errors/McpUnavailableError", + message: "No Google Cloud connection has metrics on.", + capability: "google_cloud_metrics", + }) + }) + + it("asks for a retry when Google did not answer", () => { + const err = new GcpLiveQueryFailedError({ + message: "Google Cloud could not be read.", + failures: [{ scope: "projects/acme", message: "503" }], + }) + const mcp = toMcpQueryError("query_data")(err) + expect(mcp).toMatchObject({ + _tag: "@maple/mcp/errors/McpNotReadyError", + message: "Google Cloud could not be read.", + retryAfterSeconds: 60, + }) + }) }) diff --git a/apps/ai/src/mcp/lib/map-warehouse-error.ts b/apps/ai/src/mcp/lib/map-warehouse-error.ts index d2f8cbd4cc..c8814ab859 100644 --- a/apps/ai/src/mcp/lib/map-warehouse-error.ts +++ b/apps/ai/src/mcp/lib/map-warehouse-error.ts @@ -1,10 +1,25 @@ import { Effect } from "effect" -import { type WarehouseError, WarehouseQuotaExceededError, WarehouseSchemaDriftError } from "@maple/domain" +import { + type WarehouseError, + type WarehouseQuotaExceededError, + WarehouseSchemaDriftError, +} from "@maple/domain" import { warehouseHandlers, warehouseReadHandlers, } from "@maple/backend/services/warehouse/warehouse-error-handlers" -import { McpQueryBudgetError, McpQueryError } from "../tools/types" +import type { + GcpLiveQueryFailedError, + GcpLiveQueryRejectedError, + GcpLiveQueryUnavailableError, +} from "@maple/domain/http/gcp-live-query-errors" +import { + McpInvalidInputError, + McpNotReadyError, + McpQueryBudgetError, + McpQueryError, + McpUnavailableError, +} from "../tools/types" export { warehouseHandlers, warehouseReadHandlers } @@ -33,23 +48,54 @@ const BUDGET_GUIDANCE = { max_threads: "The query needed more parallelism than it is allowed. Scan less data.", } satisfies Record +/** Seconds the model waits before asking Google again. */ +const GCP_RETRY_AFTER_SECONDS = 60 + +/** The MCP error a warehouse error becomes, per member, so reads that cannot reach Google keep a narrow type. */ +type McpErrorFor = E extends GcpLiveQueryRejectedError + ? McpInvalidInputError + : E extends GcpLiveQueryUnavailableError + ? McpUnavailableError + : E extends GcpLiveQueryFailedError + ? McpNotReadyError + : McpQueryError | McpQueryBudgetError + +const toMcpError = (pipe: string, error: WarehouseError) => { + switch (error._tag) { + case "@maple/http/errors/GcpLiveQueryRejectedError": + return new McpInvalidInputError({ message: error.message }) + case "@maple/http/errors/GcpLiveQueryUnavailableError": + return new McpUnavailableError({ message: error.message, capability: "google_cloud_metrics" }) + case "@maple/http/errors/GcpLiveQueryFailedError": + return new McpNotReadyError({ + message: error.message, + retryAfterSeconds: GCP_RETRY_AFTER_SECONDS, + }) + case "@maple/http/errors/WarehouseQuotaExceededError": + return new McpQueryBudgetError({ + message: BUDGET_GUIDANCE[error.setting], + pipeName: pipe, + setting: error.setting, + }) + default: + return new McpQueryError({ message: warehouseErrorText(error), pipeName: pipe, cause: error }) + } +} + /** * Curry the pipe label so call sites read as * `Effect.mapError(toMcpQueryError("service_overview"))`. * * A budget breach gets its own error with guidance: the raw text is the vendor's ("Upgrade your - * plan for higher capacity"), which no model can act on. + * plan for higher capacity"), which no model can act on. A metrics read answered live by Google + * fails like the tool's other dependencies: a refused query is the caller's to fix, no connection + * is unavailable, and Google not answering is worth a retry. */ export const toMcpQueryError = (pipe: string) => - (error: WarehouseError): McpQueryError | McpQueryBudgetError => - error instanceof WarehouseQuotaExceededError - ? new McpQueryBudgetError({ - message: BUDGET_GUIDANCE[error.setting], - pipeName: pipe, - setting: error.setting, - }) - : new McpQueryError({ message: warehouseErrorText(error), pipeName: pipe, cause: error }) + (error: E): McpErrorFor => + // TypeScript cannot narrow a conditional return type through the switch. + toMcpError(pipe, error) as McpErrorFor /** * `Effect.catchTags` handler map that converts every warehouse error tag into an diff --git a/apps/ai/src/mcp/tools/list-metrics.ts b/apps/ai/src/mcp/tools/list-metrics.ts index 4869f576cd..64e9dcc09d 100644 --- a/apps/ai/src/mcp/tools/list-metrics.ts +++ b/apps/ai/src/mcp/tools/list-metrics.ts @@ -7,6 +7,8 @@ import { doc } from "../lib/tool-doc" import { Effect, Schema } from "effect" import { ListMetricsOutput } from "@maple/domain/mcp-outputs" import { MetricType } from "@maple/query-engine" +import { isGcpMetricType } from "@maple/domain/gcp-live-metrics" +import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" const WINDOW = P.timeWindow({ defaultHours: 6, maxHours: MCP_DISCOVERY_MAX_HOURS }) @@ -30,7 +32,8 @@ export function registerListMetricsTool(server: McpToolRegistrar) { name: "list_metrics", title: "List Metrics", description: - "Discover custom metrics with their type, unit, monotonicity and data volume. Pass a discovered `metric_name` and `metric_type` to query_data source=metrics.", + "Discover custom metrics with their type, unit, monotonicity and data volume. Pass a discovered `metric_name` and `metric_type` to query_data source=metrics. " + + "Google Cloud Monitoring metrics with data in the last day are listed too, under service `Google Cloud`, and are read live from Google.", parameters: Schema.Struct({ ...WINDOW.fields, service: P.service(), @@ -46,6 +49,7 @@ export function registerListMetricsTool(server: McpToolRegistrar) { handler: Effect.fn("McpTool.listMetrics")(function* (params) { const { st, et } = yield* WINDOW.resolve(params, "list_metrics") const tenant = yield* CurrentMcpTenant + const queryEngine = yield* QueryEngineService yield* Effect.annotateCurrentSpan({ orgId: tenant.orgId, "maple.ai.service": params.service ?? "all", @@ -54,7 +58,7 @@ export function registerListMetricsTool(server: McpToolRegistrar) { "maple.ai.offset": params.offset, }) - const [metricsResult, summaryResult] = yield* Effect.all( + const [metricsResult, summaryResult, gcpMetrics] = yield* Effect.all( [ queryWarehouse("list_metrics", { start_time: st, @@ -70,21 +74,29 @@ export function registerListMetricsTool(server: McpToolRegistrar) { end_time: et, service: params.service, }), + queryEngine.listGcpMetrics(tenant, { + service: params.service, + metricType: params.metric_type, + search: params.search, + limit: params.limit, + offset: params.offset, + }), ], { concurrency: "unbounded" }, ) - const metrics = metricsResult.data + // Google rows ride on the first page; paging counts the warehouse's alone. + const hasMore = metricsResult.data.length === params.limit + const metrics: ReadonlyArray = [...metricsResult.data, ...gcpMetrics] yield* Effect.annotateCurrentSpan("result.rowCount", metrics.length) - const hasMore = metrics.length === params.limit return { timeRange: { start: st, end: et }, pagination: { offset: params.offset, limit: params.limit, hasMore, - ...(hasMore ? { nextOffset: params.offset + metrics.length } : undefined), + ...(hasMore ? { nextOffset: params.offset + metricsResult.data.length } : undefined), }, summary: summaryResult.data.map((s) => ({ metricType: s.metricType, @@ -164,7 +176,9 @@ export function registerListMetricsTool(server: McpToolRegistrar) { m.isMonotonic ? "yes" : "-", m.serviceName, m.metricUnit || "-", - formatNumber(m.dataPointCount), + isGcpMetricType(m.metricName) + ? "live" + : formatNumber(m.dataPointCount), ]), ), ]), diff --git a/apps/ai/src/mcp/tools/query-data.ts b/apps/ai/src/mcp/tools/query-data.ts index 1a7c4a8975..85713d6e3c 100644 --- a/apps/ai/src/mcp/tools/query-data.ts +++ b/apps/ai/src/mcp/tools/query-data.ts @@ -98,7 +98,9 @@ type Params = typeof queryDataSchema.Type const queryDataDescription = "Aggregate traces, logs, metrics or product events into a timeseries or a top-N breakdown. " + "Start here for trends, comparisons and top-N; for errors prefer find_errors and error_detail; " + - "for a shape this tool cannot express, run_sql. Applied defaults are listed in the result." + "for a shape this tool cannot express, run_sql. Applied defaults are listed in the result. " + + "A Google Cloud Monitoring metric type (e.g. cloudsql.googleapis.com/database/cpu/utilization) works as metric_name and is read live from Google; " + + "it has no service, so filter and group by its labels with attribute_key (e.g. database_id, project_id, monitored_resource)." /** Default metric/group_by per source, as `[default, available]` for the decisions list. */ const DEFAULT_METRIC = { diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 4a4ea9a544..254e87486c 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -1484,8 +1484,14 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query .handle("listMetrics", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context - const rows = yield* runQuery(Queries.listMetrics, tenant, payload) - return new ListMetricsResponse({ data: rows }) + const [rows, gcpRows] = yield* Effect.all( + [ + runQuery(Queries.listMetrics, tenant, payload), + queryEngine.listGcpMetrics(tenant, payload), + ], + { concurrency: "unbounded" }, + ) + return new ListMetricsResponse({ data: [...rows, ...gcpRows] }) }), ) .handle("metricsSummary", ({ payload }) => diff --git a/apps/api/src/routes/v2/telemetry.http.test.ts b/apps/api/src/routes/v2/telemetry.http.test.ts index 9dbce10cb0..db8f3ba2fa 100644 --- a/apps/api/src/routes/v2/telemetry.http.test.ts +++ b/apps/api/src/routes/v2/telemetry.http.test.ts @@ -2,7 +2,7 @@ import { afterEach, describe, expect, it } from "@effect/vitest" import { OrgId, UserId, WarehouseQueryError } from "@maple/domain/http" import { MapleApiV2 } from "@maple/domain/http/v2" import { QueryEngineExecuteResponse, type QueryEngineExecuteRequest } from "@maple/query-engine" -import { ConfigProvider, Context, Effect, Layer, ManagedRuntime, Option, Schema } from "effect" +import { ConfigProvider, Context, DateTime, Effect, Layer, ManagedRuntime, Option, Schema } from "effect" import { HttpRouter } from "effect/http" import { HttpApiBuilder } from "effect/http-api" import { Env } from "@maple/backend/platform/Env" @@ -296,6 +296,7 @@ const queryEngineStub = { evaluate: () => Effect.die(new Error("not used")), evaluateSeries: () => Effect.die(new Error("not used")), cachedDirect: (_tenant, _route, _payload, effect) => effect, + listGcpMetrics: () => Effect.succeed([]), } satisfies QueryEngineServiceApi const makeHarness = ( @@ -697,6 +698,46 @@ describe("v2 telemetry reads over HTTP", () => { await harness.dispose() }) + it("lists Google Cloud metrics after the warehouse's on the first page", async () => { + const gcpRequests: Array[1]> = [] + const queryEngine: QueryEngineServiceApi = { + ...queryEngineStub, + listGcpMetrics: (_tenant, request) => { + gcpRequests.push(request) + return Effect.succeed([ + { + metricName: "cloudsql.googleapis.com/database/cpu/utilization", + metricType: "gauge", + serviceName: "Google Cloud", + metricDescription: "CPU utilization", + metricUnit: "10^2.%", + dataPointCount: 0, + firstSeen: DateTime.makeUnsafe("2026-07-14T11:00:00Z"), + lastSeen: DateTime.makeUnsafe("2026-07-15T12:00:00Z"), + isMonotonic: 0, + }, + ]) + }, + } + const harness = makeHarness(warehouseStub, queryEngine) + const key = await harness.bootstrapKey() + + const metrics = await harness.request("GET", `/v2/metrics?${windowQuery}&search=cpu`, key.secret) + expect(metrics.status).toBe(200) + expect(metrics.body.data.map((metric: { name: string }) => metric.name)).toEqual([ + "http.server.duration", + "cloudsql.googleapis.com/database/cpu/utilization", + ]) + expect(metrics.body.data[1]).toMatchObject({ + type: "gauge", + service_name: "Google Cloud", + is_monotonic: false, + last_seen: "2026-07-15T12:00:00.000Z", + }) + expect(gcpRequests).toEqual([{ search: "cpu", limit: 20 }]) + await harness.dispose() + }) + // Regression: timeseries and breakdowns read the hourly rollups whenever the // filters allow, and those floor their bounds through `toDateTime` or compare // them against plain `DateTime` columns. Millisecond bounds made every diff --git a/apps/api/src/routes/v2/telemetry.http.ts b/apps/api/src/routes/v2/telemetry.http.ts index 1055116d30..f7bf931fb2 100644 --- a/apps/api/src/routes/v2/telemetry.http.ts +++ b/apps/api/src/routes/v2/telemetry.http.ts @@ -8,6 +8,7 @@ import { TraceId, } from "@maple/domain/http" import { + LIST_LIMIT_DEFAULT, MapleApiV2, paginateOffsetQuery, timestamp, @@ -63,6 +64,7 @@ import { type ServiceBaselines, } from "@maple/backend/services/warehouse/TelemetryReadService" import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" +import type { ListMetricsOutput } from "@maple/query-engine/ch" const decodeTraceId = Schema.decodeSync(TraceId) const decodeSpanId = Schema.decodeSync(SpanId) @@ -801,37 +803,52 @@ export const HttpV2MetricsLive = HttpApiBuilder.group(MapleApiV2, "metrics", (ha const window = yield* parseWindow(query.start_time, query.end_time, { precision: "second", }) - const page = yield* paginateOffsetQuery(query, ({ limit, offset }) => - telemetry - .listMetrics( - tenant, - { - serviceName: query.service_name, - metricType: query.metric_type, - search: query.search, - limit, - offset, - }, - window, - ) - .pipe( - Effect.map((rows): ReadonlyArray => - rows.map((row) => ({ - object: "metric", - name: decodeMetricName(row.metricName), - type: row.metricType, - service_name: row.serviceName, - description: row.metricDescription, - unit: row.metricUnit, - is_monotonic: row.isMonotonic !== 0, - data_point_count: row.dataPointCount, - first_seen: timestamp(DateTime.formatIso(row.firstSeen)), - last_seen: timestamp(DateTime.formatIso(row.lastSeen)), - })), - ), + const toMetric = (row: ListMetricsOutput): V2Metric => ({ + object: "metric", + name: decodeMetricName(row.metricName), + type: row.metricType, + service_name: row.serviceName, + description: row.metricDescription, + unit: row.metricUnit, + is_monotonic: row.isMonotonic !== 0, + data_point_count: row.dataPointCount, + first_seen: timestamp(DateTime.formatIso(row.firstSeen)), + last_seen: timestamp(DateTime.formatIso(row.lastSeen)), + }) + const [page, gcpRows] = yield* Effect.all( + [ + paginateOffsetQuery(query, ({ limit, offset }) => + telemetry + .listMetrics( + tenant, + { + serviceName: query.service_name, + metricType: query.metric_type, + search: query.search, + limit, + offset, + }, + window, + ) + .pipe(Effect.map((rows) => rows.map(toMetric))), ), + // Google Cloud metrics ride on the first page, after the warehouse's. + query.cursor === undefined + ? queryEngine.listGcpMetrics(tenant, { + service: query.service_name, + metricType: query.metric_type, + search: query.search, + limit: query.limit ?? LIST_LIMIT_DEFAULT, + }) + : Effect.succeed([]), + ], + { concurrency: "unbounded" }, ) - return { object: "list" as const, ...page } + return { + object: "list" as const, + ...page, + data: [...page.data, ...gcpRows.map(toMetric)], + } }), ) .handle("timeseries", ({ payload }) => diff --git a/apps/api/src/routes/v2/v2-test-support.ts b/apps/api/src/routes/v2/v2-test-support.ts index c7be1b4810..142d635ae3 100644 --- a/apps/api/src/routes/v2/v2-test-support.ts +++ b/apps/api/src/routes/v2/v2-test-support.ts @@ -322,6 +322,7 @@ export const TelemetryServiceStubsLayer = Layer.mergeAll( evaluate: die, evaluateSeries: die, cachedDirect: die, + listGcpMetrics: die, }), ) diff --git a/apps/api/src/routes/v2/widget-summary.http.test.ts b/apps/api/src/routes/v2/widget-summary.http.test.ts index 754ffe64fc..a7bd331428 100644 --- a/apps/api/src/routes/v2/widget-summary.http.test.ts +++ b/apps/api/src/routes/v2/widget-summary.http.test.ts @@ -128,6 +128,7 @@ const queryEngineStub = (execute: QueryEngineServiceApi["execute"]): QueryEngine evaluate: () => Effect.die(new Error("not used")), evaluateSeries: () => Effect.die(new Error("not used")), cachedDirect: (_tenant, _route, _payload, effect) => effect, + listGcpMetrics: () => Effect.die(new Error("not used")), }) /** Grouped requests carry a `service` group-by; the ungrouped one does not. */ diff --git a/apps/api/src/runtime/query-http-graph.ts b/apps/api/src/runtime/query-http-graph.ts index acd7e7257e..618eeebb39 100644 --- a/apps/api/src/runtime/query-http-graph.ts +++ b/apps/api/src/runtime/query-http-graph.ts @@ -14,6 +14,7 @@ import { SessionAuthorizationLive } from "@maple/backend/services/auth/SessionAu import { OrganizationRegionService } from "@maple/backend/services/org/OrganizationRegionService" import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" +import { GcpLiveMetricsPort } from "@maple/backend/services/warehouse/GcpLiveMetricsPort" // Select the already-decorated group: its session auth and error middleware // are exactly those of the complete internal API. The service key keeps the API id. @@ -39,7 +40,7 @@ export const QueryRoutes = Layer.mergeAll( WarehouseQueryService.layer, AuditLogService.layer, OrganizationRegionService.layer, - ), + ).pipe(Layer.provide(GcpLiveMetricsPort.layer)), ), Layer.provide(Layer.mergeAll(Env.layer, EdgeCacheServiceLive)), ) diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 807468dbc3..5f7d143da0 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -73,6 +73,7 @@ import { ProductEventsService } from "@maple/backend/services/product-events/Pro import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" +import { GcpLiveMetricsPort } from "@maple/backend/services/warehouse/GcpLiveMetricsPort" import { TelemetryReadService } from "@maple/backend/services/warehouse/TelemetryReadService" import { OrgClickHouseSettingsService } from "@maple/backend/services/org/OrgClickHouseSettingsService" import { VcsSourceService } from "@maple/backend/services/integrations/vcs/VcsSourceService" @@ -157,5 +158,7 @@ export const HttpServicesLive = Layer.mergeAll( ChatWorkspaceService.layer, ).pipe( Layer.provide(PullRequestLookupLive), + // Graph-wide, so every `QueryEngineService.layer` build (it is memoized) reads Google metrics. + Layer.provide(GcpLiveMetricsPort.layer), Layer.provideMerge(Layer.mergeAll(Env.layer, EdgeCacheServiceLive)), ) diff --git a/apps/landing/src/content/docs/integrations/gcp.md b/apps/landing/src/content/docs/integrations/gcp.md index 51c27570df..c723b17c10 100644 --- a/apps/landing/src/content/docs/integrations/gcp.md +++ b/apps/landing/src/content/docs/integrations/gcp.md @@ -150,6 +150,18 @@ One-minute resolution, named like OpenTelemetry metrics under `gcp.`: Every hour Maple lists projects, Cloud Run services and jobs, Cloud Functions, GKE clusters, Compute Engine instances, Cloud SQL instances, Pub/Sub topics and subscriptions, and load balancers from Cloud Asset Inventory, with their location, state and labels. +## Query any Cloud Monitoring metric live + +Besides the `gcp.*` metrics Maple stores, you can query any Cloud Monitoring metric with data in the last 25 hours. Maple reads it live from Google. Use its metric type as the metric name, for example `cloudsql.googleapis.com/database/cpu/utilization`, in dashboards, alert rules and the MCP tools. The metrics explorer and the metric pickers list these metrics under the service **Google Cloud**. It needs a connection with **Metrics and resources** on. + +- Metric and resource labels form one set. Filter and group by them as `attr.( + listPage: ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, + pageToken: string | undefined, + ) => Effect.Effect< + { readonly items: ReadonlyArray; readonly nextPageToken?: string | undefined }, + GcpApiError + >, + reader: GcpReader, + projectId: GcpProjectId, + ) { + const items: Array = [] let pageToken: string | undefined - for (let page = 0; page < MAX_DESCRIPTOR_PAGES; page++) { - const body = yield* listActiveMetricDescriptorsPage( - httpClient, - reader, - projectId, - pageToken, - ) - descriptors.push(...body.descriptors) + for (let page = 0; page < MAX_LIST_PAGES; page++) { + const body = yield* listPage(httpClient, reader, projectId, pageToken) + items.push(...body.items) pageToken = body.nextPageToken if (pageToken === undefined) break } - return descriptors + return items }) return { @@ -521,7 +537,7 @@ export class GcpLiveMetricsService extends Context.Service value), @@ -543,6 +559,39 @@ export class GcpLiveMetricsService extends Context.Service = [...selected.failures] + const signedIn = yield* signIn(key, selected.targets, failures) + // The types are Google's, the same in every project: the first project that answers speaks. + for (const target of signedIn) { + const types = yield* edgeCache + .getOrCompute( + { + bucket: "gcp-resource-types", + key: `${tenant.orgId}:${target.projectId}`, + ttlSeconds: DAY_SECONDS, + readTimeoutMs: CACHE_READ_TIMEOUT_MS, + schema: Schema.Array(GcpMonitoredResourceType), + }, + listAll(listMonitoredResourceTypesPage, target.reader, target.projectId), + ) + .pipe( + Effect.map(({ value }) => value), + Effect.catchTag( + API_ERROR, + recordFailure(failures, `projects/${target.projectId}`, target.connector), + ), + ) + if (types !== undefined) return types + } + yield* logFailures(failures) + return failures.length > 0 ? yield* readFailed(failures) : [] + }), } satisfies Port }), }, diff --git a/packages/backend/src/services/integrations/gcp/api.ts b/packages/backend/src/services/integrations/gcp/api.ts index 1d85a24905..be6b74e468 100644 --- a/packages/backend/src/services/integrations/gcp/api.ts +++ b/packages/backend/src/services/integrations/gcp/api.ts @@ -16,7 +16,11 @@ import { type GcpMetric, type GcpMetricGroup, } from "@maple/domain/gcp-metrics" -import { GcpMetricDescriptor, type GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import { + GcpMetricDescriptor, + type GcpMonitoredResourceType, + type GcpPromQlSeries, +} from "@maple/domain/gcp-live-metrics" import { GcpLiveQueryRejectedError } from "@maple/domain/http/gcp-live-query-errors" import type { GcpConnectorId, GcpProjectId } from "@maple/domain/primitives" import { Duration, Effect, Option, Redacted, Schema, Stream } from "effect" @@ -627,7 +631,7 @@ export const listActiveMetricDescriptorsPage = ( { maxBytes: MAX_BODY_BYTES }, ).pipe( Effect.map((page) => ({ - descriptors: (page.metricDescriptors ?? []).flatMap((descriptor): Array => { + items: (page.metricDescriptors ?? []).flatMap((descriptor): Array => { const { metricKind = "", valueType = "" } = descriptor if (!isMetricKind(metricKind) || !isValueType(valueType)) return [] return [ @@ -646,6 +650,45 @@ export const listActiveMetricDescriptorsPage = ( })), ) +const decodeResourceDescriptorsPage = Schema.decodeUnknownEffect( + Schema.Struct({ + resourceDescriptors: Schema.optionalKey( + Schema.Array( + Schema.Struct({ + type: Schema.String, + labels: Schema.optionalKey(Schema.Array(Schema.Struct({ key: Schema.String }))), + }), + ), + ), + nextPageToken: Schema.optionalKey(Schema.String), + }), +) + +/** One page of Google's monitored resource types, each with its label keys. */ +export const listMonitoredResourceTypesPage = ( + httpClient: HttpClient.HttpClient, + reader: GcpReader, + projectId: GcpProjectId, + pageToken: string | undefined, +) => + call( + httpClient, + "Cloud Monitoring", + HttpClientRequest.get( + `https://monitoring.googleapis.com/v3/projects/${projectId}/monitoredResourceDescriptors`, + ).pipe(asReader(reader), HttpClientRequest.setUrlParams({ pageToken })), + decodeResourceDescriptorsPage, + { maxBytes: MAX_BODY_BYTES }, + ).pipe( + Effect.map((page) => ({ + items: (page.resourceDescriptors ?? []).map((descriptor): GcpMonitoredResourceType => ({ + type: descriptor.type, + labels: (descriptor.labels ?? []).map((label) => label.key), + })), + nextPageToken: page.nextPageToken, + })), + ) + // Cloud Asset Inventory const ResourceSearchResult = Schema.Struct({ diff --git a/packages/backend/src/services/warehouse/gcp-metric-catalog.test.ts b/packages/backend/src/services/warehouse/gcp-metric-catalog.test.ts index 58da8d52c6..6ca7e4ae07 100644 --- a/packages/backend/src/services/warehouse/gcp-metric-catalog.test.ts +++ b/packages/backend/src/services/warehouse/gcp-metric-catalog.test.ts @@ -47,6 +47,7 @@ const fakePort = (answer: Effect.Effect, GcpL const port: GcpLiveMetrics = { queryRange: unused, queryInstant: unused, + listResourceTypes: unused, listMetrics: (_tenant, scope) => { calls.listMetrics += 1 calls.scopes.push(scope) diff --git a/packages/domain/src/gcp-live-metrics.ts b/packages/domain/src/gcp-live-metrics.ts index 3c67b29bf0..23401573bc 100644 --- a/packages/domain/src/gcp-live-metrics.ts +++ b/packages/domain/src/gcp-live-metrics.ts @@ -25,6 +25,13 @@ export const GcpMetricDescriptor = Schema.Struct({ }) export type GcpMetricDescriptor = typeof GcpMetricDescriptor.Type +/** A monitored resource type (Cloud Monitoring `monitoredResourceDescriptors`) and its label keys. */ +export const GcpMonitoredResourceType = Schema.Struct({ + type: Schema.String, + labels: Schema.Array(Schema.String), +}) +export type GcpMonitoredResourceType = typeof GcpMonitoredResourceType.Type + /** How Maple's query builder treats a Cloud Monitoring metric; undefined for value types PromQL cannot chart. */ export const gcpMetricMapleType = (descriptor: GcpMetricDescriptor): MetricType | undefined => { if (descriptor.valueType === "STRING" || descriptor.valueType === "MONEY") return undefined diff --git a/packages/query-engine/src/promql/compile.test.ts b/packages/query-engine/src/promql/compile.test.ts index e0028aa555..8d2b3b2f4e 100644 --- a/packages/query-engine/src/promql/compile.test.ts +++ b/packages/query-engine/src/promql/compile.test.ts @@ -5,7 +5,6 @@ import type { AttributeFilter, MetricsFilters, MetricsMetric } from "@maple/doma import { Effect } from "effect" import { QueryEngineValidationError } from "@maple/domain/http" import { - compileGcpLabelSample, compileGcpLabelValues, compileGcpMetricQuery, compileGcpSampleCount, @@ -103,8 +102,8 @@ describe("compileGcpMetricQuery", () => { [ "avg", avgParts( - `sum by (project_id) (avg_over_time(${CPU}[60s]))`, - `count by (project_id) (avg_over_time(${CPU}[60s]))`, + `sum by (project_id) (sum_over_time(${CPU}[60s]))`, + `sum by (project_id) (count_over_time(${CPU}[60s]))`, ), "avg", ], @@ -472,15 +471,6 @@ describe("compileGcpSampleCount", () => { }) describe("discovery", () => { - it.effect("samples recent series of a distribution through its _count series", () => - Effect.gen(function* () { - assert.strictEqual( - yield* compileGcpLabelSample(latencies), - `topk(5, last_over_time(${LAT("count")}[3600s]))`, - ) - }), - ) - it.effect("counts series per label value and project", () => Effect.gen(function* () { assert.strictEqual( diff --git a/packages/query-engine/src/promql/compile.ts b/packages/query-engine/src/promql/compile.ts index dd463f2d44..bd1264b59d 100644 --- a/packages/query-engine/src/promql/compile.ts +++ b/packages/query-engine/src/promql/compile.ts @@ -149,7 +149,7 @@ const byLabels = (groupLabel: string | undefined): string => ? "project_id" : `${labelName(groupLabel)}, project_id` -/** The series discovery and sample counts read: a distribution answers as `_count` / `_sum`. */ +/** The series label values and sample counts read: a distribution answers as `_count` / `_sum`. */ const seriesName = (descriptor: GcpMetricDescriptor): string => descriptor.valueType === "DISTRIBUTION" ? `${descriptor.type}_count` : descriptor.type @@ -242,10 +242,11 @@ export const compileGcpMetricQuery = Effect.fnUntraced(function* ( const gauge = descriptor.metricKind === "GAUGE" switch (metric) { case "avg": + // Every sample weighs the same, as in ClickHouse: not a mean of per-series means. return { query: avgParts( - aggregate("sum", type, "avg_over_time"), - aggregate("count", type, "avg_over_time"), + aggregate("sum", type, "sum_over_time"), + aggregate("sum", type, "count_over_time"), ), merge: "avg", } satisfies GcpPromQl @@ -286,12 +287,6 @@ export const compileGcpSampleCount = Effect.fnUntraced(function* ( } satisfies GcpPromQl }) -/** A few recent series of the metric, to read which labels it carries. */ -export const compileGcpLabelSample = Effect.fnUntraced(function* (descriptor: GcpMetricDescriptor) { - const union = yield* compileUnion(descriptor, []) - return `topk(5, ${union(seriesName(descriptor), "last_over_time", 3_600)})` -}) - /** Series per value of one label (and project) over the window. */ export const compileGcpLabelValues = Effect.fnUntraced(function* ( descriptor: GcpMetricDescriptor, diff --git a/packages/query-engine/src/runtime/gcp-metrics.test.ts b/packages/query-engine/src/runtime/gcp-metrics.test.ts index 9627d9833b..853e1d31b0 100644 --- a/packages/query-engine/src/runtime/gcp-metrics.test.ts +++ b/packages/query-engine/src/runtime/gcp-metrics.test.ts @@ -42,11 +42,12 @@ interface Calls { readonly range: Array[1]> readonly instant: Array[1]> readonly listed: Array + readonly resourceTypes: Array warehouse: number } const makeFakes = (answer: (query: string) => ReadonlyArray = () => []) => { - const calls: Calls = { range: [], instant: [], listed: [], warehouse: 0 } + const calls: Calls = { range: [], instant: [], listed: [], resourceTypes: [], warehouse: 0 } const gcp: GcpLiveMetrics = { queryRange: (_tenant, request) => { calls.range.push(request) @@ -60,6 +61,13 @@ const makeFakes = (answer: (query: string) => ReadonlyArray = ( calls.listed.push(request) return Effect.succeed([transactions, latencies]) }, + listResourceTypes: (_tenant, request) => { + calls.resourceTypes.push(request) + return Effect.succeed([ + { type: "cloudsql_database", labels: ["project_id", "database_id", "region"] }, + { type: "gce_instance", labels: ["project_id", "instance_id", "zone"] }, + ]) + }, } const warehouse: QueryEngineWarehouse = { rawSqlQuery: () => Effect.die("unexpected rawSqlQuery"), @@ -90,9 +98,8 @@ const execute = ( }), ) -type DataOf = Extract extends { readonly data: infer Data } - ? Data - : never +type DataOf = + Extract extends { readonly data: infer Data } ? Data : never const dataOf = ( response: { readonly result: QueryEngineResult }, @@ -227,6 +234,19 @@ describe("Google Cloud metrics routing", () => { }), ) + it.effect("averages every sample alike, not each series' mean", () => + Effect.gen(function* () { + // One series with a single 0 and another with five 10s: Google sums 50 over 6 samples. + const fakes = makeFakes(() => [ + series({ project_id: "alpha", maple_part: "sum" }, [["2026-10-19T10:05:00Z", 50]]), + series({ project_id: "alpha", maple_part: "count" }, [["2026-10-19T10:05:00Z", 6]]), + ]) + const response = yield* execute(fakes, timeseries({ metric: "avg" })) + assert.include(fakes.calls.range[0]?.query ?? "", "sum by (project_id) (count_over_time(") + assert.deepStrictEqual(dataOf(response, "timeseries")[0]?.series, { all: 50 / 6 }) + }), + ) + it.effect("caps the merged groups to the highest peaks", () => Effect.gen(function* () { const fakes = makeFakes(() => [ @@ -377,7 +397,9 @@ describe("Google Cloud metric discovery", () => { metricName: TX, }) assert.isTrue( - [...fakes.calls.listed, ...fakes.calls.instant].every((call) => call.discovery === true), + [...fakes.calls.listed, ...fakes.calls.resourceTypes, ...fakes.calls.instant].every( + (call) => call.discovery === true, + ), ) }), ) @@ -392,7 +414,7 @@ describe("Google Cloud metric discovery", () => { ) assert.deepStrictEqual( dataOf(keys, "attributeKeys").map((item) => item.key), - ["log", "monitored_resource", "project_id", "severity"], + ["severity", "log", "project_id", "monitored_resource"], ) const values = yield* execute( { warehouse: fakes.warehouse, gcp }, @@ -406,6 +428,7 @@ describe("Google Cloud metric discovery", () => { ) assert.deepStrictEqual(dataOf(values, "attributeValues"), []) assert.lengthOf(fakes.calls.instant, 0) + assert.lengthOf(fakes.calls.resourceTypes, 0) // A chart still asks for the resource type. const chart = yield* Effect.flip( @@ -416,20 +439,16 @@ describe("Google Cloud metric discovery", () => { }), ) - it.effect("lists the descriptor's labels and the labels seen on recent series", () => + it.effect("lists the metric's labels, then its resource types' labels, without reading series", () => Effect.gen(function* () { - const fakes = makeFakes(() => [ - series( - { __name__: TX, project_id: "alpha", database_id: "a", transaction_type: "commit" }, - [], - ), - series({ __name__: TX, project_id: "alpha", database_id: "b" }, []), - ]) + const fakes = makeFakes() const all = yield* execute(fakes, { kind: "attributeKeys", source: "metrics", metricName: TX }) assert.deepStrictEqual(dataOf(all, "attributeKeys"), [ - { key: "database_id", count: 2 }, - { key: "project_id", count: 2 }, - { key: "transaction_type", count: 1 }, + { key: "transaction_type", count: 0 }, + { key: "project_id", count: 0 }, + { key: "database_id", count: 0 }, + { key: "region", count: 0 }, + { key: "monitored_resource", count: 0 }, ]) const resource = yield* execute(fakes, { kind: "attributeKeys", @@ -439,8 +458,9 @@ describe("Google Cloud metric discovery", () => { }) assert.deepStrictEqual( dataOf(resource, "attributeKeys").map((item) => item.key), - ["database_id", "project_id"], + ["project_id", "database_id", "region", "monitored_resource"], ) + assert.lengthOf(fakes.calls.instant, 0) }), ) diff --git a/packages/query-engine/src/runtime/gcp-metrics.ts b/packages/query-engine/src/runtime/gcp-metrics.ts index db4ff89a0c..ca9a1b6ca7 100644 --- a/packages/query-engine/src/runtime/gcp-metrics.ts +++ b/packages/query-engine/src/runtime/gcp-metrics.ts @@ -10,7 +10,6 @@ import { import type { MetricsFilters, QuerySpec } from "@maple/domain/query-engine" import { ENGINE_UNGROUPED_GROUP_KEY } from "../group-key" import { - compileGcpLabelSample, compileGcpLabelValues, compileGcpMetricQuery, compileGcpSampleCount, @@ -272,7 +271,11 @@ export const gcpMetricsAlertBuckets = Effect.fnUntraced(function* ( gcp: GcpLiveMetrics | undefined, tenant: T, @@ -280,38 +283,20 @@ export const gcpMetricAttributeKeys = Effect.fnUntraced(function* ({ key, count: 0 })) - .slice(0, request.limit ?? 200) - } - const sample = yield* compileGcpLabelSample(descriptor) - yield* annotate(sample) - const series = yield* port.queryInstant(tenant, { - ...read, - query: sample, - timeSeconds: Math.floor(request.endMs / 1000), - }) - const counts = new Map(resourceOnly ? [] : descriptor.labels.map((label) => [label, 0])) - for (const s of series) { - for (const key of Object.keys(s.labels)) { - if (key === "__name__" || (resourceOnly && descriptor.labels.includes(key))) continue - counts.set(key, (counts.get(key) ?? 0) + 1) - } - } - return [...counts] - .map(([key, count]) => ({ key, count })) - .sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)) - .slice(0, request.limit ?? 200) + const active = new Set(descriptor.monitoredResourceTypes) + const resourceTypes = active.size === 0 ? [] : yield* port.listResourceTypes(tenant, read) + const resourceLabels = resourceTypes.flatMap((type) => (active.has(type.type) ? type.labels : [])) + const keys = new Set([ + ...(request.scope === "resource" ? [] : descriptor.labels), + ...resourceLabels, + "project_id", + "monitored_resource", + ]) + return [...keys].slice(0, request.limit ?? 200).map((key) => ({ key, count: 0 })) }) /** Values of one label with their series counts over the range, summed across projects. */ diff --git a/packages/query-engine/src/runtime/live-metrics.ts b/packages/query-engine/src/runtime/live-metrics.ts index 27e80e21aa..6f910a17ff 100644 --- a/packages/query-engine/src/runtime/live-metrics.ts +++ b/packages/query-engine/src/runtime/live-metrics.ts @@ -1,5 +1,9 @@ import type { Effect } from "effect" -import type { GcpMetricDescriptor, GcpPromQlSeries } from "@maple/domain/gcp-live-metrics" +import type { + GcpMetricDescriptor, + GcpMonitoredResourceType, + GcpPromQlSeries, +} from "@maple/domain/gcp-live-metrics" import type { GcpLiveQueryError } from "@maple/domain/http/gcp-live-query-errors" import type { QueryTenant } from "./query-engine" @@ -44,4 +48,9 @@ export interface GcpLiveMetrics { tenant: T, request: GcpReadScope, ) => Effect.Effect, GcpLiveQueryError> + /** Google's monitored resource types with their label keys (cached a day). */ + readonly listResourceTypes: ( + tenant: T, + request: GcpReadScope, + ) => Effect.Effect, GcpLiveQueryError> } diff --git a/packages/query-engine/src/runtime/query-engine.ts b/packages/query-engine/src/runtime/query-engine.ts index fe77287dc5..38e3fb6305 100644 --- a/packages/query-engine/src/runtime/query-engine.ts +++ b/packages/query-engine/src/runtime/query-engine.ts @@ -2155,7 +2155,6 @@ export const makeQueryEngineExecute = ( metricName: request.query.metricName, scope: request.query.scope, limit: request.query.limit, - endMs: range.endMs, }) return new QueryEngineExecuteResponse({ result: { kind: "attributeKeys", source: "metrics", data }, From c4c7c072c6bc8ef64b53f64a2cf426ea709e81be Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Mon, 19 Oct 2026 19:31:30 +0200 Subject: [PATCH 12/16] fix(gcp): stop live reads when metrics are switched off, widen gauge rate windows, show label lookup failures --- apps/landing/src/content/docs/integrations/gcp.md | 2 +- apps/web/src/components/metrics/metric-breakdown.tsx | 7 +++++++ .../integrations/GcpLiveMetricsService.test.ts | 12 ++++++++++-- .../services/integrations/GcpLiveMetricsService.ts | 6 +++++- packages/query-engine/src/promql/compile.test.ts | 4 ++-- packages/query-engine/src/promql/compile.ts | 4 ++-- 6 files changed, 27 insertions(+), 8 deletions(-) diff --git a/apps/landing/src/content/docs/integrations/gcp.md b/apps/landing/src/content/docs/integrations/gcp.md index e2a523c015..2028ccdb96 100644 --- a/apps/landing/src/content/docs/integrations/gcp.md +++ b/apps/landing/src/content/docs/integrations/gcp.md @@ -152,7 +152,7 @@ Every hour Maple lists projects, Cloud Run services and jobs, Cloud Functions, G ## Query any Cloud Monitoring metric live -With **Metrics and resources** on, you can chart and alert on any Cloud Monitoring metric that had data in the last day. Maple reads it live from Google. The metrics explorer lists them under the service **Google Cloud**. In dashboards, alert rules and the MCP tools, use the metric type as the metric name, e.g. `cloudsql.googleapis.com/database/cpu/utilization`. +With **Metrics and resources** on, you can chart and alert on any Cloud Monitoring metric with numeric, boolean or distribution values that had data in the last day. Maple reads it live from Google. The metrics explorer lists them under the service **Google Cloud**. In dashboards, alert rules and the MCP tools, use the metric type as the metric name, e.g. `cloudsql.googleapis.com/database/cpu/utilization`. - Filter and group by the metric's labels as `attr.