feat(infra): Infrastructure → Google Cloud page

.github/workflows/ci.yml

@@ -122,6 +122,8 @@ jobs:
                         # it. Without these two the PR that broke every main chart
                         # would still skip the only job that could have caught it.
                         - 'packages/query-engine/**'
+                        # The sweep analyzes the integration builders too.
+                        - 'packages/query-engine-integrations/**'
                         - 'packages/domain/src/clickhouse/**'
                         - 'packages/domain/src/tinybird/**'
                         # The migrations this job replays, and the datasource

apps/api/src/routes/internal/integrations.http.ts

@@ -10,6 +10,7 @@ import {
 	CloudflareUsageResponse,
 	CurrentTenant,
 	ExternalUserId,
+	GcpResourcesResponse,
 	GithubDeleteRepositoryResponse,
 	GithubDisconnectResponse,
 	GithubIntegrationStatus,
@@ -54,6 +55,7 @@ import {
 	topTrafficQuery,
 	type TopTrafficGroupDefinition,
 } from "@maple/backend/services/integrations/cloudflare-analytics/queries"
+import { GcpConnectorService } from "@maple/backend/services/integrations/GcpConnectorService"
 import { RailwayMetricsService } from "@maple/backend/services/integrations/RailwayMetricsService"
 import { GithubConnectService } from "@maple/backend/services/integrations/vcs/vendor/github/GithubConnectService"
 import { VcsCommitService } from "@maple/backend/services/integrations/vcs/VcsCommitService"
@@ -118,6 +120,7 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte
 		const cloudflare = yield* CloudflareOAuthService
 		const cloudflareAnalytics = yield* CloudflareAnalyticsService
 		const railway = yield* RailwayMetricsService
+		const gcp = yield* GcpConnectorService
 		const edgeCache = yield* EdgeCacheService
 		const env = yield* Env
 
@@ -464,6 +467,12 @@ export const HttpIntegrationsLive = HttpApiBuilder.group(MapleInternalApi, "inte
 						return new RailwayDisconnectResponse(yield* railway.disconnect(tenant.orgId))
 					}),
 				)
+				.handle("gcpResources", ({ query }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						return new GcpResourcesResponse(yield* gcp.resources(tenant.orgId, query))
+					}),
+				)
 				.handle("githubStatus", () =>
 					Effect.gen(function* () {
 						const tenant = yield* CurrentTenant.Context

apps/api/src/routes/internal/query-engine.http.ts

@@ -19,6 +19,12 @@ import {
 	ServiceApdexResponse,
 	PlanetScaleInfraTimeseriesResponse,
 	RailwayInfraRowsResponse,
+	GcpInfraMetricsResponse,
+	GcpInfraPresenceResponse,
+	GcpInfraSparklinesResponse,
+	GcpCloudSqlDatabasesResponse,
+	GcpCloudSqlTopQueriesResponse,
+	GcpInfraTimeseriesResponse,
 	ServiceCloudflareStatsResponse,
 	ServicePlanetScaleStatsResponse,
 	CloudflareInfraZonesResponse,
@@ -97,6 +103,7 @@ import {
 	TraceId,
 	SpanId,
 } from "@maple/domain/http"
+import { GCP_INFRA_SERVICE_IDS, gcpInfraMetrics } from "@maple/domain/gcp-infra"
 import { SESSION_LIVE_WINDOW_SECONDS } from "@maple/domain/query-engine"
 import { isAiContentFormat } from "@maple/domain/ai-traffic"
 import { Cause, Clock, DateTime, Effect, Option, Schema } from "effect"
@@ -848,6 +855,57 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query
 						})
 					}),
 				)
+				.handle("gcpInfraPresence", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.gcpInfraPresence, tenant, payload)
+						const reported = new Set(rows.map((row) => row.metric))
+						return new GcpInfraPresenceResponse({
+							services: GCP_INFRA_SERVICE_IDS.filter((service) =>
+								gcpInfraMetrics(service).some((metric) => reported.has(metric.name)),
+							),
+						})
+					}),
+				)
+				.handle("gcpInfraMetrics", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.gcpInfraMetrics, tenant, payload)
+						return new GcpInfraMetricsResponse({ data: rows })
+					}),
+				)
+				.handle("gcpInfraTimeseries", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.gcpInfraTimeseries, tenant, payload)
+						return new GcpInfraTimeseriesResponse({
+							data: rows.map((row) => ({ ...row, bucket: DateTime.formatIso(row.bucket) })),
+						})
+					}),
+				)
+				.handle("gcpInfraSparklines", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.gcpInfraSparklines, tenant, payload)
+						return new GcpInfraSparklinesResponse({
+							data: rows.map((row) => ({ ...row, bucket: DateTime.formatIso(row.bucket) })),
+						})
+					}),
+				)
+				.handle("gcpCloudSqlTopQueries", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.gcpCloudSqlTopQueries, tenant, payload)
+						return new GcpCloudSqlTopQueriesResponse({ data: rows })
+					}),
+				)
+				.handle("gcpCloudSqlDatabases", ({ payload }) =>
+					Effect.gen(function* () {
+						const tenant = yield* CurrentTenant.Context
+						const rows = yield* runQuery(Queries.gcpCloudSqlDatabases, tenant, payload)
+						return new GcpCloudSqlDatabasesResponse({ databases: rows.map((row) => row.database) })
+					}),
+				)
 				.handle("cloudflareInfraZones", ({ payload }) =>
 					Effect.gen(function* () {
 						const tenant = yield* CurrentTenant.Context

apps/api/src/routes/queries.ts

@@ -16,6 +16,11 @@ import type {
 	CloudflareInfraZoneTimeseriesRequest,
 	CloudflareInfraZonesRequest,
 	ContainerInfraTimeseriesRequest,
+	GcpInfraMetricsRequest,
+	GcpInfraPresenceRequest,
+	GcpCloudSqlInstanceRequest,
+	GcpInfraSparklinesRequest,
+	GcpInfraTimeseriesRequest,
 	GetLogRequest,
 	NodeInfraTimeseriesRequest,
 	PlanetScaleInfraTimeseriesRequest,
@@ -289,6 +294,83 @@ const railwayInfraServiceTimeseries = defineQuery({
 		}),
 })
 
+// A small read of the hourly catalog that decides which tabs /infra/gcp shows.
+const gcpInfraPresence = defineQuery({
+	id: "gcpInfraPresence",
+	profile: "discovery",
+	cache: 60,
+	compile: (payload: GcpInfraPresenceRequest, orgId: OrgId) =>
+		CH.compile(Integrations.gcpInfraPresenceSQL(), {
+			orgId,
+			startTime: payload.startTime,
+			endTime: payload.endTime,
+		}),
+})
+
+const gcpInfraMetrics = defineQuery({
+	id: "gcpInfraMetrics",
+	profile: "aggregation",
+	cache: 15,
+	compile: (payload: GcpInfraMetricsRequest, orgId: OrgId) =>
+		CH.compile(Integrations.gcpInfraMetricsSQL(payload.service), {
+			orgId,
+			startTime: payload.startTime,
+			endTime: payload.endTime,
+		}),
+})
+
+const gcpInfraTimeseries = defineQuery({
+	id: "gcpInfraTimeseries",
+	profile: "aggregation",
+	cache: 15,
+	compile: (payload: GcpInfraTimeseriesRequest, orgId: OrgId) =>
+		CH.compile(Integrations.gcpInfraTimeseriesSQL(payload.source, payload.keys), {
+			orgId,
+			startTime: payload.startTime,
+			endTime: payload.endTime,
+			// The poller stores one point per minute.
+			bucketSeconds: Math.max(60, payload.bucketSeconds),
+		}),
+})
+
+const gcpInfraSparklines = defineQuery({
+	id: "gcpInfraSparklines",
+	profile: "aggregation",
+	cache: 15,
+	compile: (payload: GcpInfraSparklinesRequest, orgId: OrgId) =>
+		CH.compile(Integrations.gcpInfraSparklinesSQL(payload.service), {
+			orgId,
+			startTime: payload.startTime,
+			endTime: payload.endTime,
+			// The poller stores one point per minute.
+			bucketSeconds: Math.max(60, payload.bucketSeconds),
+		}),
+})
+
+const gcpCloudSqlTopQueries = defineQuery({
+	id: "gcpCloudSqlTopQueries",
+	profile: "aggregation",
+	cache: 15,
+	compile: (payload: GcpCloudSqlInstanceRequest, orgId: OrgId) =>
+		CH.compile(Integrations.gcpCloudSqlTopQueriesSQL(payload.keys), {
+			orgId,
+			startTime: payload.startTime,
+			endTime: payload.endTime,
+		}),
+})
+
+const gcpCloudSqlDatabases = defineQuery({
+	id: "gcpCloudSqlDatabases",
+	profile: "list",
+	cache: 15,
+	compile: (payload: GcpCloudSqlInstanceRequest, orgId: OrgId) =>
+		CH.compile(Integrations.gcpCloudSqlDatabasesSQL(payload.keys), {
+			orgId,
+			startTime: payload.startTime,
+			endTime: payload.endTime,
+		}),
+})
+
 const zoneDetailParams = (payload: CloudflareInfraZoneDetailRequest, orgId: OrgId) => ({
 	orgId,
 	serviceName: payload.serviceName,
@@ -808,6 +890,12 @@ export const Queries = {
 	planetscaleInfraTimeseries,
 	railwayInfraServices,
 	railwayInfraServiceTimeseries,
+	gcpInfraPresence,
+	gcpInfraMetrics,
+	gcpInfraTimeseries,
+	gcpInfraSparklines,
+	gcpCloudSqlTopQueries,
+	gcpCloudSqlDatabases,
 
 	// ZoneDetail / PlanetScaleStats sub-queries, declared above.
 	cloudflareInfraZoneDetailStatus,

apps/api/src/routes/v2/integrations-gcp.http.test.ts

@@ -213,7 +213,7 @@ describe("v2 gcp integration over HTTP", () => {
 		const path = `${CONNECTORS}/${created.body.id}`
 
 		const both = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {
-			log_filter: "include_gke_container_logs",
+			application_logs: ["cloud_run", "cloud_functions", "app_engine", "gke"],
 		})
 		expect(both.status).toBe(200)
 		expect(Object.keys(both.body).sort()).toEqual(["cleanup_script", "object", "setup_script"])
@@ -236,13 +236,13 @@ describe("v2 gcp integration over HTTP", () => {
 		// The cleanup script tells Maple that it ran, with the connector's secret.
 		expect(both.body.cleanup_script).toContain("?secret=maple_gcp_")
 
-		// Without an option an existing sink keeps its filter and a new one gets the default, which
+		// Without the list an existing sink keeps its filter and a new one gets the default, which
 		// leaves out GKE container logs; with logs off there is none to carry.
 		const kept = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {})
 		expect(kept.body.setup_script).toContain("LOG_FILTER_MODE='keep'")
 		expect(kept.body.setup_script).toContain(` AND NOT resource.type="k8s_container"'\n`)
 		const unknown = await harness.request("POST", `${path}/setup_scripts`, admin.secret, {
-			log_filter: "everything",
+			application_logs: ["everything"],
 		})
 		expect(unknown.status).toBe(400)
 

apps/api/src/routes/v2/integrations-gcp.http.ts

@@ -103,7 +103,7 @@ export const HttpV2GcpIntegrationsLive = HttpApiBuilder.group(MapleApiV2, "gcpIn
 					const tenant = yield* CurrentTenant.Context
 					yield* requireAdmin(tenant.roles, adminOnly)
 					const scripts = yield* gcp.scripts(tenant.orgId, params.id, {
-						logFilter: payload.log_filter ?? "keep",
+						applicationLogs: payload.application_logs ?? null,
 					})
 					return {
 						object: "gcp_connector.setup_scripts" as const,

apps/landing/src/content/docs/integrations/gcp-opentelemetry.md

@@ -20,15 +20,15 @@ Your workloads send traces, logs and metrics straight to Maple over OpenTelemetr
 ### GKE
 
 1. Instrument each service with an OpenTelemetry SDK. Export to Maple, or to the [Maple Kubernetes collector](/docs/infrastructure/kubernetes) in the cluster.
-2. [Connect Google Cloud](/docs/integrations/gcp#connect) and keep the log filter on **Recommended: no GKE container logs**.
+2. [Connect Google Cloud](/docs/integrations/gcp#connect) and leave **Log filter** as it is: no application output is forwarded.
 
 The connection then adds node and cluster logs, audit logs and the `gcp.gke.*` metrics.
 
 ### Cloud Run
 
 1. Instrument the service with an OpenTelemetry SDK that exports to Maple, with the settings under [Instrument a Cloud Run service](#instrument-a-cloud-run-service).
 2. [Connect Google Cloud](/docs/integrations/gcp#connect).
-3. If the service exports logs over OpenTelemetry, [leave out its container output](#change-the-filter) too: the recommended filter forwards what a Cloud Run container writes to stdout and stderr.
+3. If the service exports logs over OpenTelemetry, untick **Cloud Run** under [**Log filter**](#change-the-filter): by default the connection forwards what a Cloud Run container writes to stdout and stderr.
 
 The connection adds Cloud Run's request logs and the `gcp.cloud_run.*` metrics. Cloud Run sends a `traceparent` header with every request. An SDK that continues it gives the server span the trace ID of Google's request log for that request, so the request log, your spans and your log lines join on one trace.
 
@@ -40,35 +40,39 @@ The connection adds Cloud Run's request logs and the `gcp.cloud_run.*` metrics.
 
 The connection adds audit logs and the `gcp.gce.*` metrics.
 
-## GKE container logs
+## Application output
 
-The recommended filter leaves out `resource.type="k8s_container"`: what the containers of a GKE cluster write to stdout and stderr.
+Application output is what the code of a runtime writes to stdout and stderr. The connection forwards none by default: send it over OpenTelemetry. Platform, request and audit logs are forwarded either way. For workloads that can't, click **Can't forward via OpenTelemetry** under **Log filter** and tick their runtimes.
 
-| Reason        | Detail                                                                                                                                                                       |
-| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Duplicates    | A workload that sends its logs over OpenTelemetry, or a collector that reads pod logs, already delivers each line to Maple. Forwarding it from Google Cloud stores it twice. |
-| No trace link | The OpenTelemetry copy carries the trace and span ID of the request that wrote it. The copy from Google Cloud has neither.                                                   |
-| Volume        | Container output grows with traffic. Every forwarded line counts toward your Maple plan and toward the Pub/Sub usage Google bills to your account.                           |
+| Runtime             | Left out when unticked                                                                                | Still forwarded                          |
+| ------------------- | ----------------------------------------------------------------------------------------------------- | ---------------------------------------- |
+| **Cloud Run**       | `run.googleapis.com/stdout` and `stderr` of services, jobs, worker pools and 2nd gen functions        | Request logs, system logs, audit logs    |
+| **Cloud Functions** | `cloudfunctions.googleapis.com/cloud-functions` of 1st gen functions, with its start and finish lines | Audit logs                               |
+| **App Engine**      | `stdout` and `stderr` of `gae_app`                                                                    | Request logs, audit logs                 |
+| **GKE containers**  | Everything of `resource.type="k8s_container"`                                                         | Node, pod and cluster events, audit logs |
 
-Include GKE container logs when the workloads don't send their logs over OpenTelemetry and no collector reads them. Cloud Logging is then the only place those lines exist.
+Don't tick a runtime whose workloads already send their logs to Maple:
 
-## Change the filter
+| Reason        | Detail                                                                                                                                                                                 |
+| ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Duplicates    | A workload that exports logs over OpenTelemetry and also prints them, or a collector that reads pod logs, already delivers each line. Forwarding it from Google Cloud stores it twice. |
+| No trace link | The OpenTelemetry copy carries the trace and span ID of the request that wrote it. The copy from Google Cloud has neither.                                                             |
+| Volume        | Application output grows with traffic. Every forwarded line counts toward your Maple plan and toward the Pub/Sub usage Google bills to your account.                                   |
+
+Maple does not check which workloads send their logs another way. A runtime with both kinds of workloads is one tick: tick it when the uninstrumented ones matter more, since Cloud Logging is the only place their lines exist.
 
-On the connection's card, click **Configure** and choose a filter under **Log filter**. Click **Continue to the script**, then copy the script and run it in Cloud Shell again.
+Other places your code runs have no tick and are always forwarded: Compute Engine logs collected by the Ops Agent, and the job output of Batch, Dataflow, Dataproc, Cloud Composer and Vertex AI. Leave them out with a filter of your own.
 
-| Log filter                             | The sink forwards                                                                                     |
-| -------------------------------------- | ----------------------------------------------------------------------------------------------------- |
-| **Recommended: no GKE container logs** | Platform logs. Leaves out GKE container logs and the [other exclusions](/docs/integrations/gcp#logs). |
-| **Include GKE container logs**         | The same, plus GKE container logs. Maple asks you to acknowledge the duplicates before you continue.  |
-| **Keep the sink's current filter**     | What it forwards now. Offered, and preselected, once a run has set log forwarding up.                 |
+## Change the filter
 
-For any other filter, paste the script into an editor, edit `LOG_FILTER` near its top, set `LOG_FILTER_MODE` to `set` and run it. To also leave out Cloud Run container output, for services that send their logs over OpenTelemetry, add this at the end of `LOG_FILTER`, inside the quotes:
+On the connection's row, click **Configure**. Under **Log filter**, choose **Replace filter**. To forward application output, click **Can't forward via OpenTelemetry** and tick its runtimes. Click **Continue to the script**, then copy the script and run it in Cloud Shell again.
 
-```text
-AND NOT log_id("run.googleapis.com/stdout") AND NOT log_id("run.googleapis.com/stderr")
-```
+| Log filter              | The sink forwards                                                                     |
+| ----------------------- | ------------------------------------------------------------------------------------- |
+| **Keep current filter** | What it forwards now. Preselected once a run has set log forwarding up.               |
+| **Replace filter**      | Platform, request and audit logs, plus the application output of any ticked runtimes. |
 
-Cloud Run's request logs are in `run.googleapis.com/requests` and still pass. The filter uses the [Logging query language](https://cloud.google.com/logging/docs/view/logging-query-language).
+For any other filter, paste the script into an editor, edit `LOG_FILTER` near its top, set `LOG_FILTER_MODE` to `set` and run it. The filter uses the [Logging query language](https://cloud.google.com/logging/docs/view/logging-query-language).
 
 ## Instrument a Cloud Run service