From 8a77b1610a43b24d449209db1efd75c09a894bfd Mon Sep 17 00:00:00 2001 From: JeremyFunk Date: Mon, 19 Oct 2026 19:39:04 +0200 Subject: [PATCH 1/4] feat(landing): log parsing guide Covers log formats (syslog, nginx combined, JSON, logfmt, multiline), regex and grok, Linux command-line parsing, and Collector filelog operators plus OTTL, with a tested nginx access log config that exports to Maple. --- .../guides/_log-parsing/nginx-collector.yaml | 66 ++ apps/landing/src/pages/guides/index.astro | 7 + .../src/pages/guides/log-parsing.astro | 733 ++++++++++++++++++ apps/landing/src/pages/llms.txt.ts | 1 + 4 files changed, 807 insertions(+) create mode 100644 apps/landing/src/pages/guides/_log-parsing/nginx-collector.yaml create mode 100644 apps/landing/src/pages/guides/log-parsing.astro diff --git a/apps/landing/src/pages/guides/_log-parsing/nginx-collector.yaml b/apps/landing/src/pages/guides/_log-parsing/nginx-collector.yaml new file mode 100644 index 0000000000..687a4f12cd --- /dev/null +++ b/apps/landing/src/pages/guides/_log-parsing/nginx-collector.yaml @@ -0,0 +1,66 @@ +receivers: + filelog/nginx: + include: [/var/log/nginx/access.log] + operators: + - type: regex_parser + regex: '^(?P\S+) \S+ \S+ \[(?P[^\]]+)\] "(?P[A-Z]+) (?P[^ ?"]+)[^ "]* HTTP/[0-9.]+" (?P\d{3}) (?P\d+) "[^"]*" "(?P[^"]*)"$' + timestamp: + parse_from: attributes.time_local + layout: "%d/%b/%Y:%H:%M:%S %z" + severity: + parse_from: attributes.status + overwrite_text: true + mapping: + info: [2xx, 3xx] + warn: 4xx + error: 5xx + - type: move + from: attributes.client_address + to: attributes["client.address"] + on_error: send_quiet + - type: move + from: attributes.method + to: attributes["http.request.method"] + on_error: send_quiet + - type: move + from: attributes.path + to: attributes["url.path"] + on_error: send_quiet + - type: move + from: attributes.status + to: attributes["http.response.status_code"] + on_error: send_quiet + - type: move + from: attributes.body_bytes + to: attributes["http.response.body.size"] + on_error: send_quiet + - type: move + from: attributes.user_agent + to: attributes["user_agent.original"] + on_error: send_quiet + - type: remove + field: attributes.time_local + on_error: send_quiet + +processors: + transform/nginx: + error_mode: ignore + log_statements: + - set(resource.attributes["service.name"], "nginx") + - set(log.attributes["http.response.status_code"], Int(log.attributes["http.response.status_code"])) where log.attributes["http.response.status_code"] != nil + - set(log.attributes["http.response.body.size"], Int(log.attributes["http.response.body.size"])) where log.attributes["http.response.body.size"] != nil + batch: {} + +exporters: + otlphttp/maple: + endpoint: https://ingest.maple.dev + compression: gzip + headers: + x-maple-ingest-key: ${env:MAPLE_INGEST_KEY} + +service: + pipelines: + logs: + receivers: [filelog/nginx] + processors: [transform/nginx, batch] + exporters: [otlphttp/maple] diff --git a/apps/landing/src/pages/guides/index.astro b/apps/landing/src/pages/guides/index.astro index 7ca330c958..bbd5944c58 100644 --- a/apps/landing/src/pages/guides/index.astro +++ b/apps/landing/src/pages/guides/index.astro @@ -66,6 +66,13 @@ const investigationGuides = [ description: "Measure the right latency clock, isolate the slow requests, and trace a regression to the release and operation responsible.", read: "14 min", }, + { + href: "/guides/log-parsing", + label: "Logs", + title: "Log parsing: from raw lines to structured fields", + description: "Parse syslog, nginx, JSON and multiline logs with regex, grok and Linux commands, then in the OpenTelemetry Collector.", + read: "14 min", + }, ]; const fieldNotes = (await getCollection("blog") as BlogPost[]) diff --git a/apps/landing/src/pages/guides/log-parsing.astro b/apps/landing/src/pages/guides/log-parsing.astro new file mode 100644 index 0000000000..fbdde8de05 --- /dev/null +++ b/apps/landing/src/pages/guides/log-parsing.astro @@ -0,0 +1,733 @@ +--- +import Layout from "../../layouts/Layout.astro"; +import Nav from "../../components/Nav.astro"; +import Footer from "../../components/Footer.astro"; +import BottomCTA from "../../components/BottomCTA.astro"; +import Breadcrumb from "../../components/Breadcrumb.astro"; +import FAQ from "../../components/FAQ.astro"; +import SeoHead from "../../components/SeoHead.astro"; +import TableOfContents from "../../components/TableOfContents.astro"; +import { CONTRIBUTORS } from "../../lib/changelog-meta"; +import nginxCollectorConfig from "./_log-parsing/nginx-collector.yaml?raw"; + +const pageTitle = "Log Parsing: Regex, Grok, Linux Commands and the OTel Collector | Maple"; +const pageDescription = + "Log parsing turns raw log lines into structured fields. Parse syslog, nginx, JSON and multiline logs with regex, grok, Linux commands and the OTel Collector."; +const published = "2026-10-19"; +const updatedLabel = "October 19, 2026"; +const author = CONTRIBUTORS.jeremyfunk; +const authorUrl = `https://github.com/${author.github}`; + +const toc = [ + ["definition", "Log parsing defined"], + ["formats", "Common log formats"], + ["techniques", "Parsing techniques"], + ["linux", "Linux commands"], + ["collector", "Collector operators"], + ["nginx", "Full nginx config"], + ["ottl", "OTTL transform"], + ["maple", "Parsed logs in Maple"], + ["structured", "Structured logging"], +] as const; +const tocHeadings = toc.map(([slug, text]) => ({ depth: 2, slug, text })); + +const nginxLine = String.raw`203.0.113.7 - - [19/Oct/2026:14:02:11 +0000] "GET /api/orders/42?expand=items HTTP/1.1" 200 1532 "https://shop.example.com/cart" "Mozilla/5.0 (X11; Linux x86_64)"`; + +const parsedFields = [ + ["Timestamp", "2026-10-19 14:02:11 UTC"], + ["SeverityText", "INFO"], + ["client.address", "203.0.113.7"], + ["http.request.method", "GET"], + ["url.path", "/api/orders/42"], + ["http.response.status_code", "200"], + ["http.response.body.size", "1532"], + ["user_agent.original", "Mozilla/5.0 (X11; Linux x86_64)"], +]; + +const formatSamples = String.raw`# syslog, RFC 3164 (BSD) +<34>Oct 19 14:02:11 web-01 sshd[2211]: Failed password for invalid user admin from 198.51.100.4 port 52113 ssh2 + +# syslog, RFC 5424 +<34>1 2026-10-19T14:02:11.003Z web-01 sshd 2211 - - Failed password for invalid user admin from 198.51.100.4 port 52113 ssh2 + +# nginx / Apache combined +${nginxLine} + +# JSON +{"ts":"2026-10-19T14:02:11.482Z","level":"error","msg":"payment declined","order_id":"ord_8812","attempt":2} + +# logfmt +time=2026-10-19T14:02:11Z level=warn msg="cache miss" key=user:42 duration_ms=18 + +# multiline (Java stack trace) +2026-10-19 14:02:11,482 ERROR [http-nio-8080-exec-3] c.e.CheckoutController - Checkout failed +java.lang.IllegalStateException: cart is empty + at com.example.CheckoutService.submit(CheckoutService.java:88) + at com.example.CheckoutController.post(CheckoutController.java:41)`; + +const nginxRegex = String.raw`^(?P\S+) \S+ \S+ \[(?P[^\]]+)\] "(?P[A-Z]+) (?P[^ ?"]+)[^ "]* HTTP/[0-9.]+" (?P\d{3}) (?P\d+) "[^"]*" "(?P[^"]*)"$`; + +const pythonParser = String.raw`import re +import sys +from collections import Counter + +LINE = re.compile( + r'(?P\S+) \S+ \S+ \[(?P