feat(landing): log parsing guide

apps/landing/src/lib/guide-highlight.ts

@@ -1,7 +1,18 @@
 // Tiny highlighter for guide code blocks. The token classes match the colors a
 // guide uses in its diagrams (metric name, labels, value), so a block reads the
 // same way as the annotated example above it. Style the `tok-*` classes on the page.
-export type GuideLang = "exposition" | "promql" | "yaml" | "python" | "typescript" | "go" | "rust" | "java" | "csharp";
+export type GuideLang =
+    | "exposition"
+    | "promql"
+    | "yaml"
+    | "python"
+    | "typescript"
+    | "go"
+    | "rust"
+    | "java"
+    | "csharp"
+    | "shell"
+    | "otel-debug";
 
 const escape = (s: string) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 const span = (cls: string, text: string) => `<span class="tok-${cls}">${escape(text)}</span>`;
@@ -98,5 +109,31 @@ const rust = braces("use|fn|let|mut|for|in");
 const java = braces("import|public|static|final|class|void|new|try|throws");
 const csharp = braces("using|var|new|await");
 
+// Shell commands: the program name, flags and quoted arguments. Single-quoted
+// arguments (awk programs, jq filters) are one string, whatever quotes they hold.
+const shell = (code: string) =>
+    tokenize(code, [
+        [/#.*/y, (m) => span("comment", m[0])],
+        [/(?<=^|\n|\| )(\s*)([\w.-]+)/y, (m) => escape(m[1]) + span("fn", m[2])],
+        [/'[^'\n]*'/y, (m) => span("labels", m[0])],
+        [/"[^"\n]*"/y, (m) => span("labels", m[0])],
+        [/(?<=\s)--?[\w-]+/y, (m) => span("value", m[0])],
+    ]);
+
+// The OpenTelemetry Collector's debug exporter output: field and attribute keys
+// as names, values as values, the type wrappers (Str, Int) as comments. A value
+// that runs over several lines leaves its closing parenthesis to the last one.
+const otelDebug = (code: string) =>
+    code
+        .split("\n")
+        .map((line) => {
+            const attr = /^(\s*-> )([\w.]+)(: )(\w+\()(.*?)(\)?)$/.exec(line);
+            if (attr) return span("comment", attr[1]) + span("name", attr[2]) + escape(attr[3]) + span("comment", attr[4]) + span("value", attr[5]) + span("comment", attr[6]);
+            const field = /^(\w[\w ]*)(:)(.*)$/.exec(line);
+            if (field) return span("name", field[1]) + escape(field[2]) + span("value", field[3]);
+            return escape(line);
+        })
+        .join("\n");
+
 export const highlight = (code: string, lang: GuideLang) =>
-    ({ exposition, promql, yaml, python, typescript, go, rust, java, csharp })[lang](code);
+    ({ exposition, promql, yaml, python, typescript, go, rust, java, csharp, shell, "otel-debug": otelDebug })[lang](code);

apps/landing/src/pages/guides/_log-parsing/debug.yaml

@@ -0,0 +1,9 @@
+# debug.yaml: print parsed records instead of exporting them
+exporters:
+    debug:
+        verbosity: detailed
+
+service:
+    pipelines:
+        logs:
+            exporters: [debug]

apps/landing/src/pages/guides/_log-parsing/json.yaml

@@ -0,0 +1,14 @@
+receivers:
+    file_log/app:
+        include: [/var/log/app/app.json.log]
+        operators:
+            - type: json_parser
+              timestamp:
+                  parse_from: attributes.ts
+                  layout_type: gotime
+                  layout: "2006-01-02T15:04:05Z07:00"
+              severity:
+                  parse_from: attributes.level
+            - type: move
+              from: attributes.msg
+              to: body

apps/landing/src/pages/guides/_log-parsing/logfmt.yaml

@@ -0,0 +1,11 @@
+receivers:
+    file_log/worker:
+        include: [/var/log/worker/worker.log]
+        operators:
+            - type: key_value_parser
+              timestamp:
+                  parse_from: attributes.time
+                  layout_type: gotime
+                  layout: "2006-01-02T15:04:05Z07:00"
+              severity:
+                  parse_from: attributes.level

apps/landing/src/pages/guides/_log-parsing/multiline.yaml

@@ -0,0 +1,14 @@
+receivers:
+    file_log/java:
+        include: [/var/log/checkout/app.log]
+        multiline:
+            line_start_pattern: '^\d{4}-\d{2}-\d{2} '
+        operators:
+            - type: regex_parser
+              regex: '(?s)^(?P<time>\S+ \S+) (?P<level>[A-Z]+)\s+\[(?P<thread>[^\]]+)\] (?P<logger>\S+) - (?P<message>.*)$'
+              timestamp:
+                  parse_from: attributes.time
+                  layout: "%Y-%m-%d %H:%M:%S,%L"
+                  location: UTC
+              severity:
+                  parse_from: attributes.level

apps/landing/src/pages/guides/_log-parsing/nginx-collector.yaml

@@ -0,0 +1,69 @@
+receivers:
+    file_log/nginx:
+        include: [/var/log/nginx/access.log]
+        operators:
+            - type: regex_parser
+              regex: '^(?P<client_address>\S+) \S+ \S+ \[(?P<time_local>[^\]]+)\] "(?P<method>[A-Z]+) (?P<path>[^ ?"]+)[^ "]* HTTP/[0-9.]+" (?P<status>\d{3}) (?P<body_bytes>\d+|-) "[^"]*" "(?P<user_agent>[^"]*)"$'
+              timestamp:
+                  parse_from: attributes.time_local
+                  layout: "%d/%b/%Y:%H:%M:%S %z"
+              severity:
+                  parse_from: attributes.status
+                  overwrite_text: true
+                  mapping:
+                      info:
+                          - min: 100
+                            max: 399
+                      warn: 4xx
+                      error: 5xx
+            - type: move
+              from: attributes.client_address
+              to: attributes["client.address"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.method
+              to: attributes["http.request.method"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.path
+              to: attributes["url.path"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.status
+              to: attributes["http.response.status_code"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.body_bytes
+              to: attributes["http.response.body.size"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.user_agent
+              to: attributes["user_agent.original"]
+              on_error: send_quiet
+            - type: remove
+              field: attributes.time_local
+              on_error: send_quiet
+
+processors:
+    transform/nginx:
+        error_mode: ignore
+        log_statements:
+            - set(resource.attributes["service.name"], "nginx")
+            - delete_key(log.attributes, "http.response.body.size") where log.attributes["http.response.body.size"] == "-"
+            - set(log.attributes["http.response.status_code"], Int(log.attributes["http.response.status_code"])) where log.attributes["http.response.status_code"] != nil
+            - set(log.attributes["http.response.body.size"], Int(log.attributes["http.response.body.size"])) where log.attributes["http.response.body.size"] != nil
+    batch: {}
+
+exporters:
+    otlphttp/maple:
+        endpoint: https://ingest.maple.dev
+        compression: gzip
+        headers:
+            x-maple-ingest-key: ${env:MAPLE_INGEST_KEY}
+
+service:
+    pipelines:
+        logs:
+            receivers: [file_log/nginx]
+            processors: [transform/nginx, batch]
+            exporters: [otlphttp/maple]

apps/landing/src/pages/guides/_log-parsing/syslog.yaml

@@ -0,0 +1,6 @@
+receivers:
+    file_log/syslog:
+        include: [/var/log/remote/hosts.log]
+        operators:
+            - type: syslog_parser
+              protocol: rfc5424

apps/landing/src/pages/guides/_log-parsing/transform.yaml

@@ -0,0 +1,7 @@
+processors:
+    transform/parse:
+        error_mode: ignore
+        log_statements:
+            - merge_maps(log.attributes, ParseJSON(log.body), "upsert") where IsMatch(log.body, "^\\{")
+            - merge_maps(log.attributes, ParseKeyValue(log.body), "upsert") where IsMatch(log.body, "^time=")
+            - merge_maps(log.attributes, ExtractGrokPatterns(log.body, "%{HTTPD_COMBINEDLOG}", true), "upsert") where log.attributes["log.file.name"] == "access.log"

apps/landing/src/pages/guides/index.astro

@@ -66,6 +66,13 @@ const investigationGuides = [
         description: "Measure the right latency clock, isolate the slow requests, and trace a regression to the release and operation responsible.",
         read: "14 min",
     },
+    {
+        href: "/guides/log-parsing",
+        label: "Logs",
+        title: "Log parsing: from raw lines to structured fields",
+        description: "Parse syslog, nginx, JSON and multiline logs with regex, grok and Linux commands, then in the OpenTelemetry Collector.",
+        read: "14 min",
+    },
 ];
 
 const fieldNotes = (await getCollection("blog") as BlogPost[])