From 90b3c562c5fbdac375ad065fb354203f9c13dc7d Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Tue, 17 Nov 2026 13:03:51 +0800 Subject: [PATCH 1/2] Add cluster remote client so11y metrics and harden observability edges. --- docs/en/changes/changes.md | 1 + docs/en/setup/backend/dashboards-so11y.md | 11 + docs/en/setup/backend/grafana-cluster.json | 1345 ++++++++++++++--- docs/en/setup/backend/grafana-instance.json | 1339 +++++++++++++--- .../scripts/mal/test-otel-rules/oap.data.yaml | 316 ++++ .../core/remote/RemoteServiceHandler.java | 8 +- .../core/remote/client/GRPCRemoteClient.java | 254 +++- .../remote/client/RemoteClientMetrics.java | 197 +++ .../core/remote/client/SelfRemoteClient.java | 39 +- .../client/GRPCRemoteClientTestCase.java | 652 +++++++- .../client/RemoteClientManagerTestCase.java | 57 +- .../client/RemoteClientMetricsTest.java | 253 ++++ .../remote/client/SelfRemoteClientTest.java | 232 +++ .../src/main/resources/otel-rules/oap.yaml | 23 + test/e2e-v2/cases/cluster/cluster-cases.yaml | 41 +- .../cluster/expected/cluster-size-value.yml | 16 + .../cluster/expected/cluster-up-oap1.yml | 8 + .../cluster/expected/cluster-up-oap2.yml | 10 +- .../expected/remote-so11y-semantics.yml | 16 + .../zk/banyandb/banyandb-cluster-cases.yaml | 39 +- .../metrics-has-value-label-remote.yml | 40 + .../metrics-has-value-percentile-remote.yml | 59 + test/e2e-v2/cases/so11y/so11y-cases.yaml | 8 + 23 files changed, 4370 insertions(+), 594 deletions(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java create mode 100644 oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java create mode 100644 test/e2e-v2/cases/cluster/expected/cluster-size-value.yml create mode 100644 test/e2e-v2/cases/cluster/expected/remote-so11y-semantics.yml create mode 100644 test/e2e-v2/cases/so11y/expected/metrics-has-value-label-remote.yml create mode 100644 test/e2e-v2/cases/so11y/expected/metrics-has-value-percentile-remote.yml diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 4553ed81f184..8b23b7fdb0a0 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -69,6 +69,7 @@ * Fix the async-profiler task OAP sends to an agent whose service has two tasks created within five minutes of each other: it could send the older one. * Fix alarm windows retaining empty layers after an initial metadata lookup miss. Later metrics can supply the missing layers for subsequent alarms, allowing layer-filtered `queryAlarms` queries to match the persisted layer. * Rename the etcd dynamic configuration password environment variable from `SW_CONFIG_ETCD_password` to `SW_CONFIG_ETCD_PASSWORD`, matching every other environment variable. The lower-case name no longer has any effect, so a deployment that sets it must switch to the new name. +* Add cluster remote self-observability metrics (out latency/pressure/error reason, remote in, and cluster size) to the OAP so11y MAL and dashboards. #### UI diff --git a/docs/en/setup/backend/dashboards-so11y.md b/docs/en/setup/backend/dashboards-so11y.md index 0cc662efd113..a2eb2f60b2e0 100644 --- a/docs/en/setup/backend/dashboards-so11y.md +++ b/docs/en/setup/backend/dashboards-so11y.md @@ -49,6 +49,17 @@ Self observability monitoring provides monitoring of the status and resources of | Count per minute | meter_oap_instance_alarm_http_delivery_count | Alarm hook deliveries by `hook_type` and `result` (Per Minute), including `gRPC` | oap self observability | | Count per minute | meter_oap_instance_alarm_http_delivery_status_count | HTTP alarm hook responses by `hook_type` and `status_class` (Per Minute) | oap self observability | | ms | meter_oap_instance_alarm_http_delivery_latency_percentile | Alarm hook delivery latency by `hook_type` (ms), including `gRPC` | oap self observability | +| Count per minute | meter_oap_instance_remote_out_count | Cluster remote client push count by `dest` and `self` (Per Minute); peers: send attempts, `self=Y`: successful local fan-out | oap self observability | +| Count per minute | meter_oap_instance_remote_out_error_count | Cluster remote client push errors by `dest`, `self` and `reason` (Per Minute); one sample per failed stream | oap self observability | +| ms | meter_oap_instance_remote_out_latency_percentile | Cluster remote client push latency by `dest` and `self` (ms, one sample per successful stream) | oap self observability | +| Count | meter_oap_instance_remote_out_stream_inflight | In-flight remote gRPC streams by `dest` and `self` (peers only) | oap self observability | +| Milliseconds per minute | meter_oap_instance_remote_out_stream_block_milliseconds | Time waiting for a free remote stream slot by `dest` and `self` (ms Per Minute, peers only) | oap self observability | +| Count | meter_oap_instance_remote_out_queue_used_per_ten_thousand | Remote send queue used percentage ×100 by `dest` and `self` (peers only; divide by 100 for 0–100%) | oap self observability | +| Count per minute | meter_oap_instance_remote_in_count | Cluster remote server receive count (Per Minute) | oap self observability | +| Count per minute | meter_oap_instance_remote_in_error_count | Cluster remote server receive errors (Per Minute) | oap self observability | +| Count per minute | meter_oap_instance_remote_in_target_not_found_count | Cluster remote server target-worker-not-found count (Per Minute) | oap self observability | +| ms | meter_oap_instance_remote_in_latency_percentile | Cluster remote server receive latency (ms, includes discarded messages) | oap self observability | +| Count | meter_oap_instance_cluster_size | OAP cluster size seen by the current node | oap self observability | ## Customizations You can customize your own metrics/expression/dashboard panel. diff --git a/docs/en/setup/backend/grafana-cluster.json b/docs/en/setup/backend/grafana-cluster.json index 6715eb84ae32..6863b809daa2 100644 --- a/docs/en/setup/backend/grafana-cluster.json +++ b/docs/en/setup/backend/grafana-cluster.json @@ -4376,217 +4376,6 @@ "title": "Metrics Aggregation (Increment) / Minute", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 80 - }, - "id": 29, - "options": { - "dataLinks": [], - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(rate(remote_out_count{job=\"$job\"}[1m]))", - "legendFormat": "total", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(remote_out_count{job=\"$job\"}[1m])", - "legendFormat": "{{instance}} -> {{dest}}", - "refId": "B" - } - ], - "title": "Remote Out Count / Second", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 80 - }, - "id": 30, - "options": { - "dataLinks": [], - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.3.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "sum(rate(remote_in_latency_count{job=\"$job\"}[1m]))", - "legendFormat": "total", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "expr": "rate(remote_in_latency_count{job=\"$job\"}[1m])", - "legendFormat": "{{instance}}", - "refId": "B" - } - ], - "title": "Remote In Count / Second", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -4723,7 +4512,7 @@ "h": 6, "w": 8, "x": 0, - "y": 86 + "y": 98 }, "id": 18, "interval": "1m", @@ -4823,7 +4612,7 @@ "h": 6, "w": 8, "x": 8, - "y": 86 + "y": 98 }, "id": 31, "interval": "1m", @@ -4977,7 +4766,7 @@ "h": 6, "w": 8, "x": 0, - "y": 92 + "y": 104 }, "id": 32, "interval": "1m", @@ -5077,7 +4866,7 @@ "h": 6, "w": 8, "x": 8, - "y": 92 + "y": 104 }, "id": 16, "interval": "1m", @@ -5230,7 +5019,7 @@ "h": 6, "w": 8, "x": 16, - "y": 92 + "y": 104 }, "id": 46, "interval": "1m", @@ -5328,7 +5117,7 @@ "h": 6, "w": 8, "x": 0, - "y": 98 + "y": 110 }, "id": 134, "interval": "1m", @@ -5429,7 +5218,7 @@ "h": 6, "w": 8, "x": 8, - "y": 98 + "y": 110 }, "id": 135, "interval": "1m", @@ -5582,7 +5371,7 @@ "h": 6, "w": 8, "x": 16, - "y": 98 + "y": 110 }, "id": 137, "interval": "1m", @@ -5734,7 +5523,7 @@ "h": 6, "w": 8, "x": 0, - "y": 104 + "y": 116 }, "id": 136, "interval": "1m", @@ -5834,7 +5623,7 @@ "h": 6, "w": 8, "x": 8, - "y": 104 + "y": 116 }, "id": 138, "interval": "1m", @@ -5948,7 +5737,7 @@ "h": 6, "w": 8, "x": 16, - "y": 104 + "y": 116 }, "id": 139, "interval": "1m", @@ -6100,7 +5889,7 @@ "h": 6, "w": 8, "x": 0, - "y": 110 + "y": 122 }, "id": 140, "interval": "1m", @@ -6214,7 +6003,7 @@ "h": 6, "w": 8, "x": 8, - "y": 110 + "y": 122 }, "id": 141, "interval": "1m", @@ -6367,7 +6156,7 @@ "h": 6, "w": 8, "x": 16, - "y": 110 + "y": 122 }, "id": 142, "interval": "1m", @@ -6468,7 +6257,7 @@ "h": 6, "w": 8, "x": 0, - "y": 116 + "y": 128 }, "id": 144, "interval": "1m", @@ -6569,7 +6358,7 @@ "h": 6, "w": 8, "x": 8, - "y": 116 + "y": 128 }, "id": 143, "interval": "1m", @@ -6669,7 +6458,7 @@ "h": 6, "w": 8, "x": 16, - "y": 116 + "y": 128 }, "id": 147, "interval": "1m", @@ -6769,7 +6558,7 @@ "h": 6, "w": 8, "x": 0, - "y": 122 + "y": 134 }, "id": 148, "interval": "1m", @@ -6869,7 +6658,7 @@ "h": 6, "w": 8, "x": 8, - "y": 122 + "y": 134 }, "id": 149, "interval": "1m", @@ -6969,7 +6758,7 @@ "h": 6, "w": 8, "x": 16, - "y": 122 + "y": 134 }, "id": 152, "interval": "1m", @@ -7070,7 +6859,7 @@ "h": 6, "w": 8, "x": 0, - "y": 128 + "y": 140 }, "id": 146, "interval": "1m", @@ -7222,7 +7011,7 @@ "h": 6, "w": 8, "x": 8, - "y": 128 + "y": 140 }, "id": 145, "interval": "1m", @@ -7317,7 +7106,1087 @@ "h": 1, "w": 24, "x": 0, - "y": 134 + "y": 146 + }, + "id": 166, + "panels": [], + "title": "Cluster Remote", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "gRPC peers: send attempts; self=Y: successful local fan-out.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 147 + }, + "id": 29, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(rate(remote_out_count{job=\"$job\"}[1m]))", + "legendFormat": "total", + "refId": "A" + } + ], + "title": "Remote Out Count / Second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 147 + }, + "id": 30, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(rate(remote_in_count{job=\"$job\"}[1m]))", + "legendFormat": "total", + "refId": "A" + } + ], + "title": "Remote In Count / Second", + "type": "timeseries", + "description": "Server-side remote receive rate." + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Failed peer outbound streams by reason (self=N).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 153 + }, + "id": 157, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum by (dest, reason) (increase(remote_out_error_count{job=\"$job\",self=\"N\"}[1m]))", + "legendFormat": "{{dest}} {{reason}}", + "refId": "A" + } + ], + "title": "Remote Out Error Count / Minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Peer outbound stream latency p99 (self=N).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 153 + }, + "id": 158, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum by (le, dest) (increase(remote_out_latency_bucket{job=\"$job\",self=\"N\"}[5m])))", + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Latency p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "In-flight peer gRPC streams (self=N).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 153 + }, + "id": 159, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "remote_out_stream_inflight{job=\"$job\",self=\"N\"}", + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Stream Inflight", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Peer send-queue used percentage (self=N).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 159 + }, + "id": 160, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "remote_out_queue_used_percentage{job=\"$job\",self=\"N\"}", + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Queue Used %", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Peer stream-slot wait seconds (self=N).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 159 + }, + "id": 161, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum by (dest) (increase(remote_out_stream_block_seconds{job=\"$job\",self=\"N\"}[1m]))", + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Stream Block Seconds / Minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "OAP cluster size seen by this node.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 159 + }, + "id": 162, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "cluster_size{job=\"$job\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Cluster Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Server-side remote receive failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 165 + }, + "id": 163, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(increase(remote_in_error_count{job=\"$job\"}[1m]))", + "legendFormat": "total", + "refId": "A" + } + ], + "title": "Remote In Error Count / Minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Inbound messages with a missing target worker.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 165 + }, + "id": 164, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(increase(remote_in_target_not_found_count{job=\"$job\"}[1m]))", + "legendFormat": "total", + "refId": "A" + } + ], + "title": "Remote In Target Not Found / Minute", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Per-message server handling time (includes discards).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 165 + }, + "id": 165, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum by (le) (increase(remote_in_latency_bucket{job=\"$job\"}[5m])))", + "legendFormat": "cluster", + "refId": "A" + } + ], + "title": "Remote In Latency p99", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 171 }, "id": 156, "panels": [], @@ -7387,7 +8256,7 @@ "h": 6, "w": 8, "x": 0, - "y": 135 + "y": 172 }, "id": 153, "interval": "1m", @@ -7487,7 +8356,7 @@ "h": 6, "w": 8, "x": 8, - "y": 135 + "y": 172 }, "id": 154, "interval": "1m", @@ -7588,7 +8457,7 @@ "h": 6, "w": 8, "x": 16, - "y": 135 + "y": 172 }, "id": 155, "interval": "1m", @@ -7730,4 +8599,4 @@ "uid": "7NEmel8mkc", "version": 101, "weekStart": "" -} \ No newline at end of file +} diff --git a/docs/en/setup/backend/grafana-instance.json b/docs/en/setup/backend/grafana-instance.json index ff0c20d97d07..87e780422bd0 100644 --- a/docs/en/setup/backend/grafana-instance.json +++ b/docs/en/setup/backend/grafana-instance.json @@ -4915,200 +4915,6 @@ "title": "Metrics Aggregation (Increment) / Minute", "type": "timeseries" }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 100 - }, - "id": 95, - "options": { - "dataLinks": [], - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.3.1", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(remote_out_count{instance=\"$instance\",job=\"$job\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}} -> {{dest}}", - "refId": "A" - } - ], - "title": "Remote Out Count / Second", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 100 - }, - "id": 96, - "options": { - "dataLinks": [], - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.3.1", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "rate(remote_in_latency_count{instance=\"$instance\",job=\"$job\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Remote In Count / Second", - "type": "timeseries" - }, { "datasource": { "uid": "$datasource" @@ -5245,7 +5051,7 @@ "h": 6, "w": 8, "x": 0, - "y": 106 + "y": 118 }, "id": 18, "interval": "1m", @@ -5344,7 +5150,7 @@ "h": 6, "w": 8, "x": 8, - "y": 106 + "y": 118 }, "id": 98, "interval": "1m", @@ -5497,7 +5303,7 @@ "h": 6, "w": 8, "x": 0, - "y": 112 + "y": 124 }, "id": 97, "interval": "1m", @@ -5596,7 +5402,7 @@ "h": 6, "w": 8, "x": 8, - "y": 112 + "y": 124 }, "id": 99, "interval": "1m", @@ -5749,7 +5555,7 @@ "h": 6, "w": 8, "x": 16, - "y": 112 + "y": 124 }, "id": 100, "interval": "1m", @@ -5847,7 +5653,7 @@ "h": 6, "w": 8, "x": 0, - "y": 118 + "y": 130 }, "id": 134, "interval": "1m", @@ -5948,7 +5754,7 @@ "h": 6, "w": 8, "x": 8, - "y": 118 + "y": 130 }, "id": 135, "interval": "1m", @@ -6100,7 +5906,7 @@ "h": 6, "w": 8, "x": 16, - "y": 118 + "y": 130 }, "id": 136, "interval": "1m", @@ -6201,7 +6007,7 @@ "h": 6, "w": 8, "x": 0, - "y": 124 + "y": 136 }, "id": 137, "interval": "1m", @@ -6353,7 +6159,7 @@ "h": 6, "w": 8, "x": 8, - "y": 124 + "y": 136 }, "id": 138, "interval": "1m", @@ -6467,7 +6273,7 @@ "h": 6, "w": 8, "x": 16, - "y": 124 + "y": 136 }, "id": 139, "interval": "1m", @@ -6619,7 +6425,7 @@ "h": 6, "w": 8, "x": 0, - "y": 130 + "y": 142 }, "id": 140, "interval": "1m", @@ -6733,7 +6539,7 @@ "h": 6, "w": 8, "x": 8, - "y": 130 + "y": 142 }, "id": 141, "interval": "1m", @@ -6886,7 +6692,7 @@ "h": 6, "w": 8, "x": 16, - "y": 130 + "y": 142 }, "id": 143, "interval": "1m", @@ -6987,7 +6793,7 @@ "h": 6, "w": 8, "x": 0, - "y": 136 + "y": 148 }, "id": 142, "interval": "1m", @@ -7088,7 +6894,7 @@ "h": 6, "w": 8, "x": 8, - "y": 136 + "y": 148 }, "id": 144, "interval": "1m", @@ -7189,7 +6995,7 @@ "h": 6, "w": 8, "x": 16, - "y": 136 + "y": 148 }, "id": 145, "interval": "1m", @@ -7342,7 +7148,7 @@ "h": 6, "w": 8, "x": 0, - "y": 142 + "y": 154 }, "id": 146, "interval": "1m", @@ -7494,7 +7300,7 @@ "h": 6, "w": 8, "x": 8, - "y": 142 + "y": 154 }, "id": 147, "interval": "1m", @@ -7594,7 +7400,7 @@ "h": 6, "w": 8, "x": 16, - "y": 142 + "y": 154 }, "id": 150, "interval": "1m", @@ -7694,7 +7500,7 @@ "h": 6, "w": 8, "x": 16, - "y": 148 + "y": 160 }, "id": 149, "interval": "1m", @@ -7794,7 +7600,7 @@ "h": 6, "w": 8, "x": 0, - "y": 154 + "y": 166 }, "id": 152, "interval": "1m", @@ -7837,7 +7643,1098 @@ "h": 1, "w": 24, "x": 0, - "y": 160 + "y": 172 + }, + "id": 166, + "panels": [], + "title": "Cluster Remote", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 173 + }, + "id": 95, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(remote_out_count{instance=\"$instance\",job=\"$job\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}} -> {{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Count / Second", + "type": "timeseries", + "description": "gRPC peers: send attempts; self=Y: successful local fan-out." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 173 + }, + "id": 96, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(remote_in_count{instance=\"$instance\",job=\"$job\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Remote In Count / Second", + "type": "timeseries", + "description": "Server-side remote receive rate." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 179 + }, + "id": 157, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (dest, reason) (increase(remote_out_error_count{instance=\"$instance\",job=\"$job\",self=\"N\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{dest}} {{reason}}", + "refId": "A" + } + ], + "title": "Remote Out Error Count / Minute", + "type": "timeseries", + "description": "Failed peer outbound streams by reason (self=N)." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 179 + }, + "id": 158, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum by (le, dest) (increase(remote_out_latency_bucket{instance=\"$instance\",job=\"$job\",self=\"N\"}[5m])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Latency p99", + "type": "timeseries", + "description": "Peer outbound stream latency p99 (self=N)." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 179 + }, + "id": 159, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "remote_out_stream_inflight{instance=\"$instance\",job=\"$job\",self=\"N\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Stream Inflight", + "type": "timeseries", + "description": "In-flight peer gRPC streams (self=N)." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 185 + }, + "id": 160, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "remote_out_queue_used_percentage{instance=\"$instance\",job=\"$job\",self=\"N\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Queue Used %", + "type": "timeseries", + "description": "Peer send-queue used percentage (self=N)." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 185 + }, + "id": 161, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum by (dest) (increase(remote_out_stream_block_seconds{instance=\"$instance\",job=\"$job\",self=\"N\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{dest}}", + "refId": "A" + } + ], + "title": "Remote Out Stream Block Seconds / Minute", + "type": "timeseries", + "description": "Peer stream-slot wait seconds (self=N)." + }, + { + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 185 + }, + "id": 162, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "cluster_size{instance=\"$instance\",job=\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Cluster Size", + "type": "timeseries", + "description": "OAP cluster size seen by this node." + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Server-side remote receive failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 191 + }, + "id": 163, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(remote_in_error_count{instance=\"$instance\",job=\"$job\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Remote In Error Count / Minute", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Inbound messages with a missing target worker.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 191 + }, + "id": 164, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "increase(remote_in_target_not_found_count{instance=\"$instance\",job=\"$job\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Remote In Target Not Found / Minute", + "type": "timeseries" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Per-message server handling time (includes discards).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 191 + }, + "id": 165, + "options": { + "dataLinks": [], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "histogram_quantile(0.99, sum by (le) (increase(remote_in_latency_bucket{instance=\"$instance\",job=\"$job\"}[5m])))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Remote In Latency p99", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 197 }, "id": 156, "panels": [], @@ -7907,7 +8804,7 @@ "h": 6, "w": 8, "x": 0, - "y": 161 + "y": 198 }, "id": 153, "interval": "1m", @@ -8007,7 +8904,7 @@ "h": 6, "w": 8, "x": 8, - "y": 161 + "y": 198 }, "id": 154, "interval": "1m", @@ -8108,7 +9005,7 @@ "h": 6, "w": 8, "x": 16, - "y": 161 + "y": 198 }, "id": 155, "interval": "1m", @@ -8266,4 +9163,4 @@ "uid": "7NEmel8mki", "version": 32, "weekStart": "" -} \ No newline at end of file +} diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml index a41f9088ab75..c9ff24eafe6d 100644 --- a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml @@ -783,6 +783,131 @@ input: sw_backend_instance: test-oap hook_type: webhook value: 50.0 + remote_out_count: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 100.0 + - labels: + service: test-service + host_name: test-host + dest: oap1_11800 + self: Y + value: 100.0 + remote_out_error_count: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + reason: rate_limited + value: 100.0 + remote_out_latency: + - labels: + le: '0.05' + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 10.0 + - labels: + le: '0.1' + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 20.0 + - labels: + le: '0.5' + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 30.0 + - labels: + le: '1' + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 40.0 + - labels: + le: '5' + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 50.0 + remote_out_stream_inflight: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 2.0 + remote_out_stream_block_seconds: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 100.0 + remote_out_queue_used_percentage: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 35.0 + remote_in_count: + - labels: + service: test-service + host_name: test-host + value: 100.0 + remote_in_error_count: + - labels: + service: test-service + host_name: test-host + value: 100.0 + remote_in_target_not_found_count: + - labels: + service: test-service + host_name: test-host + value: 100.0 + remote_in_latency: + # HistogramObservations are in seconds; histogram_percentile emits ms bucket labels (*1000). + - labels: + le: '0.05' + service: test-service + host_name: test-host + value: 10.0 + - labels: + le: '0.1' + service: test-service + host_name: test-host + value: 20.0 + - labels: + le: '0.25' + service: test-service + host_name: test-host + value: 30.0 + - labels: + le: '0.5' + service: test-service + host_name: test-host + value: 40.0 + - labels: + le: '1' + service: test-service + host_name: test-host + value: 50.0 + cluster_size: + - labels: + service: test-service + host_name: test-host + value: 3.0 expected: meter_oap_instance_cpu_percentage: entities: @@ -1689,3 +1814,194 @@ expected: hook_type: webhook le: '5000' value: 25.0 + + meter_oap_instance_remote_out_count: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 50.0 + - labels: + service: test-service + host_name: test-host + dest: oap1_11800 + self: Y + value: 50.0 + meter_oap_instance_remote_out_error_count: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + reason: rate_limited + value: 50.0 + meter_oap_instance_remote_out_latency_percentile: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + le: '50' + value: 5.0 + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + le: '100' + value: 10.0 + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + le: '500' + value: 15.0 + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + le: '1000' + value: 20.0 + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + le: '5000' + value: 25.0 + meter_oap_instance_remote_out_stream_inflight: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 2.0 + meter_oap_instance_remote_out_stream_block_milliseconds: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 50000.0 + meter_oap_instance_remote_out_queue_used_per_ten_thousand: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + dest: oap2_11800 + self: N + value: 3500.0 + meter_oap_instance_remote_in_count: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + value: 50.0 + meter_oap_instance_remote_in_error_count: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + value: 50.0 + meter_oap_instance_remote_in_target_not_found_count: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + value: 50.0 + meter_oap_instance_remote_in_latency_percentile: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + le: '50' + value: 5.0 + - labels: + service: test-service + host_name: test-host + le: '100' + value: 10.0 + - labels: + service: test-service + host_name: test-host + le: '250' + value: 15.0 + - labels: + service: test-service + host_name: test-host + le: '500' + value: 20.0 + - labels: + service: test-service + host_name: test-host + le: '1000' + value: 25.0 + meter_oap_instance_cluster_size: + entities: + - scope: SERVICE_INSTANCE + service: test-service + instance: test-host + layer: SO11Y_OAP + samples: + - labels: + service: test-service + host_name: test-host + value: 3.0 diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/RemoteServiceHandler.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/RemoteServiceHandler.java index 42bfef0bee5c..f849746546b0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/RemoteServiceHandler.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/RemoteServiceHandler.java @@ -140,7 +140,12 @@ public void onNext(RemoteMessage message) { LOGGER.error("Can't deserialize data {}, this data is discarded.", message, t); return; } - nextWorker.in(streamData); + try { + nextWorker.in(streamData); + } catch (Exception e) { + remoteInErrorCounter.inc(); + LOGGER.error("Remote worker failed to process data {}, this data is discarded.", message, e); + } } else { remoteInTargetNotFoundCounter.inc(); LOGGER.warn( @@ -163,6 +168,7 @@ public void onError(Throwable throwable) { } return; } + remoteInErrorCounter.inc(); LOGGER.error(throwable.getMessage(), throwable); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java index 3917d2403673..6a78bd949071 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java @@ -20,11 +20,14 @@ import io.grpc.ManagedChannel; import io.grpc.Status; +import io.grpc.StatusException; +import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import io.netty.handler.ssl.SslContext; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.oap.server.core.remote.data.StreamData; @@ -40,9 +43,7 @@ import org.apache.skywalking.oap.server.library.client.grpc.GRPCClient; import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; -import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; -import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; /** * This is a wrapper of the gRPC client for sending message to each other OAP server. It contains a block queue to @@ -50,16 +51,21 @@ */ @Slf4j public class GRPCRemoteClient implements RemoteClient { + static final int MAX_CONCURRENT_STREAMS = 10; + private final int bufferSize; private final Address address; private final AtomicInteger concurrentStreamObserverNumber = new AtomicInteger(0); + private final Object inflightLock = new Object(); private SslContext sslContext; private GRPCClient client; private BatchQueue queue; private String queueName; private boolean isConnect; - private CounterMetrics remoteOutCounter; - private CounterMetrics remoteOutErrorCounter; + private volatile boolean closing; + private volatile boolean closed; + private final Object queueGaugeLock = new Object(); + private final RemoteClientMetrics metrics; private int remoteTimeout; private long lastRemoteResourceExhaustedTime = 0; @@ -74,28 +80,10 @@ public GRPCRemoteClient(final ModuleDefineHolder moduleDefineHolder, this.remoteTimeout = remoteTimeout; this.sslContext = sslContext; - remoteOutCounter = moduleDefineHolder.find(TelemetryModule.NAME) - .provider() - .getService(MetricsCreator.class) - .createCounter( - "remote_out_count", - "The number(client side) of inside remote inside aggregate rpc.", - new MetricsTag.Keys("dest", "self"), new MetricsTag.Values( - address - .toString(), "N" - ) - ); - remoteOutErrorCounter = moduleDefineHolder.find(TelemetryModule.NAME) - .provider() - .getService(MetricsCreator.class) - .createCounter( - "remote_out_error_count", - "The error number(client side) of inside remote inside aggregate rpc.", - new MetricsTag.Keys("dest", "self"), new MetricsTag.Values( - address - .toString(), "N" - ) - ); + final MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME) + .provider() + .getService(MetricsCreator.class); + this.metrics = RemoteClientMetrics.forGrpc(metricsCreator, address); } @Override @@ -154,44 +142,84 @@ public void push(String nextWorkerName, StreamData streamData) { builder.setNextWorkerName(nextWorkerName); builder.setRemoteData(streamData.serialize()); - this.queue.produce(builder.build()); + if (closing || closed) { + return; + } + publishQueueUsed(); + if (!this.queue.produce(builder.build())) { + if (!closing && !closed) { + metrics.recordError(RemoteClientMetrics.REASON_ERROR); + } + return; + } + publishQueueUsed(); } class RemoteMessageConsumer implements HandlerConsumer { @Override public void consume(List remoteMessages) { + publishQueueUsed(); + final int messageCount = remoteMessages.size(); + final long[] startNanos = new long[1]; + final AtomicBoolean streamAcquired = new AtomicBoolean(false); + final AtomicBoolean streamFinished = new AtomicBoolean(false); + final AtomicBoolean errorRecorded = new AtomicBoolean(false); try { - StreamObserver streamObserver = createStreamObserver(); + StreamObserver streamObserver = createStreamObserver( + startNanos, messageCount, streamAcquired, streamFinished, errorRecorded); + publishQueueUsed(); for (RemoteMessage remoteMessage : remoteMessages) { - remoteOutCounter.inc(); + metrics.incOut(); streamObserver.onNext(remoteMessage); } streamObserver.onCompleted(); + } catch (RemoteClientClosedException t) { + // ignored: client closed during wait/acquire + } catch (RemoteStreamInterruptedException t) { + log.error(t.getMessage(), t); } catch (Throwable t) { - remoteOutErrorCounter.inc(); + if (streamAcquired.get()) { + endStream(streamFinished, startNanos[0], messageCount, false); + } + if (!closing && !closed && errorRecorded.compareAndSet(false, true)) { + if (t instanceof StatusRuntimeException || t instanceof StatusException) { + metrics.recordError(RemoteClientMetrics.reasonOf(Status.fromThrowable(t))); + } else { + metrics.recordError(RemoteClientMetrics.REASON_ERROR); + } + } log.error(t.getMessage(), t); + } finally { + publishQueueUsed(); } } } - /** - * Create a gRPC stream observer to sending stream data, one stream observer could send multiple stream data by a - * single consume. The max number of concurrency allowed at the same time is 10. - * - * @return stream observer - */ - private StreamObserver createStreamObserver() { + private StreamObserver createStreamObserver(final long[] startNanosHolder, + final int messageCount, + final AtomicBoolean streamAcquired, + final AtomicBoolean streamFinished, + final AtomicBoolean errorRecorded) { int sleepTotalMillis = 0; int sleepMillis = 10; + long blockedNanos = 0; - // Control the concurrency of gRPC streaming stub. - // If over 10 created and not finished/error, this blocks the method. - while (concurrentStreamObserverNumber.get() > 10) { + while (concurrentStreamObserverNumber.get() >= MAX_CONCURRENT_STREAMS) { + if (closed) { + throw new RemoteClientClosedException("Remote client closed while waiting for a stream slot"); + } + final long blockStart = System.nanoTime(); + publishQueueUsed(); try { Thread.sleep(sleepMillis); } catch (InterruptedException e) { - log.error(e.getMessage(), e); + Thread.currentThread().interrupt(); + blockedNanos += System.nanoTime() - blockStart; + metrics.addStreamBlockSeconds(blockedNanos / 1_000_000_000d); + metrics.recordError(RemoteClientMetrics.REASON_INTERRUPTED); + throw new RemoteStreamInterruptedException("Interrupted while waiting for a remote stream slot", e); } + blockedNanos += System.nanoTime() - blockStart; sleepTotalMillis += sleepMillis; @@ -200,52 +228,118 @@ private StreamObserver createStreamObserver() { "Remote client [{}] block times over 60 seconds. Current streaming number {}", address, concurrentStreamObserverNumber.get() ); - // Reset sleepTotalMillis to avoid too many warn logs. sleepTotalMillis = 0; } } + metrics.addStreamBlockSeconds(blockedNanos / 1_000_000_000d); + startNanosHolder[0] = System.nanoTime(); + + if (!acquireStreamSlot()) { + throw new RemoteClientClosedException("Remote client closed before acquiring a stream slot"); + } + streamAcquired.set(true); + try { + return getStub().withDeadlineAfter(remoteTimeout, TimeUnit.SECONDS) + .call(new StreamObserver() { + @Override + public void onNext(Empty empty) { + } - final StreamObserver remoteMessageStreamObserver - = getStub().withDeadlineAfter(remoteTimeout, TimeUnit.SECONDS) - .call(new StreamObserver() { - @Override - public void onNext(Empty empty) { - } + @Override + public void onError(Throwable throwable) { + endStream(streamFinished, startNanosHolder[0], messageCount, false); + Status status = Status.fromThrowable(throwable); + if (!closing && !closed && errorRecorded.compareAndSet(false, true)) { + metrics.recordError(RemoteClientMetrics.reasonOf(status)); + } + if (closing || closed) { + return; + } + if (Status.CANCELLED.getCode() == status.getCode()) { + if (log.isDebugEnabled()) { + log.debug(throwable.getMessage(), throwable); + } + return; + } else if (Status.RESOURCE_EXHAUSTED.getCode() == status.getCode()) { + if (System.currentTimeMillis() - lastRemoteResourceExhaustedTime > 120_000) { + log.warn(throwable.getMessage(), throwable); + lastRemoteResourceExhaustedTime = System.currentTimeMillis(); + } + return; + } + log.error(throwable.getMessage(), throwable); + } + + @Override + public void onCompleted() { + endStream(streamFinished, startNanosHolder[0], messageCount, true); + } + }); + } catch (RuntimeException | Error t) { + endStream(streamFinished, startNanosHolder[0], messageCount, false); + throw t; + } + } + + private boolean acquireStreamSlot() { + synchronized (inflightLock) { + if (closed) { + return false; + } + final int inflight = concurrentStreamObserverNumber.incrementAndGet(); + if (!closing) { + metrics.setStreamInflight(inflight); + } + return true; + } + } - @Override - public void onError(Throwable throwable) { - concurrentStreamObserverNumber.addAndGet(-1); - Status status = Status.fromThrowable(throwable); - if (Status.CANCELLED.getCode() == status.getCode()) { - if (log.isDebugEnabled()) { - log.debug(throwable.getMessage(), throwable); - } - return; - } else if (Status.RESOURCE_EXHAUSTED.getCode() == status.getCode()) { - if (System.currentTimeMillis() - lastRemoteResourceExhaustedTime > 120_000) { - // Only output the log every 120 seconds. - log.warn(throwable.getMessage(), throwable); - lastRemoteResourceExhaustedTime = System.currentTimeMillis(); - } - return; - } - log.error(throwable.getMessage(), throwable); - } + private void publishQueueUsed() { + if (queue == null || closing || closed) { + return; + } + synchronized (queueGaugeLock) { + if (queue == null || closing || closed) { + return; + } + metrics.setQueueUsedPercentage(queue.stats().totalUsedPercentage()); + } + } - @Override - public void onCompleted() { - concurrentStreamObserverNumber.addAndGet(-1); - } - }); - concurrentStreamObserverNumber.incrementAndGet(); - return remoteMessageStreamObserver; + private void endStream(final AtomicBoolean streamFinished, + final long startNanos, + final int messageCount, + final boolean observeLatency) { + if (!streamFinished.compareAndSet(false, true)) { + return; + } + synchronized (inflightLock) { + if (!closed) { + final int inflight = concurrentStreamObserverNumber.addAndGet(-1); + if (!closing) { + metrics.setStreamInflight(Math.max(0, inflight)); + } + } + } + if (closing || closed || !observeLatency) { + return; + } + metrics.observeLatencySeconds((System.nanoTime() - startNanos) / 1_000_000_000d); } @Override public void close() { + closing = true; if (queueName != null) { BatchQueueManager.shutdown(queueName); } + synchronized (inflightLock) { + synchronized (queueGaugeLock) { + closed = true; + concurrentStreamObserverNumber.set(0); + metrics.clearPressureGauges(); + } + } if (Objects.nonNull(this.client)) { this.client.shutdown(); } @@ -260,4 +354,16 @@ public Address getAddress() { public int compareTo(RemoteClient o) { return address.compareTo(o.getAddress()); } + + static final class RemoteStreamInterruptedException extends RuntimeException { + RemoteStreamInterruptedException(final String message, final Throwable cause) { + super(message, cause); + } + } + + static final class RemoteClientClosedException extends RuntimeException { + RemoteClientClosedException(final String message) { + super(message); + } + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java new file mode 100644 index 000000000000..8b5e0b4bbec5 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.remote.client; + +import io.grpc.Status; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; + +/** Client-side {@code remote_out_*} metrics for cluster remote RPC. */ +public final class RemoteClientMetrics { + public static final String REASON_ERROR = "error"; + public static final String REASON_TIMEOUT = "timeout"; + public static final String REASON_IO_ERROR = "io_error"; + public static final String REASON_RATE_LIMITED = "rate_limited"; + public static final String REASON_AUTH_FAILED = "auth_failed"; + public static final String REASON_SERVER_ERROR = "server_error"; + public static final String REASON_REJECTED = "rejected"; + public static final String REASON_INTERRUPTED = "interrupted"; + + private static final String[] ERROR_REASONS = { + REASON_ERROR, + REASON_TIMEOUT, + REASON_IO_ERROR, + REASON_RATE_LIMITED, + REASON_AUTH_FAILED, + REASON_SERVER_ERROR, + REASON_REJECTED, + REASON_INTERRUPTED + }; + + private final MetricsCreator metricsCreator; + private final String dest; + private final String self; + + private final CounterMetrics outCounter; + private final HistogramMetrics outLatency; + private final GaugeMetrics streamInflight; + private final CounterMetrics streamBlockSeconds; + private final GaugeMetrics queueUsedPercentage; + private final ConcurrentMap errorCounters = new ConcurrentHashMap<>(); + + public static RemoteClientMetrics forGrpc(final MetricsCreator metricsCreator, final Address address) { + return new RemoteClientMetrics(metricsCreator, address, false, true); + } + + public static RemoteClientMetrics forSelf(final MetricsCreator metricsCreator, final Address address) { + return new RemoteClientMetrics(metricsCreator, address, true, false); + } + + private RemoteClientMetrics(final MetricsCreator metricsCreator, + final Address address, + final boolean self, + final boolean trackStreamAndQueue) { + this.metricsCreator = Objects.requireNonNull(metricsCreator, "metricsCreator"); + this.dest = Objects.requireNonNull(address, "address").toString(); + this.self = self ? "Y" : "N"; + + this.outCounter = metricsCreator.createCounter( + "remote_out_count", + "The number(client side) of inside remote inside aggregate rpc.", + new MetricsTag.Keys("dest", "self"), + new MetricsTag.Values(this.dest, this.self) + ); + this.outLatency = metricsCreator.createHistogramMetric( + "remote_out_latency", + "The latency(client side) of inside remote inside aggregate rpc.", + new MetricsTag.Keys("dest", "self"), + new MetricsTag.Values(this.dest, this.self) + ); + for (final String reason : ERROR_REASONS) { + final CounterMetrics counter = metricsCreator.createCounter( + "remote_out_error_count", + "The error number(client side) of inside remote inside aggregate rpc.", + new MetricsTag.Keys("dest", "self", "reason"), + new MetricsTag.Values(this.dest, this.self, reason) + ); + counter.inc(0); + errorCounters.put(reason, counter); + } + if (trackStreamAndQueue) { + this.streamInflight = metricsCreator.createGauge( + "remote_out_stream_inflight", + "In-flight gRPC streams for one remote client.", + new MetricsTag.Keys("dest", "self"), + new MetricsTag.Values(this.dest, this.self) + ); + this.streamBlockSeconds = metricsCreator.createCounter( + "remote_out_stream_block_seconds", + "Seconds spent waiting for a free remote gRPC stream slot.", + new MetricsTag.Keys("dest", "self"), + new MetricsTag.Values(this.dest, this.self) + ); + this.streamBlockSeconds.inc(0); + this.queueUsedPercentage = metricsCreator.createGauge( + "remote_out_queue_used_percentage", + "Send queue used percentage for one remote client.", + new MetricsTag.Keys("dest", "self"), + new MetricsTag.Values(this.dest, this.self) + ); + } else { + this.streamInflight = null; + this.streamBlockSeconds = null; + this.queueUsedPercentage = null; + } + } + + public void incOut() { + outCounter.inc(); + } + + public void observeLatencySeconds(final double seconds) { + outLatency.observe(Math.max(0, seconds)); + } + + public void recordError(final String reason) { + final String outcome = reason == null || reason.isEmpty() ? REASON_ERROR : reason; + errorCounters.computeIfAbsent(outcome, ignored -> metricsCreator.createCounter( + "remote_out_error_count", + "The error number(client side) of inside remote inside aggregate rpc.", + new MetricsTag.Keys("dest", "self", "reason"), + new MetricsTag.Values(dest, self, outcome) + )).inc(); + } + + public void setStreamInflight(final int value) { + if (streamInflight != null) { + streamInflight.setValue(Math.max(0, value)); + } + } + + public void addStreamBlockSeconds(final double seconds) { + if (streamBlockSeconds != null && seconds > 0) { + streamBlockSeconds.inc(seconds); + } + } + + public void setQueueUsedPercentage(final double percentage) { + if (queueUsedPercentage != null) { + queueUsedPercentage.setValue(Math.max(0, Math.min(100, percentage))); + } + } + + public void clearPressureGauges() { + setStreamInflight(0); + setQueueUsedPercentage(0); + } + + public static String reasonOf(final Status status) { + if (status == null) { + return REASON_ERROR; + } + switch (status.getCode()) { + case OK: + return REASON_ERROR; + case DEADLINE_EXCEEDED: + return REASON_TIMEOUT; + case UNAVAILABLE: + return REASON_IO_ERROR; + case RESOURCE_EXHAUSTED: + return REASON_RATE_LIMITED; + case UNAUTHENTICATED: + case PERMISSION_DENIED: + return REASON_AUTH_FAILED; + case UNKNOWN: + case INTERNAL: + case DATA_LOSS: + case UNIMPLEMENTED: + return REASON_SERVER_ERROR; + case CANCELLED: + return REASON_REJECTED; + default: + return REASON_REJECTED; + } + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java index 40de112be308..39fd420b8821 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java @@ -24,18 +24,16 @@ import org.apache.skywalking.oap.server.core.UnexpectedException; import org.apache.skywalking.oap.server.core.remote.data.StreamData; import org.apache.skywalking.oap.server.core.worker.IWorkerInstanceGetter; +import org.apache.skywalking.oap.server.core.worker.RemoteHandleWorker; import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; -import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; -import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; @Slf4j public class SelfRemoteClient implements RemoteClient { private final Address address; - private CounterMetrics remoteOutCounter; - private CounterMetrics remoteOutErrorCounter; + private final RemoteClientMetrics metrics; private final IWorkerInstanceGetter workerInstanceGetter; public SelfRemoteClient(ModuleDefineHolder moduleDefineHolder, Address address) { @@ -43,21 +41,10 @@ public SelfRemoteClient(ModuleDefineHolder moduleDefineHolder, Address address) workerInstanceGetter = moduleDefineHolder.find(CoreModule.NAME) .provider() .getService(IWorkerInstanceGetter.class); - remoteOutCounter = moduleDefineHolder.find(TelemetryModule.NAME) - .provider() - .getService(MetricsCreator.class) - .createCounter("remote_out_count", "The number(client side) of inside remote inside aggregate rpc.", new MetricsTag.Keys("dest", "self"), new MetricsTag.Values(address - .toString(), "Y")); - remoteOutErrorCounter = moduleDefineHolder.find(TelemetryModule.NAME) - .provider() - .getService(MetricsCreator.class) - .createCounter( - "remote_out_error_count", - "The error number(client side) of inside remote inside aggregate rpc.", - new MetricsTag.Keys("dest", "self"), new MetricsTag.Values( - address - .toString(), "Y") - ); + final MetricsCreator metricsCreator = moduleDefineHolder.find(TelemetryModule.NAME) + .provider() + .getService(MetricsCreator.class); + this.metrics = RemoteClientMetrics.forSelf(metricsCreator, address); } @Override @@ -76,11 +63,19 @@ public void close() { @Override public void push(String nextWorkerName, StreamData streamData) { + final long startNanos = System.nanoTime(); try { - workerInstanceGetter.get(nextWorkerName).getWorker().in(streamData); - remoteOutCounter.inc(); + final RemoteHandleWorker handleWorker = workerInstanceGetter.get(nextWorkerName); + if (handleWorker == null) { + metrics.recordError(RemoteClientMetrics.REASON_ERROR); + log.error("Self remote push discarded: worker {} not found.", nextWorkerName); + return; + } + handleWorker.getWorker().in(streamData); + metrics.incOut(); + metrics.observeLatencySeconds((System.nanoTime() - startNanos) / 1_000_000_000d); } catch (Throwable t) { - remoteOutErrorCounter.inc(); + metrics.recordError(RemoteClientMetrics.REASON_ERROR); log.error(t.getMessage(), t); } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java index b622641a4196..51ee46371ab4 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java @@ -20,14 +20,19 @@ import io.grpc.ManagedChannel; import io.grpc.Server; +import io.grpc.Status; import io.grpc.inprocess.InProcessChannelBuilder; import io.grpc.inprocess.InProcessServerBuilder; +import io.grpc.stub.StreamObserver; import io.grpc.util.MutableHandlerRegistry; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.worker.MetricStreamKind; import org.apache.skywalking.oap.server.core.remote.RemoteServiceHandler; import org.apache.skywalking.oap.server.core.remote.data.StreamData; +import org.apache.skywalking.oap.server.core.remote.grpc.proto.Empty; import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteMessage; +import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteServiceGrpc; import org.apache.skywalking.oap.server.core.worker.AbstractWorker; import org.apache.skywalking.oap.server.core.worker.IWorkerInstanceGetter; import org.apache.skywalking.oap.server.core.worker.IWorkerInstanceSetter; @@ -45,10 +50,20 @@ import org.junit.jupiter.api.Test; import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -108,49 +123,656 @@ public void after() { } @Test - public void testPush() throws InterruptedException { - MetricsCreator metricsCreator = mock(MetricsCreator.class); - when(metricsCreator.createCounter(any(), any(), any(), any())).thenReturn(new CounterMetrics() { + public void testPush() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflight = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, null, queueUsed, inflight); + + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + serviceRegistry.addService(new RemoteServiceHandler(moduleManager)); + + Address address = new Address("not-important", 11, false); + GRPCRemoteClient remoteClient = spy(new GRPCRemoteClient(moduleManager, address, 10, 10, null)); + try { + remoteClient.connect(); + + doReturn(channel).when(remoteClient).getChannel(); + + for (int i = 0; i < 12; i++) { + remoteClient.push(nextWorkerName, new TestStreamData()); + } + + awaitCondition( + "remote push metrics to settle", + 10, + () -> outCount.get() >= 12 + && latencyCount.get() >= 1 + && queueUsed.get() != null + && inflight.get() != null + && Math.abs(inflight.get()) < 0.01 + ); + Assertions.assertTrue(outCount.get() >= 12, "expected remote_out_count >= 12, got " + outCount.get()); + Assertions.assertTrue(latencyCount.get() >= 1, "expected >=1 remote_out_latency observation per stream, got " + latencyCount.get()); + Assertions.assertTrue( + latencyCount.get() <= outCount.get(), + "stream latency observations must not exceed push attempts; latency=" + + latencyCount.get() + " out=" + outCount.get() + ); + // One observe per successful stream (batch), never more than messages pushed. + Assertions.assertTrue( + latencyCount.get() <= 12, + "expected at most one latency sample per push when each flush is its own stream, got " + + latencyCount.get() + ); + Assertions.assertNotNull(queueUsed.get(), "expected remote_out_queue_used_percentage to be set"); + Assertions.assertTrue(queueUsed.get() >= 0 && queueUsed.get() <= 100, "queue used % must stay in [0,100]"); + Assertions.assertEquals(0.0, inflight.get(), 0.01, "inflight should drain after streams finish"); + + // Leave a non-zero pressure sample, then close — departed peers must not keep stale gauges. + inflight.set(9.0); + queueUsed.set(88.0); + remoteClient.close(); + Assertions.assertEquals(0.0, inflight.get(), 0.01, "close must clear stream inflight gauge"); + Assertions.assertEquals(0.0, queueUsed.get(), 0.01, "close must clear queue used gauge"); + } finally { + try { + remoteClient.close(); + } catch (Throwable ignored) { + // already closed in the happy path above + } + } + } + + @Test + public void closeClearsQueueGaugeAgainstConcurrentPublish() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(77.0); + final AtomicReference inflight = new AtomicReference<>(0.0); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflight); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + Address address = new Address("not-important", 20, false); + GRPCRemoteClient client = new GRPCRemoteClient(moduleManager, address, 10, 10, null); + client.connect(); + + Method publishQueueUsed = GRPCRemoteClient.class.getDeclaredMethod("publishQueueUsed"); + publishQueueUsed.setAccessible(true); + final AtomicBoolean run = new AtomicBoolean(true); + Thread[] publishers = new Thread[4]; + for (int i = 0; i < publishers.length; i++) { + publishers[i] = new Thread(() -> { + while (run.get()) { + try { + publishQueueUsed.invoke(client); + } catch (Throwable ignored) { + // close may race the queue reference; ignore + } + } + }, "queue-gauge-publish-" + i); + publishers[i].start(); + } + TimeUnit.MILLISECONDS.sleep(30); + client.close(); + run.set(false); + for (Thread publisher : publishers) { + publisher.join(TimeUnit.SECONDS.toMillis(5)); + Assertions.assertFalse(publisher.isAlive()); + } + Assertions.assertEquals( + 0.0, queueUsed.get(), 0.01, "close must win over concurrent publishQueueUsed writes"); + } + + @Test + public void pushRecordsUnavailableAsIoError() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflight = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflight); + + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + serviceRegistry.addService(new RemoteServiceGrpc.RemoteServiceImplBase() { @Override - public void inc() { + public StreamObserver call(StreamObserver responseObserver) { + return new StreamObserver() { + @Override + public void onNext(RemoteMessage value) { + } + + @Override + public void onError(Throwable t) { + } + + @Override + public void onCompleted() { + responseObserver.onError(Status.UNAVAILABLE.asRuntimeException()); + } + }; + } + }); + + Address address = new Address("not-important", 12, false); + GRPCRemoteClient remoteClient = spy(new GRPCRemoteClient(moduleManager, address, 10, 10, null)); + try { + remoteClient.connect(); + doReturn(channel).when(remoteClient).getChannel(); + + remoteClient.push(nextWorkerName, new TestStreamData()); + awaitCondition( + "unavailable stream metrics to settle", + 10, + () -> outCount.get() >= 1 + && !errorReasons.isEmpty() + && inflight.get() != null + && Math.abs(inflight.get()) < 0.01 + ); + + Assertions.assertTrue(outCount.get() >= 1); + Assertions.assertEquals(0, latencyCount.get(), "failed streams must not pollute remote_out_latency"); + Assertions.assertEquals(List.of(RemoteClientMetrics.REASON_IO_ERROR), errorReasons); + Assertions.assertEquals(0.0, inflight.get(), 0.01); + } finally { + remoteClient.close(); + } + } + + @Test + public void syncOnNextFailureReleasesInflightAndRecordsErrorOnce() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflight = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflight); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + Address address = new Address("not-important", 13, false); + GRPCRemoteClient remoteClient = spy(new GRPCRemoteClient(moduleManager, address, 10, 10, null)); + RemoteServiceGrpc.RemoteServiceStub stub = mock(RemoteServiceGrpc.RemoteServiceStub.class); + RemoteServiceGrpc.RemoteServiceStub stubWithDeadline = mock(RemoteServiceGrpc.RemoteServiceStub.class); + doReturn(stub).when(remoteClient).getStub(); + when(stub.withDeadlineAfter(anyLong(), any())).thenReturn(stubWithDeadline); + when(stubWithDeadline.call(any())).thenAnswer(invocation -> new StreamObserver() { + @Override + public void onNext(RemoteMessage value) { + throw new IllegalStateException("boom after stream acquired"); } @Override - public void inc(double value) { + public void onError(Throwable t) { + } + @Override + public void onCompleted() { } }); - when(metricsCreator.createHistogramMetric(any(), any(), any(), any())).thenReturn(new HistogramMetrics() { - @Override - public Timer createTimer() { - return super.createTimer(); + try { + remoteClient.connect(); + remoteClient.push(nextWorkerName, new TestStreamData()); + awaitCondition( + "sync failure to release inflight", + 10, + () -> inflight.get() != null + && Math.abs(inflight.get()) < 0.01 + && errorReasons.size() == 1 + ); + + Assertions.assertEquals(0.0, inflight.get(), 0.01, "sync failure must release the stream slot"); + Assertions.assertEquals(0, latencyCount.get()); + Assertions.assertEquals(List.of(RemoteClientMetrics.REASON_ERROR), errorReasons); + } finally { + remoteClient.close(); + } + } + + private static void awaitCondition(String description, + long timeoutSeconds, + java.util.concurrent.Callable condition) throws Exception { + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds); + while (System.nanoTime() < deadline) { + if (Boolean.TRUE.equals(condition.call())) { + return; } + TimeUnit.MILLISECONDS.sleep(50); + } + Assertions.fail("timed out waiting for " + description); + } + private static MetricsCreator mockMetricsCreator(AtomicInteger outCount, + AtomicInteger latencyCount, + List errorReasons, + AtomicReference queueUsed, + AtomicReference inflight) { + MetricsCreator metricsCreator = mock(MetricsCreator.class); + when(metricsCreator.createCounter(any(), any(), any(), any())).thenAnswer(invocation -> { + final String name = invocation.getArgument(0); + final org.apache.skywalking.oap.server.telemetry.api.MetricsTag.Values values = invocation.getArgument(3); + return new CounterMetrics() { + @Override + public void inc() { + if ("remote_out_count".equals(name)) { + outCount.incrementAndGet(); + } + if ("remote_out_error_count".equals(name) && errorReasons != null && values.getValues().length >= 3) { + errorReasons.add(values.getValues()[2]); + } + } + + @Override + public void inc(double value) { + } + }; + }); + when(metricsCreator.createHistogramMetric(any(), any(), any(), any())).thenAnswer(invocation -> { + final String name = invocation.getArgument(0); + return new HistogramMetrics() { + @Override + public void observe(double value) { + if ("remote_out_latency".equals(name)) { + latencyCount.incrementAndGet(); + } + } + }; + }); + when(metricsCreator.createGauge(any(), any(), any(), any())).thenAnswer(invocation -> { + final String name = invocation.getArgument(0); + return new org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics() { + @Override + public void inc() { + } + + @Override + public void inc(double value) { + } + + @Override + public void dec() { + } + + @Override + public void dec(double value) { + } + + @Override + public void setValue(double value) { + if ("remote_out_queue_used_percentage".equals(name)) { + queueUsed.set(value); + } + if ("remote_out_stream_inflight".equals(name)) { + inflight.set(value); + } + } + + @Override + public double getValue() { + return 0; + } + }; + }); + return metricsCreator; + } + + @Test + public void interruptWaitingForStreamSlotRecordsInterruptedOnly() throws Exception { + final List errorReasons = new ArrayList<>(); + final AtomicInteger errorCount = new AtomicInteger(); + final AtomicReference blockSeconds = new AtomicReference<>(0d); + + MetricsCreator metricsCreator = mock(MetricsCreator.class); + when(metricsCreator.createCounter(any(), any(), any(), any())).thenAnswer(invocation -> { + final String name = invocation.getArgument(0); + final org.apache.skywalking.oap.server.telemetry.api.MetricsTag.Values values = invocation.getArgument(3); + return new CounterMetrics() { + @Override + public void inc() { + if ("remote_out_error_count".equals(name)) { + errorCount.incrementAndGet(); + if (values.getValues().length >= 3) { + errorReasons.add(values.getValues()[2]); + } + } + } + + @Override + public void inc(double value) { + if ("remote_out_stream_block_seconds".equals(name)) { + blockSeconds.updateAndGet(current -> current + value); + } + } + }; + }); + when(metricsCreator.createHistogramMetric(any(), any(), any(), any())).thenReturn(new HistogramMetrics() { @Override public void observe(double value) { - } }); + when(metricsCreator.createGauge(any(), any(), any(), any())).thenReturn( + new org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics() { + @Override + public void inc() { + } + + @Override + public void inc(double value) { + } + + @Override + public void dec() { + } + + @Override + public void dec(double value) { + } + + @Override + public void setValue(double value) { + } + + @Override + public double getValue() { + return 0; + } + }); ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); - serviceRegistry.addService(new RemoteServiceHandler(moduleManager)); + Address address = new Address("not-important", 100, false); + GRPCRemoteClient remoteClient = new GRPCRemoteClient(moduleManager, address, 10, 10, null); + + Field inflight = GRPCRemoteClient.class.getDeclaredField("concurrentStreamObserverNumber"); + inflight.setAccessible(true); + // At the max, the waiter blocks; one more would also block, but the limit itself is 10. + ((AtomicInteger) inflight.get(remoteClient)).set(GRPCRemoteClient.MAX_CONCURRENT_STREAMS); + + Method createStreamObserver = GRPCRemoteClient.class.getDeclaredMethod( + "createStreamObserver", long[].class, int.class, AtomicBoolean.class, AtomicBoolean.class, AtomicBoolean.class); + createStreamObserver.setAccessible(true); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference failure = new AtomicReference<>(); + Thread waiter = new Thread(() -> { + started.countDown(); + try { + createStreamObserver.invoke( + remoteClient, new long[1], 1, new AtomicBoolean(false), new AtomicBoolean(false), new AtomicBoolean(false)); + failure.set(new AssertionError("expected interrupt while waiting for a stream slot")); + } catch (InvocationTargetException ex) { + failure.set(ex.getCause()); + } catch (Throwable t) { + failure.set(t); + } + }, "remote-stream-slot-waiter"); + waiter.start(); + Assertions.assertTrue(started.await(2, TimeUnit.SECONDS)); + TimeUnit.MILLISECONDS.sleep(30); + waiter.interrupt(); + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assertions.assertFalse(waiter.isAlive(), "waiter thread should exit after interrupt"); + + Assertions.assertInstanceOf(GRPCRemoteClient.RemoteStreamInterruptedException.class, failure.get()); + Assertions.assertEquals(1, errorCount.get()); + Assertions.assertEquals(List.of(RemoteClientMetrics.REASON_INTERRUPTED), errorReasons); + Assertions.assertTrue(blockSeconds.get() >= 0d); + } - Address address = new Address("not-important", 11, false); + @Test + public void pushWhenClosedSkipsErrorOutAndLatency() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflight = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflight); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + Address address = new Address("not-important", 14, false); GRPCRemoteClient remoteClient = spy(new GRPCRemoteClient(moduleManager, address, 10, 10, null)); remoteClient.connect(); - doReturn(channel).when(remoteClient).getChannel(); + remoteClient.close(); + + remoteClient.push(nextWorkerName, new TestStreamData()); + + Assertions.assertEquals(0, outCount.get(), "closed client must not count remote_out"); + Assertions.assertEquals(0, latencyCount.get(), "closed client must not record latency"); + Assertions.assertTrue(errorReasons.isEmpty(), "closed client must not inflate remote_out_error"); + } + + @Test + public void endStreamAfterCloseSkipsLatency() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflight = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflight); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + Address address = new Address("not-important", 15, false); + GRPCRemoteClient remoteClient = new GRPCRemoteClient(moduleManager, address, 10, 10, null); + remoteClient.connect(); + remoteClient.close(); + + Method endStream = GRPCRemoteClient.class.getDeclaredMethod( + "endStream", AtomicBoolean.class, long.class, int.class, boolean.class); + endStream.setAccessible(true); + endStream.invoke(remoteClient, new AtomicBoolean(false), System.nanoTime(), 3, true); - for (int i = 0; i < 12; i++) { + Assertions.assertEquals(0, latencyCount.get(), "late success callback after close must not observe latency"); + } + + @Test + public void closeFlushesPendingPushesWithoutRemoteOutError() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflight = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflight); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + serviceRegistry.addService(new RemoteServiceHandler(moduleManager)); + + Address address = new Address("not-important", 18, false); + GRPCRemoteClient remoteClient = spy(new GRPCRemoteClient(moduleManager, address, 10, 10, null)); + try { + remoteClient.connect(); + doReturn(channel).when(remoteClient).getChannel(); + + final int pushes = 20; + for (int i = 0; i < pushes; i++) { + remoteClient.push(nextWorkerName, new TestStreamData()); + } + // close() sets closing, then BatchQueue final-drain must still open streams and flush. + remoteClient.close(); + + awaitCondition( + "final drain to flush queued pushes", + 10, + () -> outCount.get() >= pushes + ); + Assertions.assertTrue( + errorReasons.isEmpty(), + "closing/drain must not inflate remote_out_error, got " + errorReasons + ); + Assertions.assertEquals(0.0, inflight.get() == null ? 0.0 : inflight.get(), 0.01); + + final int afterDrain = outCount.get(); remoteClient.push(nextWorkerName, new TestStreamData()); + Assertions.assertEquals(afterDrain, outCount.get(), "push after close must not send"); + } finally { + try { + remoteClient.close(); + } catch (Throwable ignored) { + // close is idempotent for this test + } } + } - TimeUnit.SECONDS.sleep(2); + @Test + public void waitForStreamSlotWhileClosingStillAcquires() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflightGauge = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflightGauge); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + Address address = new Address("not-important", 19, false); + GRPCRemoteClient client = spy(new GRPCRemoteClient(moduleManager, address, 10, 10, null)); + client.connect(); + doReturn(channel).when(client).getChannel(); + serviceRegistry.addService(new RemoteServiceHandler(moduleManager)); + + Field closingField = GRPCRemoteClient.class.getDeclaredField("closing"); + closingField.setAccessible(true); + closingField.setBoolean(client, true); + + Field inflightField = GRPCRemoteClient.class.getDeclaredField("concurrentStreamObserverNumber"); + inflightField.setAccessible(true); + ((AtomicInteger) inflightField.get(client)).set(GRPCRemoteClient.MAX_CONCURRENT_STREAMS); + + Method createStreamObserver = GRPCRemoteClient.class.getDeclaredMethod( + "createStreamObserver", long[].class, int.class, AtomicBoolean.class, AtomicBoolean.class, AtomicBoolean.class); + createStreamObserver.setAccessible(true); + + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + AtomicReference waiterFailure = new AtomicReference<>(); + Thread waiter = new Thread(() -> { + started.countDown(); + try { + createStreamObserver.invoke( + client, new long[1], 1, new AtomicBoolean(false), new AtomicBoolean(false), new AtomicBoolean(false)); + } catch (InvocationTargetException ex) { + waiterFailure.set(ex.getCause()); + } catch (Throwable t) { + waiterFailure.set(t); + } finally { + finished.countDown(); + } + }, "remote-wait-while-closing"); + waiter.start(); + Assertions.assertTrue(started.await(2, TimeUnit.SECONDS)); + TimeUnit.MILLISECONDS.sleep(40); + + // Free one slot as if a stream finished mid-drain; closing must still allow acquire. + Method endStream = GRPCRemoteClient.class.getDeclaredMethod( + "endStream", AtomicBoolean.class, long.class, int.class, boolean.class); + endStream.setAccessible(true); + endStream.invoke(client, new AtomicBoolean(false), System.nanoTime(), 1, false); + + Assertions.assertTrue(finished.await(5, TimeUnit.SECONDS), "closing waiter must acquire a stream slot"); + Assertions.assertNull(waiterFailure.get(), "closing must not abort drain stream wait: " + waiterFailure.get()); + + Method acquire = GRPCRemoteClient.class.getDeclaredMethod("acquireStreamSlot"); + acquire.setAccessible(true); + client.close(); + Assertions.assertEquals(Boolean.FALSE, acquire.invoke(client), "fully closed client must refuse streams"); + } + + @Test + public void acquireAndWaitAfterCloseDoNotInflateInflight() throws Exception { + final AtomicInteger outCount = new AtomicInteger(); + final AtomicInteger latencyCount = new AtomicInteger(); + final List errorReasons = new ArrayList<>(); + final AtomicReference queueUsed = new AtomicReference<>(null); + final AtomicReference inflightGauge = new AtomicReference<>(null); + + MetricsCreator metricsCreator = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflightGauge); + ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); + telemetryModuleDefine.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + + Address addressH1 = new Address("not-important", 16, false); + GRPCRemoteClient clientH1 = new GRPCRemoteClient(moduleManager, addressH1, 10, 10, null); + clientH1.connect(); + clientH1.close(); + Method acquire = GRPCRemoteClient.class.getDeclaredMethod("acquireStreamSlot"); + acquire.setAccessible(true); + Assertions.assertEquals(Boolean.FALSE, acquire.invoke(clientH1)); + Assertions.assertEquals(0.0, inflightGauge.get(), 0.01, "closed client must not re-inflate inflight"); + + final AtomicReference inflightGauge2 = new AtomicReference<>(null); + MetricsCreator metricsCreator2 = mockMetricsCreator(outCount, latencyCount, errorReasons, queueUsed, inflightGauge2); + ModuleDefineTesting telemetryModuleDefine2 = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine2); + telemetryModuleDefine2.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator2); + + Address addressH2 = new Address("not-important", 17, false); + GRPCRemoteClient clientH2 = new GRPCRemoteClient(moduleManager, addressH2, 10, 10, null); + Field inflightField = GRPCRemoteClient.class.getDeclaredField("concurrentStreamObserverNumber"); + inflightField.setAccessible(true); + ((AtomicInteger) inflightField.get(clientH2)).set(GRPCRemoteClient.MAX_CONCURRENT_STREAMS); + + Method createStreamObserver = GRPCRemoteClient.class.getDeclaredMethod( + "createStreamObserver", long[].class, int.class, AtomicBoolean.class, AtomicBoolean.class, AtomicBoolean.class); + createStreamObserver.setAccessible(true); + + CountDownLatch started = new CountDownLatch(1); + AtomicReference waiterFailure = new AtomicReference<>(); + Thread waiter = new Thread(() -> { + started.countDown(); + try { + createStreamObserver.invoke( + clientH2, new long[1], 1, new AtomicBoolean(false), new AtomicBoolean(false), new AtomicBoolean(false)); + waiterFailure.set(new AssertionError("expected closed client to refuse stream create")); + } catch (InvocationTargetException ex) { + waiterFailure.set(ex.getCause()); + } catch (Throwable t) { + waiterFailure.set(t); + } + }, "remote-wait-after-close"); + waiter.start(); + Assertions.assertTrue(started.await(2, TimeUnit.SECONDS)); + TimeUnit.MILLISECONDS.sleep(40); + clientH2.close(); + waiter.join(TimeUnit.SECONDS.toMillis(5)); + Assertions.assertFalse(waiter.isAlive()); + Assertions.assertInstanceOf(GRPCRemoteClient.RemoteClientClosedException.class, waiterFailure.get()); + Assertions.assertTrue( + waiterFailure.get().getMessage().contains("closed"), + "waiter should fail because the client closed" + ); + Assertions.assertTrue( + inflightGauge2.get() == null || Math.abs(inflightGauge2.get()) < 0.01, + "closed wait must not leave a non-zero inflight gauge" + ); } public static class TestStreamData extends StreamData { diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientManagerTestCase.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientManagerTestCase.java index 93b619482935..362a77f69632 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientManagerTestCase.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientManagerTestCase.java @@ -25,7 +25,9 @@ import org.apache.skywalking.oap.server.core.cluster.RemoteInstance; import org.apache.skywalking.oap.server.core.status.ServerStatusService; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; import org.apache.skywalking.oap.server.testing.module.ModuleDefineTesting; import org.apache.skywalking.oap.server.testing.module.ModuleManagerTesting; @@ -42,6 +44,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicReference; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; @@ -53,6 +56,7 @@ public class RemoteClientManagerTestCase { private RemoteClientManager clientManager; private ClusterNodesQuery clusterNodesQuery; + private final AtomicReference clusterSize = new AtomicReference<>(); @AfterEach public void tearDown() { @@ -76,36 +80,55 @@ public void setup() { clusterModuleDefine.provider().registerServiceImplementation(ClusterNodesQuery.class, clusterNodesQuery); MetricsCreator metricsCreator = mock(MetricsCreator.class); - when(metricsCreator.createGauge(any(), any(), any(), any())).thenReturn(new GaugeMetrics() { + when(metricsCreator.createCounter(any(), any(), any(), any())).thenAnswer(invocation -> new CounterMetrics() { @Override public void inc() { - } @Override public void inc(double value) { - } - + }); + when(metricsCreator.createHistogramMetric(any(), any(), any(), any())).thenAnswer(invocation -> new HistogramMetrics() { @Override - public void dec() { - + public void observe(double value) { } + }); + when(metricsCreator.createGauge(any(), any(), any(), any())).thenAnswer(invocation -> { + final String name = invocation.getArgument(0); + return new GaugeMetrics() { + @Override + public void inc() { - @Override - public void dec(double value) { + } - } + @Override + public void inc(double value) { - @Override - public void setValue(double value) { + } - } + @Override + public void dec() { - @Override - public double getValue() { - return 0; - } + } + + @Override + public void dec(double value) { + + } + + @Override + public void setValue(double value) { + if ("cluster_size".equals(name)) { + clusterSize.set(value); + } + } + + @Override + public double getValue() { + return "cluster_size".equals(name) ? clusterSize.get() : 0; + } + }; }); ModuleDefineTesting telemetryModuleDefine = new ModuleDefineTesting(); moduleManager.put(TelemetryModule.NAME, telemetryModuleDefine); @@ -118,6 +141,7 @@ public double getValue() { public void refresh() { when(clusterNodesQuery.queryRemoteNodes()).thenReturn(groupOneInstances()); clientManager.refresh(); + Assertions.assertEquals(3.0, clusterSize.get(), 0.01); List remoteClients = clientManager.getRemoteClient(); Assertions.assertEquals("host1", remoteClients.get(0).getAddress().getHost()); @@ -130,6 +154,7 @@ public void refresh() { when(clusterNodesQuery.queryRemoteNodes()).thenReturn(groupTwoInstances()); clientManager.refresh(); + Assertions.assertEquals(4.0, clusterSize.get(), 0.01); remoteClients = clientManager.getRemoteClient(); Assertions.assertEquals("host1", remoteClients.get(0).getAddress().getHost()); diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java new file mode 100644 index 000000000000..6425225cb198 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.remote.client; + +import io.grpc.Status; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class RemoteClientMetricsTest { + + @Test + public void reasonOfMapsGrpcStatus() { + Assertions.assertEquals(RemoteClientMetrics.REASON_TIMEOUT, RemoteClientMetrics.reasonOf(Status.DEADLINE_EXCEEDED)); + Assertions.assertEquals(RemoteClientMetrics.REASON_IO_ERROR, RemoteClientMetrics.reasonOf(Status.UNAVAILABLE)); + Assertions.assertEquals(RemoteClientMetrics.REASON_RATE_LIMITED, RemoteClientMetrics.reasonOf(Status.RESOURCE_EXHAUSTED)); + Assertions.assertEquals(RemoteClientMetrics.REASON_AUTH_FAILED, RemoteClientMetrics.reasonOf(Status.UNAUTHENTICATED)); + Assertions.assertEquals(RemoteClientMetrics.REASON_AUTH_FAILED, RemoteClientMetrics.reasonOf(Status.PERMISSION_DENIED)); + Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.INTERNAL)); + Assertions.assertEquals(RemoteClientMetrics.REASON_REJECTED, RemoteClientMetrics.reasonOf(Status.CANCELLED)); + Assertions.assertEquals(RemoteClientMetrics.REASON_ERROR, RemoteClientMetrics.reasonOf(null)); + } + + @Test + public void forGrpcRecordsLabeledErrorsAndLatency() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + + metrics.incOut(); + metrics.observeLatencySeconds(0.2); + metrics.recordError(RemoteClientMetrics.REASON_RATE_LIMITED); + metrics.setStreamInflight(3); + metrics.addStreamBlockSeconds(1.5); + metrics.setQueueUsedPercentage(42); + + Assertions.assertEquals(1, creator.counterIncs.get("remote_out_count")); + Assertions.assertEquals(1, creator.counterIncs.get("remote_out_error_count")); + Assertions.assertTrue(creator.lastErrorTags.get().contains("rate_limited")); + Assertions.assertEquals(1, creator.latencyObservations.get()); + Assertions.assertEquals(3.0, creator.gaugeValues.get("remote_out_stream_inflight"), 0.01); + Assertions.assertEquals(1.5, creator.counterIncAmounts.get("remote_out_stream_block_seconds"), 0.01); + Assertions.assertEquals(42.0, creator.gaugeValues.get("remote_out_queue_used_percentage"), 0.01); + } + + @Test + public void forSelfSkipsStreamAndQueueSeries() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + RemoteClientMetrics.forSelf(creator, new Address("oap1", 11800, true)); + Assertions.assertFalse(creator.createdNames.contains("remote_out_stream_inflight")); + Assertions.assertFalse(creator.createdNames.contains("remote_out_queue_used_percentage")); + Assertions.assertTrue(creator.createdNames.contains("remote_out_count")); + Assertions.assertTrue(creator.createdNames.contains("remote_out_latency")); + } + + @Test + public void clearPressureGaugesZerosStreamAndQueue() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + metrics.setStreamInflight(4); + metrics.setQueueUsedPercentage(77); + metrics.clearPressureGauges(); + Assertions.assertEquals(0.0, creator.gaugeValues.get("remote_out_stream_inflight"), 0.01); + Assertions.assertEquals(0.0, creator.gaugeValues.get("remote_out_queue_used_percentage"), 0.01); + } + + @Test + public void queueUsedPercentageIsClamped() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + metrics.setQueueUsedPercentage(-5); + Assertions.assertEquals(0.0, creator.gaugeValues.get("remote_out_queue_used_percentage"), 0.01); + metrics.setQueueUsedPercentage(150); + Assertions.assertEquals(100.0, creator.gaugeValues.get("remote_out_queue_used_percentage"), 0.01); + } + + @Test + public void zeroBlockSecondsAreIgnoredAfterSeed() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + // Construction seeds the Prometheus child with inc(0) so scrapes always see the series. + Assertions.assertEquals(0.0, creator.counterIncAmounts.get("remote_out_stream_block_seconds"), 0.01); + metrics.addStreamBlockSeconds(0); + metrics.addStreamBlockSeconds(-1); + Assertions.assertEquals(0.0, creator.counterIncAmounts.get("remote_out_stream_block_seconds"), 0.01); + } + + @Test + public void errorReasonsAreSeededAtConstruction() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + final long errorSeries = creator.createdNames.stream().filter("remote_out_error_count"::equals).count(); + Assertions.assertEquals(8, errorSeries, "every stable reason label must be pre-registered"); + Assertions.assertEquals(0.0, creator.counterIncAmounts.getOrDefault("remote_out_error_count", -1.0), 0.01); + } + + @Test + public void reasonOfCoversAdditionalServerFailures() { + Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.UNKNOWN)); + Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.DATA_LOSS)); + Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.UNIMPLEMENTED)); + Assertions.assertEquals(RemoteClientMetrics.REASON_REJECTED, RemoteClientMetrics.reasonOf(Status.INVALID_ARGUMENT)); + } + + @Test + public void recordErrorNullFallsBackToError() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + metrics.recordError(null); + metrics.recordError(""); + Assertions.assertEquals(2, creator.counterIncs.get("remote_out_error_count")); + Assertions.assertTrue(creator.lastErrorTags.get().endsWith("error")); + } + + @Test + public void interruptedReasonIsRecordedDirectly() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + metrics.recordError(RemoteClientMetrics.REASON_INTERRUPTED); + Assertions.assertEquals(1, creator.counterIncs.get("remote_out_error_count")); + Assertions.assertTrue(creator.lastErrorTags.get().contains(RemoteClientMetrics.REASON_INTERRUPTED)); + } + + @Test + public void negativeLatencyIsClamped() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + metrics.observeLatencySeconds(-1); + Assertions.assertEquals(1, creator.latencyObservations.get()); + Assertions.assertEquals(0.0, creator.lastLatencyValue.get(), 0.01); + } + + @Test + public void streamInflightIsClampedAtZero() { + final RecordingMetricsCreator creator = new RecordingMetricsCreator(); + final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); + metrics.setStreamInflight(-3); + Assertions.assertEquals(0.0, creator.gaugeValues.get("remote_out_stream_inflight"), 0.01); + } + + @Test + public void forSelfAndForGrpcUseDistinctSelfTags() { + final RecordingMetricsCreator selfCreator = new RecordingMetricsCreator(); + final RecordingMetricsCreator peerCreator = new RecordingMetricsCreator(); + RemoteClientMetrics.forSelf(selfCreator, new Address("oap1", 11800, true)); + RemoteClientMetrics.forGrpc(peerCreator, new Address("oap2", 11800, false)); + Assertions.assertTrue(selfCreator.createdTagSets.contains("oap1_11800,Y")); + Assertions.assertTrue(peerCreator.createdTagSets.contains("oap2_11800,N")); + Assertions.assertFalse(selfCreator.createdTagSets.contains("oap1_11800,N")); + } + + private static final class RecordingMetricsCreator implements MetricsCreator { + private final List createdNames = new ArrayList<>(); + private final List createdTagSets = new ArrayList<>(); + private final java.util.Map counterIncs = new java.util.HashMap<>(); + private final java.util.Map counterIncAmounts = new java.util.HashMap<>(); + private final java.util.Map gaugeValues = new java.util.HashMap<>(); + private final AtomicInteger latencyObservations = new AtomicInteger(); + private final AtomicReference lastErrorTags = new AtomicReference<>(""); + private final AtomicReference lastLatencyValue = new AtomicReference<>(); + + @Override + public CounterMetrics createCounter(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) { + createdNames.add(name); + final String tagJoined = String.join(",", tagValues.getValues()); + createdTagSets.add(tagJoined); + return new CounterMetrics() { + @Override + public void inc() { + counterIncs.merge(name, 1, Integer::sum); + if ("remote_out_error_count".equals(name)) { + lastErrorTags.set(tagJoined); + } + } + + @Override + public void inc(double value) { + counterIncAmounts.merge(name, value, Double::sum); + } + }; + } + + @Override + public GaugeMetrics createGauge(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) { + createdNames.add(name); + createdTagSets.add(String.join(",", tagValues.getValues())); + return new GaugeMetrics() { + @Override + public void inc() { + } + + @Override + public void inc(double value) { + } + + @Override + public void dec() { + } + + @Override + public void dec(double value) { + } + + @Override + public void setValue(double value) { + gaugeValues.put(name, value); + } + + @Override + public double getValue() { + return gaugeValues.getOrDefault(name, 0d); + } + }; + } + + @Override + public HistogramMetrics createHistogramMetric(String name, String tips, MetricsTag.Keys tagKeys, + MetricsTag.Values tagValues, double... buckets) { + createdNames.add(name); + createdTagSets.add(String.join(",", tagValues.getValues())); + return new HistogramMetrics() { + @Override + public void observe(double value) { + latencyObservations.incrementAndGet(); + lastLatencyValue.set(value); + } + }; + } + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java new file mode 100644 index 000000000000..40abb10ff7cd --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.remote.client; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.UnexpectedException; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricStreamKind; +import org.apache.skywalking.oap.server.core.remote.data.StreamData; +import org.apache.skywalking.oap.server.core.remote.grpc.proto.RemoteData; +import org.apache.skywalking.oap.server.core.worker.AbstractWorker; +import org.apache.skywalking.oap.server.core.worker.IWorkerInstanceGetter; +import org.apache.skywalking.oap.server.core.worker.IWorkerInstanceSetter; +import org.apache.skywalking.oap.server.core.worker.WorkerInstancesService; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; +import org.apache.skywalking.oap.server.telemetry.TelemetryModule; +import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; +import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator; +import org.apache.skywalking.oap.server.telemetry.api.MetricsTag; +import org.apache.skywalking.oap.server.testing.module.ModuleDefineTesting; +import org.apache.skywalking.oap.server.testing.module.ModuleManagerTesting; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class SelfRemoteClientTest { + private static final String WORKER = "self-remote-test-worker"; + + private ModuleManagerTesting moduleManager; + private WorkerInstancesService workers; + private final Map counterIncs = new HashMap<>(); + private final AtomicInteger latencyObservations = new AtomicInteger(); + private final AtomicReference lastLatencySeconds = new AtomicReference<>(); + private final Map lastErrorReason = new HashMap<>(); + private final Map counterTags = new HashMap<>(); + private String[] latencyTags; + + @BeforeEach + public void setUp() { + moduleManager = new ModuleManagerTesting(); + ModuleDefineTesting core = new ModuleDefineTesting(); + moduleManager.put(CoreModule.NAME, core); + workers = new WorkerInstancesService(); + core.provider().registerServiceImplementation(IWorkerInstanceGetter.class, workers); + core.provider().registerServiceImplementation(IWorkerInstanceSetter.class, workers); + + MetricsCreator metricsCreator = mock(MetricsCreator.class); + when(metricsCreator.createCounter(any(), any(), any(), any())).thenAnswer(invocation -> { + final String name = invocation.getArgument(0); + final MetricsTag.Values values = invocation.getArgument(3); + counterTags.put(name, values.getValues()); + return new CounterMetrics() { + @Override + public void inc() { + counterIncs.computeIfAbsent(name, ignored -> new AtomicInteger()).incrementAndGet(); + if ("remote_out_error_count".equals(name) && values.getValues().length >= 3) { + lastErrorReason.put(name, values.getValues()[2]); + } + } + + @Override + public void inc(double value) { + } + }; + }); + when(metricsCreator.createHistogramMetric(any(), any(), any(), any())).thenAnswer(invocation -> { + final MetricsTag.Values values = invocation.getArgument(3); + latencyTags = values.getValues(); + return new HistogramMetrics() { + @Override + public void observe(double value) { + latencyObservations.incrementAndGet(); + lastLatencySeconds.set(value); + } + }; + }); + when(metricsCreator.createGauge(any(), any(), any(), any())).thenReturn(noopGauge()); + + ModuleDefineTesting telemetry = new ModuleDefineTesting(); + moduleManager.put(TelemetryModule.NAME, telemetry); + telemetry.provider().registerServiceImplementation(MetricsCreator.class, metricsCreator); + } + + @Test + public void pushSuccessRecordsOutAndLatency() { + workers.put(WORKER, new AcceptingWorker(moduleManager), MetricStreamKind.OAL, TestStreamData.class); + SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); + client.push(WORKER, new TestStreamData()); + + Assertions.assertEquals(1, counterIncs.get("remote_out_count").get()); + Assertions.assertEquals(1, latencyObservations.get()); + Assertions.assertFalse(counterIncs.containsKey("remote_out_error_count")); + Assertions.assertArrayEquals(new String[] {"oap1_11800", "Y"}, counterTags.get("remote_out_count")); + Assertions.assertArrayEquals(new String[] {"oap1_11800", "Y"}, latencyTags); + // Local fan-out is in-process; avoid a sub-ms bound (JVM scheduling can exceed it). + Assertions.assertNotNull(lastLatencySeconds.get()); + Assertions.assertTrue( + lastLatencySeconds.get() >= 0 && lastLatencySeconds.get() < 0.1, + "self latency should stay well under 100ms, got " + lastLatencySeconds.get() + ); + } + + @Test + public void repeatedSuccessfulPushesEachObserveLatency() { + workers.put(WORKER, new AcceptingWorker(moduleManager), MetricStreamKind.OAL, TestStreamData.class); + SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); + for (int i = 0; i < 5; i++) { + client.push(WORKER, new TestStreamData()); + } + Assertions.assertEquals(5, counterIncs.get("remote_out_count").get()); + Assertions.assertEquals(5, latencyObservations.get(), "self client observes latency per successful push"); + } + + @Test + public void pushFailureRecordsErrorReason() { + workers.put(WORKER, new FailingWorker(moduleManager), MetricStreamKind.OAL, TestStreamData.class); + SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); + client.push(WORKER, new TestStreamData()); + + Assertions.assertEquals(1, counterIncs.get("remote_out_error_count").get()); + Assertions.assertEquals(RemoteClientMetrics.REASON_ERROR, lastErrorReason.get("remote_out_error_count")); + Assertions.assertEquals(0, latencyObservations.get()); + } + + @Test + public void pushMissingWorkerRecordsErrorReason() { + SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); + client.push("missing-worker", new TestStreamData()); + + Assertions.assertEquals(1, counterIncs.get("remote_out_error_count").get()); + Assertions.assertEquals(RemoteClientMetrics.REASON_ERROR, lastErrorReason.get("remote_out_error_count")); + Assertions.assertEquals(0, latencyObservations.get()); + Assertions.assertFalse(counterIncs.containsKey("remote_out_count")); + } + + @Test + public void closeIsUnsupported() { + SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); + Assertions.assertThrows(UnexpectedException.class, client::close); + } + + private static GaugeMetrics noopGauge() { + return new GaugeMetrics() { + @Override + public void inc() { + } + + @Override + public void inc(double value) { + } + + @Override + public void dec() { + } + + @Override + public void dec(double value) { + } + + @Override + public void setValue(double value) { + } + + @Override + public double getValue() { + return 0; + } + }; + } + + public static class TestStreamData extends StreamData { + @Override + public int remoteHashCode() { + return 0; + } + + @Override + public void deserialize(RemoteData remoteData) { + } + + @Override + public RemoteData.Builder serialize() { + return RemoteData.newBuilder().addDataLongs(1); + } + } + + static class AcceptingWorker extends AbstractWorker { + AcceptingWorker(ModuleDefineHolder moduleDefineHolder) { + super(moduleDefineHolder); + } + + @Override + public void in(Object o) { + } + } + + static class FailingWorker extends AbstractWorker { + FailingWorker(ModuleDefineHolder moduleDefineHolder) { + super(moduleDefineHolder); + } + + @Override + public void in(Object o) { + throw new IllegalStateException("boom"); + } + } +} diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml index db0d281989e0..ba5a43c0a1b9 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml @@ -152,3 +152,26 @@ metricsRules: exp: alarm_http_delivery_status_total.sum(['service', 'host_name', 'hook_type', 'status_class']).increase('PT1M') - name: instance_alarm_http_delivery_latency_percentile exp: alarm_http_delivery_latency_seconds.sum(['le', 'service', 'host_name', 'hook_type']).increase('PT1M').histogram().histogram_percentile([50,75,90,95,99]) + - name: instance_remote_out_count + exp: remote_out_count.sum(['service', 'host_name', 'dest', 'self']).increase('PT1M') + - name: instance_remote_out_error_count + exp: remote_out_error_count.sum(['service', 'host_name', 'dest', 'self', 'reason']).increase('PT1M') + - name: instance_remote_out_latency_percentile + exp: remote_out_latency.sum(['le', 'service', 'host_name', 'dest', 'self']).increase('PT1M').histogram().histogram_percentile([50,75,90,95,99]) + - name: instance_remote_out_stream_inflight + exp: remote_out_stream_inflight.sum(['service', 'host_name', 'dest', 'self']) + # Store as ms / per-ten-thousand so MAL long storage does not bump (0,1) samples to 1. + - name: instance_remote_out_stream_block_milliseconds + exp: (remote_out_stream_block_seconds * 1000).sum(['service', 'host_name', 'dest', 'self']).increase('PT1M') + - name: instance_remote_out_queue_used_per_ten_thousand + exp: 100 * remote_out_queue_used_percentage.sum(['service', 'host_name', 'dest', 'self']) + - name: instance_remote_in_count + exp: remote_in_count.sum(['service', 'host_name']).increase('PT1M') + - name: instance_remote_in_error_count + exp: remote_in_error_count.sum(['service', 'host_name']).increase('PT1M') + - name: instance_remote_in_target_not_found_count + exp: remote_in_target_not_found_count.sum(['service', 'host_name']).increase('PT1M') + - name: instance_remote_in_latency_percentile + exp: remote_in_latency.sum(['le', 'service', 'host_name']).increase('PT1M').histogram().histogram_percentile([50,75,90,95,99]) + - name: instance_cluster_size + exp: cluster_size.sum(['service', 'host_name']) diff --git a/test/e2e-v2/cases/cluster/cluster-cases.yaml b/test/e2e-v2/cases/cluster/cluster-cases.yaml index 675dff3840b4..9762498f10ff 100644 --- a/test/e2e-v2/cases/cluster/cluster-cases.yaml +++ b/test/e2e-v2/cases/cluster/cluster-cases.yaml @@ -21,14 +21,49 @@ curl -s -XPOST http://${sender1_host}:${sender1_9093}/sendBatchMetrics/1642647600000/100 > /dev/null; curl -s -XPOST http://${sender2_host}:${sender2_9093}/sendBatchMetrics/1642647600000/100 > /dev/null; sleep 10; - curl -s http://${oap1_host}:${oap1_1234} | grep remote_out_count{ | awk '{print $1}' + curl -s http://${oap1_host}:${oap1_1234} | grep -E '^(remote_out_count|remote_out_latency_count|remote_out_stream_inflight|remote_out_stream_block_seconds|remote_out_queue_used_percentage|remote_in_count|remote_in_latency_count|cluster_size)\{' | awk '{print $1}' | sort expected: expected/cluster-up-oap1.yml - query: | curl -s -XPOST http://${sender1_host}:${sender1_9093}/sendBatchMetrics/1642647600000/100 > /dev/null; curl -s -XPOST http://${sender2_host}:${sender2_9093}/sendBatchMetrics/1642647600000/100 > /dev/null; - sleep 5; - curl -s http://${oap2_host}:${oap2_1234} | grep remote_out_count{ | awk '{print $1}' + sleep 10; + curl -s http://${oap2_host}:${oap2_1234} | grep -E '^(remote_out_count|remote_out_latency_count|remote_out_stream_inflight|remote_out_stream_block_seconds|remote_out_queue_used_percentage|remote_in_count|remote_in_latency_count|cluster_size)\{' | awk '{print $1}' | sort expected: expected/cluster-up-oap2.yml + - query: | + curl -s http://${oap1_host}:${oap1_1234} | awk '/^cluster_size\{/{printf "%.0f\n", $2}' + expected: expected/cluster-size-value.yml + - query: | + curl -s http://${oap2_host}:${oap2_1234} | awk '/^cluster_size\{/{printf "%.0f\n", $2}' + expected: expected/cluster-size-value.yml + # Semantics: self latency near-zero; peer has out+latency; inbound remote traffic present. + - query: | + curl -s http://${oap1_host}:${oap1_1234} | awk ' + /^remote_out_latency_sum\{[^}]*self="Y"/{ys=$2} + /^remote_out_latency_count\{[^}]*self="Y"/{yc=$2} + /^remote_out_latency_sum\{[^}]*self="N"/{ns=$2} + /^remote_out_latency_count\{[^}]*self="N"/{nc=$2} + /^remote_out_count\{[^}]*self="N"/{no=$2} + /^remote_in_count\{/{ic=$2} + END { + if (yc+0 < 1 || nc+0 < 1 || no+0 < 1 || ic+0 < 1) { print "fail-missing"; exit } + self_avg = ys / yc + if (self_avg < 0.05 && no >= 1 && nc >= 1 && ic >= 1) print "ok" + else printf "fail self_avg=%.6f peer_out=%s peer_lat=%s in=%s\n", self_avg, no, nc, ic + }' + expected: expected/remote-so11y-semantics.yml + - query: | + curl -s http://${oap2_host}:${oap2_1234} | awk ' + /^remote_out_latency_sum\{[^}]*self="Y"/{ys=$2} + /^remote_out_latency_count\{[^}]*self="Y"/{yc=$2} + /^remote_out_count\{[^}]*self="N"/{no=$2} + /^remote_in_count\{/{ic=$2} + END { + if (yc+0 < 1 || no+0 < 1 || ic+0 < 1) { print "fail-missing"; exit } + self_avg = ys / yc + if (self_avg < 0.05 && no >= 1 && ic >= 1) print "ok" + else printf "fail self_avg=%.6f peer_out=%s in=%s\n", self_avg, no, ic + }' + expected: expected/remote-so11y-semantics.yml # send metrics to each oap first time, UTC 2022-01-26 03:00:00 - query: | curl -s -XPOST http://${sender1_host}:${sender1_9093}/sendBatchMetrics/1643166000000/100; diff --git a/test/e2e-v2/cases/cluster/expected/cluster-size-value.yml b/test/e2e-v2/cases/cluster/expected/cluster-size-value.yml new file mode 100644 index 000000000000..90debd192223 --- /dev/null +++ b/test/e2e-v2/cases/cluster/expected/cluster-size-value.yml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +2 diff --git a/test/e2e-v2/cases/cluster/expected/cluster-up-oap1.yml b/test/e2e-v2/cases/cluster/expected/cluster-up-oap1.yml index c02b4a544523..198e032deac5 100644 --- a/test/e2e-v2/cases/cluster/expected/cluster-up-oap1.yml +++ b/test/e2e-v2/cases/cluster/expected/cluster-up-oap1.yml @@ -13,5 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +cluster_size{sw_backend_instance="0.0.0.0_11800",} +remote_in_count{sw_backend_instance="0.0.0.0_11800",} +remote_in_latency_count{sw_backend_instance="0.0.0.0_11800",} remote_out_count{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="Y",} remote_out_count{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="N",} +remote_out_latency_count{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="Y",} +remote_out_latency_count{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="N",} +remote_out_queue_used_percentage{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="N",} +remote_out_stream_block_seconds{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="N",} +remote_out_stream_inflight{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="N",} diff --git a/test/e2e-v2/cases/cluster/expected/cluster-up-oap2.yml b/test/e2e-v2/cases/cluster/expected/cluster-up-oap2.yml index 0dceddf48d62..1e0729dcc9cf 100644 --- a/test/e2e-v2/cases/cluster/expected/cluster-up-oap2.yml +++ b/test/e2e-v2/cases/cluster/expected/cluster-up-oap2.yml @@ -13,5 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -remote_out_count{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="Y",} +cluster_size{sw_backend_instance="0.0.0.0_11800",} +remote_in_count{sw_backend_instance="0.0.0.0_11800",} +remote_in_latency_count{sw_backend_instance="0.0.0.0_11800",} remote_out_count{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="N",} +remote_out_count{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="Y",} +remote_out_latency_count{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="N",} +remote_out_latency_count{sw_backend_instance="0.0.0.0_11800",dest="oap2_11800",self="Y",} +remote_out_queue_used_percentage{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="N",} +remote_out_stream_block_seconds{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="N",} +remote_out_stream_inflight{sw_backend_instance="0.0.0.0_11800",dest="oap1_11800",self="N",} diff --git a/test/e2e-v2/cases/cluster/expected/remote-so11y-semantics.yml b/test/e2e-v2/cases/cluster/expected/remote-so11y-semantics.yml new file mode 100644 index 000000000000..e33f0553dd1f --- /dev/null +++ b/test/e2e-v2/cases/cluster/expected/remote-so11y-semantics.yml @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ok diff --git a/test/e2e-v2/cases/cluster/zk/banyandb/banyandb-cluster-cases.yaml b/test/e2e-v2/cases/cluster/zk/banyandb/banyandb-cluster-cases.yaml index 7f551acbdc54..ce6985e9cf31 100644 --- a/test/e2e-v2/cases/cluster/zk/banyandb/banyandb-cluster-cases.yaml +++ b/test/e2e-v2/cases/cluster/zk/banyandb/banyandb-cluster-cases.yaml @@ -28,15 +28,48 @@ curl -s -XPOST http://${sender1_host}:${sender1_9093}/sendBatchMetrics/$NOW/100 > /dev/null; curl -s -XPOST http://${sender2_host}:${sender2_9093}/sendBatchMetrics/$NOW/100 > /dev/null; sleep 10; - curl -s http://${oap1_host}:${oap1_1234} | grep remote_out_count{ | awk '{print $1}' + curl -s http://${oap1_host}:${oap1_1234} | grep -E '^(remote_out_count|remote_out_latency_count|remote_out_stream_inflight|remote_out_stream_block_seconds|remote_out_queue_used_percentage|remote_in_count|remote_in_latency_count|cluster_size)\{' | awk '{print $1}' | sort expected: ../../expected/cluster-up-oap1.yml - query: | NOW=$(( ($(date -u +%s) - 90000) * 1000 )); curl -s -XPOST http://${sender1_host}:${sender1_9093}/sendBatchMetrics/$NOW/100 > /dev/null; curl -s -XPOST http://${sender2_host}:${sender2_9093}/sendBatchMetrics/$NOW/100 > /dev/null; - sleep 5; - curl -s http://${oap2_host}:${oap2_1234} | grep remote_out_count{ | awk '{print $1}' + sleep 10; + curl -s http://${oap2_host}:${oap2_1234} | grep -E '^(remote_out_count|remote_out_latency_count|remote_out_stream_inflight|remote_out_stream_block_seconds|remote_out_queue_used_percentage|remote_in_count|remote_in_latency_count|cluster_size)\{' | awk '{print $1}' | sort expected: ../../expected/cluster-up-oap2.yml + - query: | + curl -s http://${oap1_host}:${oap1_1234} | awk '/^cluster_size\{/{printf "%.0f\n", $2}' + expected: ../../expected/cluster-size-value.yml + - query: | + curl -s http://${oap2_host}:${oap2_1234} | awk '/^cluster_size\{/{printf "%.0f\n", $2}' + expected: ../../expected/cluster-size-value.yml + - query: | + curl -s http://${oap1_host}:${oap1_1234} | awk ' + /^remote_out_latency_sum\{[^}]*self="Y"/{ys=$2} + /^remote_out_latency_count\{[^}]*self="Y"/{yc=$2} + /^remote_out_latency_count\{[^}]*self="N"/{nc=$2} + /^remote_out_count\{[^}]*self="N"/{no=$2} + /^remote_in_count\{/{ic=$2} + END { + if (yc+0 < 1 || nc+0 < 1 || no+0 < 1 || ic+0 < 1) { print "fail-missing"; exit } + self_avg = ys / yc + if (self_avg < 0.05 && no >= 1 && nc >= 1 && ic >= 1) print "ok" + else printf "fail self_avg=%.6f peer_out=%s peer_lat=%s in=%s\n", self_avg, no, nc, ic + }' + expected: ../../expected/remote-so11y-semantics.yml + - query: | + curl -s http://${oap2_host}:${oap2_1234} | awk ' + /^remote_out_latency_sum\{[^}]*self="Y"/{ys=$2} + /^remote_out_latency_count\{[^}]*self="Y"/{yc=$2} + /^remote_out_count\{[^}]*self="N"/{no=$2} + /^remote_in_count\{/{ic=$2} + END { + if (yc+0 < 1 || no+0 < 1 || ic+0 < 1) { print "fail-missing"; exit } + self_avg = ys / yc + if (self_avg < 0.05 && no >= 1 && ic >= 1) print "ok" + else printf "fail self_avg=%.6f peer_out=%s in=%s\n", self_avg, no, ic + }' + expected: ../../expected/remote-so11y-semantics.yml # compute two recent, same-hour minute slots and send the FIRST batch (2 senders x 100 = 200) at T1. # T1 = 5 minutes ago (truncated to the minute); the :59 guard keeps T1 and T2 in the same hour/day. - query: | diff --git a/test/e2e-v2/cases/so11y/expected/metrics-has-value-label-remote.yml b/test/e2e-v2/cases/so11y/expected/metrics-has-value-label-remote.yml new file mode 100644 index 000000000000..9cd86285cfbb --- /dev/null +++ b/test/e2e-v2/cases/so11y/expected/metrics-has-value-label-remote.yml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +debuggingtrace: null +type: TIME_SERIES_VALUES +results: + {{- contains .results }} + - metric: + labels: + {{- contains .metric.labels }} + - key: dest + value: {{ notEmpty .value }} + - key: self + value: "Y" + {{- end}} + values: + {{- contains .values }} + - id: {{ notEmpty .id }} + value: {{ notEmpty .value }} + owner: null + traceid: null + - id: {{ notEmpty .id }} + value: null + owner: null + traceid: null + {{- end}} + {{- end}} +error: null diff --git a/test/e2e-v2/cases/so11y/expected/metrics-has-value-percentile-remote.yml b/test/e2e-v2/cases/so11y/expected/metrics-has-value-percentile-remote.yml new file mode 100644 index 000000000000..bff2db07fdf3 --- /dev/null +++ b/test/e2e-v2/cases/so11y/expected/metrics-has-value-percentile-remote.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +debuggingtrace: null +type: TIME_SERIES_VALUES +results: + {{- contains .results }} +- metric: + labels: + {{- contains .metric.labels }} + - key: p + value: "50" + - key: self + value: "Y" + {{- end}} + values: + {{- contains .values }} + - id: {{ notEmpty .id }} + value: {{ notEmpty .value }} + owner: null + traceid: null + - id: {{ notEmpty .id }} + value: null + owner: null + traceid: null + {{- end}} +- metric: + labels: + {{- contains .metric.labels }} + - key: p + value: "99" + - key: self + value: "Y" + {{- end}} + values: + {{- contains .values }} + - id: {{ notEmpty .id }} + value: {{ notEmpty .value }} + owner: null + traceid: null + - id: {{ notEmpty .id }} + value: null + owner: null + traceid: null + {{- end}} + {{- end}} +error: null diff --git a/test/e2e-v2/cases/so11y/so11y-cases.yaml b/test/e2e-v2/cases/so11y/so11y-cases.yaml index 14e908f102b8..81db1963282f 100644 --- a/test/e2e-v2/cases/so11y/so11y-cases.yaml +++ b/test/e2e-v2/cases/so11y/so11y-cases.yaml @@ -47,6 +47,14 @@ - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="meter_oap_instance_trace_latency_percentile{p='90,99'}" --instance-name=http://localhost:1234 --service-name=oap-server expected: expected/metrics-has-value-percentile.yml + # cluster remote so11y: single-node still fans out via SelfRemoteClient (self=Y) and exposes cluster_size. + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="meter_oap_instance_remote_out_count{self='Y'}" --instance-name=http://localhost:1234 --service-name=oap-server + expected: expected/metrics-has-value-label-remote.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="meter_oap_instance_remote_out_latency_percentile{p='50,99',self='Y'}" --instance-name=http://localhost:1234 --service-name=oap-server + expected: expected/metrics-has-value-percentile-remote.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression=meter_oap_instance_cluster_size --instance-name=http://localhost:1234 --service-name=oap-server + expected: expected/metrics-has-value.yml + # alarm HTTP hook delivery, fed by the webhook in alarm-settings.yml. Alarm evaluation, scraping, MAL # initialization and persistence can outlast the normal retry window, so wait before each attempt for the series. - query: | From 3fc6ea56b26d810dad9d9fa1911d880d3dbf81d6 Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Wed, 18 Nov 2026 14:14:19 +0800 Subject: [PATCH 2/2] Drop reason labels from remote_out_error_count and keep counting async failures. --- docs/en/changes/changes.md | 2 +- docs/en/setup/backend/dashboards-so11y.md | 2 +- docs/en/setup/backend/grafana-cluster.json | 6 +- docs/en/setup/backend/grafana-instance.json | 6 +- .../scripts/mal/test-otel-rules/oap.data.yaml | 2 - .../core/remote/client/GRPCRemoteClient.java | 14 +-- .../remote/client/RemoteClientMetrics.java | 100 ++++-------------- .../core/remote/client/SelfRemoteClient.java | 4 +- .../client/GRPCRemoteClientTestCase.java | 14 ++- .../client/RemoteClientMetricsTest.java | 67 +++--------- .../remote/client/SelfRemoteClientTest.java | 10 +- .../src/main/resources/otel-rules/oap.yaml | 2 +- 12 files changed, 56 insertions(+), 173 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 8b23b7fdb0a0..1aa8685469bb 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -69,7 +69,7 @@ * Fix the async-profiler task OAP sends to an agent whose service has two tasks created within five minutes of each other: it could send the older one. * Fix alarm windows retaining empty layers after an initial metadata lookup miss. Later metrics can supply the missing layers for subsequent alarms, allowing layer-filtered `queryAlarms` queries to match the persisted layer. * Rename the etcd dynamic configuration password environment variable from `SW_CONFIG_ETCD_password` to `SW_CONFIG_ETCD_PASSWORD`, matching every other environment variable. The lower-case name no longer has any effect, so a deployment that sets it must switch to the new name. -* Add cluster remote self-observability metrics (out latency/pressure/error reason, remote in, and cluster size) to the OAP so11y MAL and dashboards. +* Add the existing cluster remote metrics (remote in/out counts, errors, in latency, and cluster size) to the OAP self-observability MAL rules and dashboards, add remote client out latency, in-flight streams, stream-slot wait time and send-queue usage metrics, and count asynchronous gRPC send failures in `remote_out_error_count`, which were only logged before. #### UI diff --git a/docs/en/setup/backend/dashboards-so11y.md b/docs/en/setup/backend/dashboards-so11y.md index a2eb2f60b2e0..0ef981d95cac 100644 --- a/docs/en/setup/backend/dashboards-so11y.md +++ b/docs/en/setup/backend/dashboards-so11y.md @@ -50,7 +50,7 @@ Self observability monitoring provides monitoring of the status and resources of | Count per minute | meter_oap_instance_alarm_http_delivery_status_count | HTTP alarm hook responses by `hook_type` and `status_class` (Per Minute) | oap self observability | | ms | meter_oap_instance_alarm_http_delivery_latency_percentile | Alarm hook delivery latency by `hook_type` (ms), including `gRPC` | oap self observability | | Count per minute | meter_oap_instance_remote_out_count | Cluster remote client push count by `dest` and `self` (Per Minute); peers: send attempts, `self=Y`: successful local fan-out | oap self observability | -| Count per minute | meter_oap_instance_remote_out_error_count | Cluster remote client push errors by `dest`, `self` and `reason` (Per Minute); one sample per failed stream | oap self observability | +| Count per minute | meter_oap_instance_remote_out_error_count | Cluster remote client push errors by `dest` and `self` (Per Minute); one sample per failed stream | oap self observability | | ms | meter_oap_instance_remote_out_latency_percentile | Cluster remote client push latency by `dest` and `self` (ms, one sample per successful stream) | oap self observability | | Count | meter_oap_instance_remote_out_stream_inflight | In-flight remote gRPC streams by `dest` and `self` (peers only) | oap self observability | | Milliseconds per minute | meter_oap_instance_remote_out_stream_block_milliseconds | Time waiting for a free remote stream slot by `dest` and `self` (ms Per Minute, peers only) | oap self observability | diff --git a/docs/en/setup/backend/grafana-cluster.json b/docs/en/setup/backend/grafana-cluster.json index 6863b809daa2..21ffe930769f 100644 --- a/docs/en/setup/backend/grafana-cluster.json +++ b/docs/en/setup/backend/grafana-cluster.json @@ -7312,7 +7312,7 @@ "type": "prometheus", "uid": "$datasource" }, - "description": "Failed peer outbound streams by reason (self=N).", + "description": "Failed peer outbound streams (self=N).", "fieldConfig": { "defaults": { "color": { @@ -7396,8 +7396,8 @@ "type": "prometheus", "uid": "$datasource" }, - "expr": "sum by (dest, reason) (increase(remote_out_error_count{job=\"$job\",self=\"N\"}[1m]))", - "legendFormat": "{{dest}} {{reason}}", + "expr": "sum by (dest) (increase(remote_out_error_count{job=\"$job\",self=\"N\"}[1m]))", + "legendFormat": "{{dest}}", "refId": "A" } ], diff --git a/docs/en/setup/backend/grafana-instance.json b/docs/en/setup/backend/grafana-instance.json index 87e780422bd0..5ff778dae642 100644 --- a/docs/en/setup/backend/grafana-instance.json +++ b/docs/en/setup/backend/grafana-instance.json @@ -7932,17 +7932,17 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum by (dest, reason) (increase(remote_out_error_count{instance=\"$instance\",job=\"$job\",self=\"N\"}[1m]))", + "expr": "sum by (dest) (increase(remote_out_error_count{instance=\"$instance\",job=\"$job\",self=\"N\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, - "legendFormat": "{{dest}} {{reason}}", + "legendFormat": "{{dest}}", "refId": "A" } ], "title": "Remote Out Error Count / Minute", "type": "timeseries", - "description": "Failed peer outbound streams by reason (self=N)." + "description": "Failed peer outbound streams (self=N)." }, { "datasource": { diff --git a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml index c9ff24eafe6d..45c7191b5165 100644 --- a/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml +++ b/oap-server/analyzer/meter-analyzer-scripts-test/src/test/resources/scripts/mal/test-otel-rules/oap.data.yaml @@ -802,7 +802,6 @@ input: host_name: test-host dest: oap2_11800 self: N - reason: rate_limited value: 100.0 remote_out_latency: - labels: @@ -1846,7 +1845,6 @@ expected: host_name: test-host dest: oap2_11800 self: N - reason: rate_limited value: 50.0 meter_oap_instance_remote_out_latency_percentile: entities: diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java index 6a78bd949071..fea853dfb559 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClient.java @@ -20,8 +20,6 @@ import io.grpc.ManagedChannel; import io.grpc.Status; -import io.grpc.StatusException; -import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import io.netty.handler.ssl.SslContext; import java.util.List; @@ -148,7 +146,7 @@ public void push(String nextWorkerName, StreamData streamData) { publishQueueUsed(); if (!this.queue.produce(builder.build())) { if (!closing && !closed) { - metrics.recordError(RemoteClientMetrics.REASON_ERROR); + metrics.recordError(); } return; } @@ -182,11 +180,7 @@ public void consume(List remoteMessages) { endStream(streamFinished, startNanos[0], messageCount, false); } if (!closing && !closed && errorRecorded.compareAndSet(false, true)) { - if (t instanceof StatusRuntimeException || t instanceof StatusException) { - metrics.recordError(RemoteClientMetrics.reasonOf(Status.fromThrowable(t))); - } else { - metrics.recordError(RemoteClientMetrics.REASON_ERROR); - } + metrics.recordError(); } log.error(t.getMessage(), t); } finally { @@ -216,7 +210,7 @@ private StreamObserver createStreamObserver(final long[] startNan Thread.currentThread().interrupt(); blockedNanos += System.nanoTime() - blockStart; metrics.addStreamBlockSeconds(blockedNanos / 1_000_000_000d); - metrics.recordError(RemoteClientMetrics.REASON_INTERRUPTED); + metrics.recordError(); throw new RemoteStreamInterruptedException("Interrupted while waiting for a remote stream slot", e); } blockedNanos += System.nanoTime() - blockStart; @@ -250,7 +244,7 @@ public void onError(Throwable throwable) { endStream(streamFinished, startNanosHolder[0], messageCount, false); Status status = Status.fromThrowable(throwable); if (!closing && !closed && errorRecorded.compareAndSet(false, true)) { - metrics.recordError(RemoteClientMetrics.reasonOf(status)); + metrics.recordError(); } if (closing || closed) { return; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java index 8b5e0b4bbec5..1314ef9d063c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetrics.java @@ -18,10 +18,7 @@ package org.apache.skywalking.oap.server.core.remote.client; -import io.grpc.Status; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import org.apache.skywalking.oap.server.telemetry.api.CounterMetrics; import org.apache.skywalking.oap.server.telemetry.api.GaugeMetrics; import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; @@ -30,36 +27,12 @@ /** Client-side {@code remote_out_*} metrics for cluster remote RPC. */ public final class RemoteClientMetrics { - public static final String REASON_ERROR = "error"; - public static final String REASON_TIMEOUT = "timeout"; - public static final String REASON_IO_ERROR = "io_error"; - public static final String REASON_RATE_LIMITED = "rate_limited"; - public static final String REASON_AUTH_FAILED = "auth_failed"; - public static final String REASON_SERVER_ERROR = "server_error"; - public static final String REASON_REJECTED = "rejected"; - public static final String REASON_INTERRUPTED = "interrupted"; - - private static final String[] ERROR_REASONS = { - REASON_ERROR, - REASON_TIMEOUT, - REASON_IO_ERROR, - REASON_RATE_LIMITED, - REASON_AUTH_FAILED, - REASON_SERVER_ERROR, - REASON_REJECTED, - REASON_INTERRUPTED - }; - - private final MetricsCreator metricsCreator; - private final String dest; - private final String self; - private final CounterMetrics outCounter; private final HistogramMetrics outLatency; + private final CounterMetrics errorCounter; private final GaugeMetrics streamInflight; private final CounterMetrics streamBlockSeconds; private final GaugeMetrics queueUsedPercentage; - private final ConcurrentMap errorCounters = new ConcurrentHashMap<>(); public static RemoteClientMetrics forGrpc(final MetricsCreator metricsCreator, final Address address) { return new RemoteClientMetrics(metricsCreator, address, false, true); @@ -73,51 +46,48 @@ private RemoteClientMetrics(final MetricsCreator metricsCreator, final Address address, final boolean self, final boolean trackStreamAndQueue) { - this.metricsCreator = Objects.requireNonNull(metricsCreator, "metricsCreator"); - this.dest = Objects.requireNonNull(address, "address").toString(); - this.self = self ? "Y" : "N"; + Objects.requireNonNull(metricsCreator, "metricsCreator"); + final String dest = Objects.requireNonNull(address, "address").toString(); + final String selfTag = self ? "Y" : "N"; this.outCounter = metricsCreator.createCounter( "remote_out_count", "The number(client side) of inside remote inside aggregate rpc.", new MetricsTag.Keys("dest", "self"), - new MetricsTag.Values(this.dest, this.self) + new MetricsTag.Values(dest, selfTag) ); this.outLatency = metricsCreator.createHistogramMetric( "remote_out_latency", "The latency(client side) of inside remote inside aggregate rpc.", new MetricsTag.Keys("dest", "self"), - new MetricsTag.Values(this.dest, this.self) + new MetricsTag.Values(dest, selfTag) ); - for (final String reason : ERROR_REASONS) { - final CounterMetrics counter = metricsCreator.createCounter( - "remote_out_error_count", - "The error number(client side) of inside remote inside aggregate rpc.", - new MetricsTag.Keys("dest", "self", "reason"), - new MetricsTag.Values(this.dest, this.self, reason) - ); - counter.inc(0); - errorCounters.put(reason, counter); - } + this.errorCounter = metricsCreator.createCounter( + "remote_out_error_count", + "The error number(client side) of inside remote inside aggregate rpc.", + new MetricsTag.Keys("dest", "self"), + new MetricsTag.Values(dest, selfTag) + ); + this.errorCounter.inc(0); if (trackStreamAndQueue) { this.streamInflight = metricsCreator.createGauge( "remote_out_stream_inflight", "In-flight gRPC streams for one remote client.", new MetricsTag.Keys("dest", "self"), - new MetricsTag.Values(this.dest, this.self) + new MetricsTag.Values(dest, selfTag) ); this.streamBlockSeconds = metricsCreator.createCounter( "remote_out_stream_block_seconds", "Seconds spent waiting for a free remote gRPC stream slot.", new MetricsTag.Keys("dest", "self"), - new MetricsTag.Values(this.dest, this.self) + new MetricsTag.Values(dest, selfTag) ); this.streamBlockSeconds.inc(0); this.queueUsedPercentage = metricsCreator.createGauge( "remote_out_queue_used_percentage", "Send queue used percentage for one remote client.", new MetricsTag.Keys("dest", "self"), - new MetricsTag.Values(this.dest, this.self) + new MetricsTag.Values(dest, selfTag) ); } else { this.streamInflight = null; @@ -134,14 +104,8 @@ public void observeLatencySeconds(final double seconds) { outLatency.observe(Math.max(0, seconds)); } - public void recordError(final String reason) { - final String outcome = reason == null || reason.isEmpty() ? REASON_ERROR : reason; - errorCounters.computeIfAbsent(outcome, ignored -> metricsCreator.createCounter( - "remote_out_error_count", - "The error number(client side) of inside remote inside aggregate rpc.", - new MetricsTag.Keys("dest", "self", "reason"), - new MetricsTag.Values(dest, self, outcome) - )).inc(); + public void recordError() { + errorCounter.inc(); } public void setStreamInflight(final int value) { @@ -166,32 +130,4 @@ public void clearPressureGauges() { setStreamInflight(0); setQueueUsedPercentage(0); } - - public static String reasonOf(final Status status) { - if (status == null) { - return REASON_ERROR; - } - switch (status.getCode()) { - case OK: - return REASON_ERROR; - case DEADLINE_EXCEEDED: - return REASON_TIMEOUT; - case UNAVAILABLE: - return REASON_IO_ERROR; - case RESOURCE_EXHAUSTED: - return REASON_RATE_LIMITED; - case UNAUTHENTICATED: - case PERMISSION_DENIED: - return REASON_AUTH_FAILED; - case UNKNOWN: - case INTERNAL: - case DATA_LOSS: - case UNIMPLEMENTED: - return REASON_SERVER_ERROR; - case CANCELLED: - return REASON_REJECTED; - default: - return REASON_REJECTED; - } - } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java index 39fd420b8821..66aad2292b71 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClient.java @@ -67,7 +67,7 @@ public void push(String nextWorkerName, StreamData streamData) { try { final RemoteHandleWorker handleWorker = workerInstanceGetter.get(nextWorkerName); if (handleWorker == null) { - metrics.recordError(RemoteClientMetrics.REASON_ERROR); + metrics.recordError(); log.error("Self remote push discarded: worker {} not found.", nextWorkerName); return; } @@ -75,7 +75,7 @@ public void push(String nextWorkerName, StreamData streamData) { metrics.incOut(); metrics.observeLatencySeconds((System.nanoTime() - startNanos) / 1_000_000_000d); } catch (Throwable t) { - metrics.recordError(RemoteClientMetrics.REASON_ERROR); + metrics.recordError(); log.error(t.getMessage(), t); } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java index 51ee46371ab4..3e2e0afcb3c5 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/GRPCRemoteClientTestCase.java @@ -285,7 +285,7 @@ public void onCompleted() { Assertions.assertTrue(outCount.get() >= 1); Assertions.assertEquals(0, latencyCount.get(), "failed streams must not pollute remote_out_latency"); - Assertions.assertEquals(List.of(RemoteClientMetrics.REASON_IO_ERROR), errorReasons); + Assertions.assertEquals(1, errorReasons.size()); Assertions.assertEquals(0.0, inflight.get(), 0.01); } finally { remoteClient.close(); @@ -340,7 +340,7 @@ public void onCompleted() { Assertions.assertEquals(0.0, inflight.get(), 0.01, "sync failure must release the stream slot"); Assertions.assertEquals(0, latencyCount.get()); - Assertions.assertEquals(List.of(RemoteClientMetrics.REASON_ERROR), errorReasons); + Assertions.assertEquals(1, errorReasons.size()); } finally { remoteClient.close(); } @@ -374,8 +374,8 @@ public void inc() { if ("remote_out_count".equals(name)) { outCount.incrementAndGet(); } - if ("remote_out_error_count".equals(name) && errorReasons != null && values.getValues().length >= 3) { - errorReasons.add(values.getValues()[2]); + if ("remote_out_error_count".equals(name) && errorReasons != null) { + errorReasons.add("error"); } } @@ -448,9 +448,7 @@ public void interruptWaitingForStreamSlotRecordsInterruptedOnly() throws Excepti public void inc() { if ("remote_out_error_count".equals(name)) { errorCount.incrementAndGet(); - if (values.getValues().length >= 3) { - errorReasons.add(values.getValues()[2]); - } + errorReasons.add("error"); } } @@ -534,7 +532,7 @@ public double getValue() { Assertions.assertInstanceOf(GRPCRemoteClient.RemoteStreamInterruptedException.class, failure.get()); Assertions.assertEquals(1, errorCount.get()); - Assertions.assertEquals(List.of(RemoteClientMetrics.REASON_INTERRUPTED), errorReasons); + Assertions.assertEquals(1, errorReasons.size()); Assertions.assertTrue(blockSeconds.get() >= 0d); } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java index 6425225cb198..938237fb7a7b 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/RemoteClientMetricsTest.java @@ -18,7 +18,6 @@ package org.apache.skywalking.oap.server.core.remote.client; -import io.grpc.Status; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicInteger; @@ -34,32 +33,19 @@ public class RemoteClientMetricsTest { @Test - public void reasonOfMapsGrpcStatus() { - Assertions.assertEquals(RemoteClientMetrics.REASON_TIMEOUT, RemoteClientMetrics.reasonOf(Status.DEADLINE_EXCEEDED)); - Assertions.assertEquals(RemoteClientMetrics.REASON_IO_ERROR, RemoteClientMetrics.reasonOf(Status.UNAVAILABLE)); - Assertions.assertEquals(RemoteClientMetrics.REASON_RATE_LIMITED, RemoteClientMetrics.reasonOf(Status.RESOURCE_EXHAUSTED)); - Assertions.assertEquals(RemoteClientMetrics.REASON_AUTH_FAILED, RemoteClientMetrics.reasonOf(Status.UNAUTHENTICATED)); - Assertions.assertEquals(RemoteClientMetrics.REASON_AUTH_FAILED, RemoteClientMetrics.reasonOf(Status.PERMISSION_DENIED)); - Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.INTERNAL)); - Assertions.assertEquals(RemoteClientMetrics.REASON_REJECTED, RemoteClientMetrics.reasonOf(Status.CANCELLED)); - Assertions.assertEquals(RemoteClientMetrics.REASON_ERROR, RemoteClientMetrics.reasonOf(null)); - } - - @Test - public void forGrpcRecordsLabeledErrorsAndLatency() { + public void forGrpcRecordsErrorsAndLatency() { final RecordingMetricsCreator creator = new RecordingMetricsCreator(); final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); metrics.incOut(); metrics.observeLatencySeconds(0.2); - metrics.recordError(RemoteClientMetrics.REASON_RATE_LIMITED); + metrics.recordError(); metrics.setStreamInflight(3); metrics.addStreamBlockSeconds(1.5); metrics.setQueueUsedPercentage(42); Assertions.assertEquals(1, creator.counterIncs.get("remote_out_count")); Assertions.assertEquals(1, creator.counterIncs.get("remote_out_error_count")); - Assertions.assertTrue(creator.lastErrorTags.get().contains("rate_limited")); Assertions.assertEquals(1, creator.latencyObservations.get()); Assertions.assertEquals(3.0, creator.gaugeValues.get("remote_out_stream_inflight"), 0.01); Assertions.assertEquals(1.5, creator.counterIncAmounts.get("remote_out_stream_block_seconds"), 0.01); @@ -74,6 +60,7 @@ public void forSelfSkipsStreamAndQueueSeries() { Assertions.assertFalse(creator.createdNames.contains("remote_out_queue_used_percentage")); Assertions.assertTrue(creator.createdNames.contains("remote_out_count")); Assertions.assertTrue(creator.createdNames.contains("remote_out_latency")); + Assertions.assertTrue(creator.createdNames.contains("remote_out_error_count")); } @Test @@ -101,7 +88,6 @@ public void queueUsedPercentageIsClamped() { public void zeroBlockSecondsAreIgnoredAfterSeed() { final RecordingMetricsCreator creator = new RecordingMetricsCreator(); final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); - // Construction seeds the Prometheus child with inc(0) so scrapes always see the series. Assertions.assertEquals(0.0, creator.counterIncAmounts.get("remote_out_stream_block_seconds"), 0.01); metrics.addStreamBlockSeconds(0); metrics.addStreamBlockSeconds(-1); @@ -109,39 +95,17 @@ public void zeroBlockSecondsAreIgnoredAfterSeed() { } @Test - public void errorReasonsAreSeededAtConstruction() { + public void errorCounterIsSeededAtConstruction() { final RecordingMetricsCreator creator = new RecordingMetricsCreator(); RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); final long errorSeries = creator.createdNames.stream().filter("remote_out_error_count"::equals).count(); - Assertions.assertEquals(8, errorSeries, "every stable reason label must be pre-registered"); + Assertions.assertEquals(1, errorSeries, "one dest/self error series must be pre-registered"); Assertions.assertEquals(0.0, creator.counterIncAmounts.getOrDefault("remote_out_error_count", -1.0), 0.01); - } - - @Test - public void reasonOfCoversAdditionalServerFailures() { - Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.UNKNOWN)); - Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.DATA_LOSS)); - Assertions.assertEquals(RemoteClientMetrics.REASON_SERVER_ERROR, RemoteClientMetrics.reasonOf(Status.UNIMPLEMENTED)); - Assertions.assertEquals(RemoteClientMetrics.REASON_REJECTED, RemoteClientMetrics.reasonOf(Status.INVALID_ARGUMENT)); - } - - @Test - public void recordErrorNullFallsBackToError() { - final RecordingMetricsCreator creator = new RecordingMetricsCreator(); - final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); - metrics.recordError(null); - metrics.recordError(""); - Assertions.assertEquals(2, creator.counterIncs.get("remote_out_error_count")); - Assertions.assertTrue(creator.lastErrorTags.get().endsWith("error")); - } - - @Test - public void interruptedReasonIsRecordedDirectly() { - final RecordingMetricsCreator creator = new RecordingMetricsCreator(); - final RemoteClientMetrics metrics = RemoteClientMetrics.forGrpc(creator, new Address("oap2", 11800, false)); - metrics.recordError(RemoteClientMetrics.REASON_INTERRUPTED); - Assertions.assertEquals(1, creator.counterIncs.get("remote_out_error_count")); - Assertions.assertTrue(creator.lastErrorTags.get().contains(RemoteClientMetrics.REASON_INTERRUPTED)); + Assertions.assertTrue(creator.createdTagSets.stream().anyMatch(t -> t.equals("oap2_11800,N"))); + Assertions.assertTrue( + creator.createdTagKeys.stream().noneMatch(k -> k.contains("reason")), + "remote_out_error_count must not carry a reason label" + ); } @Test @@ -175,25 +139,22 @@ public void forSelfAndForGrpcUseDistinctSelfTags() { private static final class RecordingMetricsCreator implements MetricsCreator { private final List createdNames = new ArrayList<>(); private final List createdTagSets = new ArrayList<>(); + private final List createdTagKeys = new ArrayList<>(); private final java.util.Map counterIncs = new java.util.HashMap<>(); private final java.util.Map counterIncAmounts = new java.util.HashMap<>(); private final java.util.Map gaugeValues = new java.util.HashMap<>(); private final AtomicInteger latencyObservations = new AtomicInteger(); - private final AtomicReference lastErrorTags = new AtomicReference<>(""); private final AtomicReference lastLatencyValue = new AtomicReference<>(); @Override public CounterMetrics createCounter(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) { createdNames.add(name); - final String tagJoined = String.join(",", tagValues.getValues()); - createdTagSets.add(tagJoined); + createdTagKeys.add(String.join(",", tagKeys.getKeys())); + createdTagSets.add(String.join(",", tagValues.getValues())); return new CounterMetrics() { @Override public void inc() { counterIncs.merge(name, 1, Integer::sum); - if ("remote_out_error_count".equals(name)) { - lastErrorTags.set(tagJoined); - } } @Override @@ -206,6 +167,7 @@ public void inc(double value) { @Override public GaugeMetrics createGauge(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues) { createdNames.add(name); + createdTagKeys.add(String.join(",", tagKeys.getKeys())); createdTagSets.add(String.join(",", tagValues.getValues())); return new GaugeMetrics() { @Override @@ -240,6 +202,7 @@ public double getValue() { public HistogramMetrics createHistogramMetric(String name, String tips, MetricsTag.Keys tagKeys, MetricsTag.Values tagValues, double... buckets) { createdNames.add(name); + createdTagKeys.add(String.join(",", tagKeys.getKeys())); createdTagSets.add(String.join(",", tagValues.getValues())); return new HistogramMetrics() { @Override diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java index 40abb10ff7cd..351fb6195434 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/remote/client/SelfRemoteClientTest.java @@ -56,7 +56,6 @@ public class SelfRemoteClientTest { private final Map counterIncs = new HashMap<>(); private final AtomicInteger latencyObservations = new AtomicInteger(); private final AtomicReference lastLatencySeconds = new AtomicReference<>(); - private final Map lastErrorReason = new HashMap<>(); private final Map counterTags = new HashMap<>(); private String[] latencyTags; @@ -78,9 +77,6 @@ public void setUp() { @Override public void inc() { counterIncs.computeIfAbsent(name, ignored -> new AtomicInteger()).incrementAndGet(); - if ("remote_out_error_count".equals(name) && values.getValues().length >= 3) { - lastErrorReason.put(name, values.getValues()[2]); - } } @Override @@ -137,23 +133,21 @@ public void repeatedSuccessfulPushesEachObserveLatency() { } @Test - public void pushFailureRecordsErrorReason() { + public void pushFailureRecordsError() { workers.put(WORKER, new FailingWorker(moduleManager), MetricStreamKind.OAL, TestStreamData.class); SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); client.push(WORKER, new TestStreamData()); Assertions.assertEquals(1, counterIncs.get("remote_out_error_count").get()); - Assertions.assertEquals(RemoteClientMetrics.REASON_ERROR, lastErrorReason.get("remote_out_error_count")); Assertions.assertEquals(0, latencyObservations.get()); } @Test - public void pushMissingWorkerRecordsErrorReason() { + public void pushMissingWorkerRecordsError() { SelfRemoteClient client = new SelfRemoteClient(moduleManager, new Address("oap1", 11800, true)); client.push("missing-worker", new TestStreamData()); Assertions.assertEquals(1, counterIncs.get("remote_out_error_count").get()); - Assertions.assertEquals(RemoteClientMetrics.REASON_ERROR, lastErrorReason.get("remote_out_error_count")); Assertions.assertEquals(0, latencyObservations.get()); Assertions.assertFalse(counterIncs.containsKey("remote_out_count")); } diff --git a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml index ba5a43c0a1b9..fbbc956063c7 100644 --- a/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml +++ b/oap-server/server-starter/src/main/resources/otel-rules/oap.yaml @@ -155,7 +155,7 @@ metricsRules: - name: instance_remote_out_count exp: remote_out_count.sum(['service', 'host_name', 'dest', 'self']).increase('PT1M') - name: instance_remote_out_error_count - exp: remote_out_error_count.sum(['service', 'host_name', 'dest', 'self', 'reason']).increase('PT1M') + exp: remote_out_error_count.sum(['service', 'host_name', 'dest', 'self']).increase('PT1M') - name: instance_remote_out_latency_percentile exp: remote_out_latency.sum(['le', 'service', 'host_name', 'dest', 'self']).increase('PT1M').histogram().histogram_percentile([50,75,90,95,99]) - name: instance_remote_out_stream_inflight