From c1a6ee06ef481552535411be2c19254d7bccc23d Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Tue, 17 Nov 2026 19:26:19 +0800 Subject: [PATCH] Fix alarm windows retaining initially unresolved layers NotifyHandler resolves layers through MetadataQueryService for incoming metric notifications. Metrics reach alarm rules during persistence preparation, so a new service's first notification can arrive before its traffic metadata is queryable or visible in the metadata cache. RunningRule reuses windows keyed by AlarmEntity, whose equality excludes layers. When the first lookup returned no layers, the window retained that entity even after later lookups succeeded. Subsequent firing messages therefore kept an empty layer list, persisted as UNDEFINED, and did not match queries filtering for the service's actual layer. The window now fills missing layers from a later notification with resolved layers. The replacement entity is volatile because the alarm check runs on another thread. This affects subsequently built firing messages; it does not backfill existing records or refresh non-empty layer lists. --- docs/en/changes/changes.md | 1 + .../core/alarm/provider/RunningRule.java | 18 +++++++++- .../core/alarm/provider/RunningRuleTest.java | 36 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index f58f1b5f00f1..3d217945e071 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -66,6 +66,7 @@ - The table check at boot logged wrong table and column states. * Fix creating an async-profiler task for a service while another one runs on it: the check that refuses it never found the running task on the JDBC and Elasticsearch storage. * Fix the async-profiler task OAP sends to an agent whose service has two tasks created within five minutes of each other: it could send the older one. +* Fix alarm windows retaining empty layers after an initial metadata lookup miss. Later metrics can supply the missing layers for subsequent alarms, allowing layer-filtered `queryAlarms` queries to match the persisted layer. #### UI diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java index a3522f80dc30..d4d116c9d1d2 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java @@ -152,6 +152,7 @@ public void in(MetaInAlarm meta, Metrics metrics) { Window window = windows.computeIfAbsent(entity, ignored -> new Window(entity, this.period, this.silencePeriod, this.recoveryObservationPeriod, this.additionalPeriod)); + window.updateLayers(entity); window.add(meta.getMetricsName(), metrics); } @@ -285,7 +286,8 @@ public class Window { private AlarmMessage lastAlarmMessage; @Getter private JsonObject mqeMetricsSnapshot; - private AlarmEntity entity; + // Read by the alarm check thread, replaced by updateLayers on the metrics thread. + private volatile AlarmEntity entity; public Window(AlarmEntity entity, int period, int silencePeriod, int recoveryObservationPeriod, int additionalPeriod) { @@ -297,6 +299,20 @@ public Window(AlarmEntity entity, int period, int silencePeriod, int recoveryObs this.init(); } + /** + * Fill missing layers from a later metric's entity. An initial metadata lookup in {@code NotifyHandler} can + * miss a service whose traffic metadata is not yet queryable or visible in the metadata cache. + * {@link AlarmEntity} equality excludes layers, so otherwise the window retains the initially unresolved + * layers. + * + * @param latest the entity of the metric being added + */ + void updateLayers(final AlarmEntity latest) { + if (CollectionUtils.isEmpty(entity.getLayers()) && CollectionUtils.isNotEmpty(latest.getLayers())) { + entity = latest; + } + } + public void moveTo(LocalDateTime current) { lock.lock(); try { diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java index 0b7c3b8f090f..69f0d4fb8e1b 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java @@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test; import org.apache.skywalking.oap.server.testing.util.ReflectUtil; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -131,6 +132,32 @@ public void testAlarm() throws IllegalExpressionException { Assertions.assertEquals(1, alarmMessages.size()); } + @Test + public void testAlarmTakesTheLayersResolvedAfterTheFirstMetric() throws IllegalExpressionException { + AlarmRule alarmRule = new AlarmRule(null); + alarmRule.setAlarmRuleName("endpoint_percent_rule"); + alarmRule.setExpression("sum(endpoint_percent < 75) >= 3"); + alarmRule.getIncludeMetrics().add("endpoint_percent"); + alarmRule.setPeriod(15); + alarmRule.setMessage("Successful rate of endpoint {name} is lower than 75%"); + alarmRule.setTags(new HashMap<>()); + RunningRule runningRule = new RunningRule(alarmRule, null); + + DateTime startTime = DateTime.now(); + // Simulate an initial metadata lookup miss followed by successful layer resolution for later metrics of the + // same entity. + runningRule.in(getMetaInAlarm(123, "endpoint_percent", Collections.emptyList()), + getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(6).getMillis()), 70)); + runningRule.in(getMetaInAlarm(123, "endpoint_percent", List.of("GENERAL")), + getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(4).getMillis()), 71)); + runningRule.in(getMetaInAlarm(123, "endpoint_percent", List.of("GENERAL")), + getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(2).getMillis()), 74)); + + List alarmMessages = getAlarmFiringMessageList(runningRule.check()); + Assertions.assertEquals(1, alarmMessages.size()); + Assertions.assertEquals(List.of("GENERAL"), alarmMessages.get(0).getLayers()); + } + @Test public void testAlarmMetricsOutOfDate() throws IllegalExpressionException { AlarmRule alarmRule = new AlarmRule(null); @@ -476,7 +503,16 @@ private MetaInAlarm getMetaInAlarm(int id) { } private MetaInAlarm getMetaInAlarm(int id, String metricName) { + return getMetaInAlarm(id, metricName, Collections.emptyList()); + } + + private MetaInAlarm getMetaInAlarm(int id, String metricName, List layers) { return new MetaInAlarm() { + @Override + public List getLayers() { + return layers; + } + @Override public String getScope() { return "SERVICE";