diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index f58f1b5f00f1..3d217945e071 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -66,6 +66,7 @@ - The table check at boot logged wrong table and column states. * Fix creating an async-profiler task for a service while another one runs on it: the check that refuses it never found the running task on the JDBC and Elasticsearch storage. * Fix the async-profiler task OAP sends to an agent whose service has two tasks created within five minutes of each other: it could send the older one. +* Fix alarm windows retaining empty layers after an initial metadata lookup miss. Later metrics can supply the missing layers for subsequent alarms, allowing layer-filtered `queryAlarms` queries to match the persisted layer. #### UI diff --git a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java index a3522f80dc30..d4d116c9d1d2 100644 --- a/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java +++ b/oap-server/server-alarm-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java @@ -152,6 +152,7 @@ public void in(MetaInAlarm meta, Metrics metrics) { Window window = windows.computeIfAbsent(entity, ignored -> new Window(entity, this.period, this.silencePeriod, this.recoveryObservationPeriod, this.additionalPeriod)); + window.updateLayers(entity); window.add(meta.getMetricsName(), metrics); } @@ -285,7 +286,8 @@ public class Window { private AlarmMessage lastAlarmMessage; @Getter private JsonObject mqeMetricsSnapshot; - private AlarmEntity entity; + // Read by the alarm check thread, replaced by updateLayers on the metrics thread. + private volatile AlarmEntity entity; public Window(AlarmEntity entity, int period, int silencePeriod, int recoveryObservationPeriod, int additionalPeriod) { @@ -297,6 +299,20 @@ public Window(AlarmEntity entity, int period, int silencePeriod, int recoveryObs this.init(); } + /** + * Fill missing layers from a later metric's entity. An initial metadata lookup in {@code NotifyHandler} can + * miss a service whose traffic metadata is not yet queryable or visible in the metadata cache. + * {@link AlarmEntity} equality excludes layers, so otherwise the window retains the initially unresolved + * layers. + * + * @param latest the entity of the metric being added + */ + void updateLayers(final AlarmEntity latest) { + if (CollectionUtils.isEmpty(entity.getLayers()) && CollectionUtils.isNotEmpty(latest.getLayers())) { + entity = latest; + } + } + public void moveTo(LocalDateTime current) { lock.lock(); try { diff --git a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java index 0b7c3b8f090f..69f0d4fb8e1b 100644 --- a/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java +++ b/oap-server/server-alarm-plugin/src/test/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRuleTest.java @@ -44,6 +44,7 @@ import org.junit.jupiter.api.Test; import org.apache.skywalking.oap.server.testing.util.ReflectUtil; +import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -131,6 +132,32 @@ public void testAlarm() throws IllegalExpressionException { Assertions.assertEquals(1, alarmMessages.size()); } + @Test + public void testAlarmTakesTheLayersResolvedAfterTheFirstMetric() throws IllegalExpressionException { + AlarmRule alarmRule = new AlarmRule(null); + alarmRule.setAlarmRuleName("endpoint_percent_rule"); + alarmRule.setExpression("sum(endpoint_percent < 75) >= 3"); + alarmRule.getIncludeMetrics().add("endpoint_percent"); + alarmRule.setPeriod(15); + alarmRule.setMessage("Successful rate of endpoint {name} is lower than 75%"); + alarmRule.setTags(new HashMap<>()); + RunningRule runningRule = new RunningRule(alarmRule, null); + + DateTime startTime = DateTime.now(); + // Simulate an initial metadata lookup miss followed by successful layer resolution for later metrics of the + // same entity. + runningRule.in(getMetaInAlarm(123, "endpoint_percent", Collections.emptyList()), + getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(6).getMillis()), 70)); + runningRule.in(getMetaInAlarm(123, "endpoint_percent", List.of("GENERAL")), + getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(4).getMillis()), 71)); + runningRule.in(getMetaInAlarm(123, "endpoint_percent", List.of("GENERAL")), + getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(2).getMillis()), 74)); + + List alarmMessages = getAlarmFiringMessageList(runningRule.check()); + Assertions.assertEquals(1, alarmMessages.size()); + Assertions.assertEquals(List.of("GENERAL"), alarmMessages.get(0).getLayers()); + } + @Test public void testAlarmMetricsOutOfDate() throws IllegalExpressionException { AlarmRule alarmRule = new AlarmRule(null); @@ -476,7 +503,16 @@ private MetaInAlarm getMetaInAlarm(int id) { } private MetaInAlarm getMetaInAlarm(int id, String metricName) { + return getMetaInAlarm(id, metricName, Collections.emptyList()); + } + + private MetaInAlarm getMetaInAlarm(int id, String metricName, List layers) { return new MetaInAlarm() { + @Override + public List getLayers() { + return layers; + } + @Override public String getScope() { return "SERVICE";