Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/en/changes/changes.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
- The table check at boot logged wrong table and column states.
* Fix creating an async-profiler task for a service while another one runs on it: the check that refuses it never found the running task on the JDBC and Elasticsearch storage.
* Fix the async-profiler task OAP sends to an agent whose service has two tasks created within five minutes of each other: it could send the older one.
* Fix alarm windows retaining empty layers after an initial metadata lookup miss. Later metrics can supply the missing layers for subsequent alarms, allowing layer-filtered `queryAlarms` queries to match the persisted layer.

#### UI

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ public void in(MetaInAlarm meta, Metrics metrics) {

Window window = windows.computeIfAbsent(entity, ignored -> new Window(entity, this.period,
this.silencePeriod, this.recoveryObservationPeriod, this.additionalPeriod));
window.updateLayers(entity);
window.add(meta.getMetricsName(), metrics);
}

Expand Down Expand Up @@ -285,7 +286,8 @@ public class Window {
private AlarmMessage lastAlarmMessage;
@Getter
private JsonObject mqeMetricsSnapshot;
private AlarmEntity entity;
// Read by the alarm check thread, replaced by updateLayers on the metrics thread.
private volatile AlarmEntity entity;

public Window(AlarmEntity entity, int period, int silencePeriod, int recoveryObservationPeriod,
int additionalPeriod) {
Expand All @@ -297,6 +299,20 @@ public Window(AlarmEntity entity, int period, int silencePeriod, int recoveryObs
this.init();
}

/**
* Fill missing layers from a later metric's entity. An initial metadata lookup in {@code NotifyHandler} can
* miss a service whose traffic metadata is not yet queryable or visible in the metadata cache.
* {@link AlarmEntity} equality excludes layers, so otherwise the window retains the initially unresolved
* layers.
*
* @param latest the entity of the metric being added
*/
void updateLayers(final AlarmEntity latest) {
if (CollectionUtils.isEmpty(entity.getLayers()) && CollectionUtils.isNotEmpty(latest.getLayers())) {
entity = latest;
}
}

public void moveTo(LocalDateTime current) {
lock.lock();
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import org.junit.jupiter.api.Test;
import org.apache.skywalking.oap.server.testing.util.ReflectUtil;

import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
Expand Down Expand Up @@ -131,6 +132,32 @@ public void testAlarm() throws IllegalExpressionException {
Assertions.assertEquals(1, alarmMessages.size());
}

@Test
public void testAlarmTakesTheLayersResolvedAfterTheFirstMetric() throws IllegalExpressionException {
AlarmRule alarmRule = new AlarmRule(null);
alarmRule.setAlarmRuleName("endpoint_percent_rule");
alarmRule.setExpression("sum(endpoint_percent < 75) >= 3");
alarmRule.getIncludeMetrics().add("endpoint_percent");
alarmRule.setPeriod(15);
alarmRule.setMessage("Successful rate of endpoint {name} is lower than 75%");
alarmRule.setTags(new HashMap<>());
RunningRule runningRule = new RunningRule(alarmRule, null);

DateTime startTime = DateTime.now();
// Simulate an initial metadata lookup miss followed by successful layer resolution for later metrics of the
// same entity.
runningRule.in(getMetaInAlarm(123, "endpoint_percent", Collections.emptyList()),
getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(6).getMillis()), 70));
runningRule.in(getMetaInAlarm(123, "endpoint_percent", List.of("GENERAL")),
getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(4).getMillis()), 71));
runningRule.in(getMetaInAlarm(123, "endpoint_percent", List.of("GENERAL")),
getMetrics(TimeBucket.getMinuteTimeBucket(startTime.minusMinutes(2).getMillis()), 74));

List<AlarmMessage> alarmMessages = getAlarmFiringMessageList(runningRule.check());
Assertions.assertEquals(1, alarmMessages.size());
Assertions.assertEquals(List.of("GENERAL"), alarmMessages.get(0).getLayers());
}

@Test
public void testAlarmMetricsOutOfDate() throws IllegalExpressionException {
AlarmRule alarmRule = new AlarmRule(null);
Expand Down Expand Up @@ -476,7 +503,16 @@ private MetaInAlarm getMetaInAlarm(int id) {
}

private MetaInAlarm getMetaInAlarm(int id, String metricName) {
return getMetaInAlarm(id, metricName, Collections.emptyList());
}

private MetaInAlarm getMetaInAlarm(int id, String metricName, List<String> layers) {
return new MetaInAlarm() {
@Override
public List<String> getLayers() {
return layers;
}

@Override
public String getScope() {
return "SERVICE";
Expand Down
Loading