2 Commits
master ... main

4 changed files with 155 additions and 344 deletions

View File

@@ -1,40 +0,0 @@
kind: pipeline
type: docker
name: test-on-amd64
platform:
arch: amd64
steps:
- name: build-release
image: rust:1.89
commands:
- cargo build --verbose --workspace --release
- name: test
image: rust:1.89
commands:
- cargo test --verbose --workspace
- name: lint
image: rust:1.89
commands:
- rustup component add clippy
- cargo clippy --all-targets --all-features
- name: push-binary-release
image: plugins/gitea-release
settings:
base_url: https://gitea.lardenois.cc
api_key:
from_secret: package-drone
files:
- target/release/node-notifier
title: "Release ${DRONE_TAG:-v${DRONE_BUILD_NUMBER}}"
note: "Automated release from commit ${DRONE_COMMIT_SHA}"
checksum:
- sha256
when:
event: tag
status: success
depends_on:
- build-release
- test
- lint

View File

@@ -53,4 +53,8 @@ impl ForFormattingMessage for AlertFormatter {
result result
} }
fn format_resolution(&self, metric_name: &str) -> String {
format!("{} has been resolved", metric_name)
}
} }

View File

@@ -18,6 +18,7 @@ pub trait ForMonitoringSystem {
pub trait ForFormattingMessage { pub trait ForFormattingMessage {
fn format_violation(&self, violation: Box<dyn ForGettingViolationData>) -> String; fn format_violation(&self, violation: Box<dyn ForGettingViolationData>) -> String;
fn format_summary(&self, metrics: &HashMap<String, f32>) -> String; fn format_summary(&self, metrics: &HashMap<String, f32>) -> String;
fn format_resolution(&self, metric_name: &str) -> String;
} }
#[cfg_attr(test, automock)] #[cfg_attr(test, automock)]

View File

@@ -4,28 +4,11 @@ use crate::app::ports::driven::{
ForFormattingMessage, ForMonitoringSystem, ForSendingNotification, ForFormattingMessage, ForMonitoringSystem, ForSendingNotification,
}; };
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum AlertLevel {
Resolved = 0, // No violation
Warning = 1,
Critical = 2,
}
impl AlertLevel {
fn from_is_critical(is_critical: bool) -> Self {
if is_critical {
AlertLevel::Critical
} else {
AlertLevel::Warning
}
}
}
pub struct AlertService { pub struct AlertService {
notifier: Box<dyn ForSendingNotification>, notifier: Box<dyn ForSendingNotification>,
monitor: Box<dyn ForMonitoringSystem>, monitor: Box<dyn ForMonitoringSystem>,
formatter: Box<dyn ForFormattingMessage>, formatter: Box<dyn ForFormattingMessage>,
highest_alert_levels: HashMap<String, AlertLevel>, map_alerts_criticity: HashMap<String, bool>,
} }
pub fn new( pub fn new(
@@ -37,7 +20,7 @@ pub fn new(
notifier, notifier,
monitor, monitor,
formatter, formatter,
highest_alert_levels: HashMap::new(), map_alerts_criticity: HashMap::new(),
} }
} }
@@ -45,328 +28,191 @@ impl AlertService {
/// Check for threshold violations and send alerts on level changes (escalation, improvement, resolution) /// Check for threshold violations and send alerts on level changes (escalation, improvement, resolution)
pub fn alert_on_threshold_violation(&mut self) -> Result<(), Box<dyn std::error::Error>> { pub fn alert_on_threshold_violation(&mut self) -> Result<(), Box<dyn std::error::Error>> {
let violations = self.monitor.check_thresholds(); let violations = self.monitor.check_thresholds();
let current_levels = self.violations_to_map(&violations);
// Build a map of current violation levels
let mut current_levels: HashMap<String, AlertLevel> = HashMap::new(); self.process_resolved_violations(&current_levels);
for violation in &violations { self.process_current_violations(violations)?;
let metric_name = violation.get_metric_name();
let level = AlertLevel::from_is_critical(violation.is_critical()); self.update_map_alerts_criticity(current_levels);
current_levels.insert(metric_name, level);
} Ok(())
}
// Check for metrics that had violations but now don't (resolution)
for (metric_name, &previous_level) in &self.highest_alert_levels { fn violations_to_map(
&self,
violations: &[Box<dyn crate::app::ports::driven::ForGettingViolationData>],
) -> HashMap<String, bool> {
violations
.iter()
.map(|v| (v.get_metric_name(), v.is_critical()))
.collect()
}
fn process_resolved_violations(&mut self, current_levels: &HashMap<String, bool>) {
for (metric_name, &_previous_level) in &self.map_alerts_criticity {
if !current_levels.contains_key(metric_name) { if !current_levels.contains_key(metric_name) {
// This metric is now resolved let message = self.formatter.format_resolution(metric_name);
let message = format!("{} has been resolved (was {:?})", metric_name, previous_level); self.notifier.send_notification(&message).unwrap();
self.notifier.send_notification(&message)?;
} }
} }
}
// Process current violations
fn process_current_violations(
&mut self,
violations: Vec<Box<dyn crate::app::ports::driven::ForGettingViolationData>>,
) -> Result<(), Box<dyn std::error::Error>> {
for violation in violations { for violation in violations {
let metric_name = violation.get_metric_name(); let metric_name = violation.get_metric_name();
let current_level = AlertLevel::from_is_critical(violation.is_critical()); let current_level = violation.is_critical();
let should_alert = match self.highest_alert_levels.get(&metric_name) { let should_alert = match self.map_alerts_criticity.get(&metric_name) {
None => true, // First time seeing this metric None => true, // First time seeing this metric
Some(&previous_level) => current_level != previous_level, // Any level change Some(&previous_level) => current_level != previous_level, // Any level change
}; };
if should_alert { if should_alert {
let message = self.formatter.format_violation(violation); let message = self.formatter.format_violation(violation);
self.notifier.send_notification(&message)?; self.notifier.send_notification(&message)?;
} }
// Update the level for this metric // Update the level for this metric
self.highest_alert_levels.insert(metric_name, current_level); self.map_alerts_criticity.insert(metric_name, current_level);
} }
// Remove resolved metrics from our tracking
self.highest_alert_levels.retain(|metric_name, _| current_levels.contains_key(metric_name));
Ok(()) Ok(())
} }
}
fn update_map_alerts_criticity(&mut self, current_levels: HashMap<String, bool>) {
self.map_alerts_criticity = current_levels;
}
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::collections::HashMap;
use rstest::rstest; use rstest::rstest;
use std::collections::HashMap;
use std::collections::HashSet;
use crate::app::ports::driven::{MockForFormattingMessage, MockForMonitoringSystem, MockForSendingNotification}; use crate::app::{
ports::driven::{
ForGettingViolationData, MockForFormattingMessage, MockForGettingViolationData,
MockForMonitoringSystem, MockForSendingNotification,
},
services::alert_service,
};
use super::*; const TEST_VALUE: f32 = 75.0;
const TEST_THRESHOLD: f32 = 70.0;
#[rstest] #[rstest]
#[case(vec![], 0, true, "no violations")] // No violations
#[case(vec![true], 1, true, "single critical violation")] #[case(HashMap::new(), HashMap::new(), vec![])]
#[case(vec![false], 1, true, "single warning violation")] // Single critical violation
#[case(HashMap::from([("CPU".to_string(), true)]), HashMap::new(), vec!["critical CPU".to_string(), "resolved CPU".to_string()])]
// Single warning violation
#[case(HashMap::from([("Memory".to_string(), false)]), HashMap::new(), vec!["warning Memory".to_string(), "resolved Memory".to_string()])]
// Multiple violations with level changes and one resolution
#[case(HashMap::from([("CPU".to_string(), true), ("Memory".to_string(), false), ("Disk".to_string(), false)]),
HashMap::from([("CPU".to_string(), false), ("Disk".to_string(), true)]),
vec![
"critical CPU".to_string(),
"warning Memory".to_string(),
"warning Disk".to_string(),
"resolved Memory".to_string(),
"warning CPU".to_string(),
"critical Disk".to_string()
]
)]
// No level changes
#[case(
HashMap::from([("CPU".to_string(), true), ("Memory".to_string(), false)]),
HashMap::from([("CPU".to_string(), true), ("Memory".to_string(), false)]),
vec!["critical CPU".to_string(), "warning Memory".to_string()]
)]
fn test_alert_on_threshold_violation_scenarios( fn test_alert_on_threshold_violation_scenarios(
#[case] violations_criticality: Vec<bool>, #[case] first_violations: HashMap<String, bool>,
#[case] expected_notifications: usize, #[case] second_violations: HashMap<String, bool>,
#[case] should_succeed: bool, #[case] expected_notifications: Vec<String>,
#[case] _description: &str,
) { ) {
let mut notifier = MockForSendingNotification::new(); let mut mock_monitor = MockForMonitoringSystem::new();
let mut monitor = MockForMonitoringSystem::new(); let mut mock_formatter = MockForFormattingMessage::new();
let mut formatter = MockForFormattingMessage::new(); let mut mock_notifier = MockForSendingNotification::new();
// Setup monitor expectations // Helper to build violations from a HashMap
monitor.expect_check_thresholds().returning(move || { fn build_violations(map: &HashMap<String, bool>) -> Vec<Box<dyn ForGettingViolationData>> {
violations_criticality.iter().enumerate().map(|(i, &is_critical)| { map.iter()
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new(); .map(|(name, &is_critical)| {
violation.expect_get_metric_name().return_const(format!("metric_{}", i)); let mut violation = MockForGettingViolationData::new();
violation.expect_is_critical().return_const(is_critical); let name = name.clone();
Box::new(violation) as Box<dyn crate::app::ports::driven::ForGettingViolationData> violation.expect_is_critical().return_const(is_critical);
}).collect() violation
}); .expect_get_metric_name()
.return_const(name.clone());
monitor.expect_get_metrics().returning(|| HashMap::new()); violation.expect_get_metric_value().return_const(TEST_VALUE);
violation
if expected_notifications > 0 { .expect_get_threshold()
formatter.expect_format_violation() .return_const(TEST_THRESHOLD);
.times(expected_notifications) Box::new(violation) as Box<dyn ForGettingViolationData>
.returning(|_| "Violation message".to_string()); })
.collect()
formatter.expect_format_summary().returning(|_| "Summary".to_string());
notifier.expect_send_notification()
.times(expected_notifications)
.returning(move |_| {
if should_succeed {
Ok(())
} else {
Err("Network error".into())
}
});
} else {
formatter.expect_format_summary().returning(|_| "Summary".to_string());
} }
let mut service = new( // The test will call check_thresholds twice: first and second run
Box::new(notifier), let mut call_count = 0;
Box::new(monitor), let first_violations_clone = first_violations.clone();
Box::new(formatter), let second_violations_clone = second_violations.clone();
); mock_monitor.expect_check_thresholds().returning(move || {
call_count += 1;
let result = service.alert_on_threshold_violation(); if call_count == 1 {
if should_succeed { build_violations(&first_violations_clone)
assert!(result.is_ok()); } else {
} else { build_violations(&second_violations_clone)
assert!(result.is_err()); }
}
}
#[test]
fn test_alert_on_threshold_violation_handles_send_error() {
let mut notifier = MockForSendingNotification::new();
let mut monitor = MockForMonitoringSystem::new();
let mut formatter = MockForFormattingMessage::new();
monitor.expect_check_thresholds().returning(|| {
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new();
violation.expect_get_metric_name().return_const("cpu_usage".to_string());
violation.expect_is_critical().return_const(true);
vec![Box::new(violation)]
});
formatter.expect_format_violation().returning(|_| "Test violation".to_string());
notifier.expect_send_notification()
.returning(|_| Err("Network error".into()));
let mut service = new(
Box::new(notifier),
Box::new(monitor),
Box::new(formatter),
);
assert!(service.alert_on_threshold_violation().is_err());
}
#[test]
fn test_multiple_different_metrics() {
let mut notifier = MockForSendingNotification::new();
let mut monitor = MockForMonitoringSystem::new();
let mut formatter = MockForFormattingMessage::new();
// Different metrics should all be sent
monitor.expect_check_thresholds().returning(|| {
vec![
{
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new();
violation.expect_get_metric_name().return_const("cpu_usage".to_string());
violation.expect_is_critical().return_const(true);
Box::new(violation) as Box<dyn crate::app::ports::driven::ForGettingViolationData>
},
{
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new();
violation.expect_get_metric_name().return_const("memory_usage".to_string());
violation.expect_is_critical().return_const(false);
Box::new(violation) as Box<dyn crate::app::ports::driven::ForGettingViolationData>
}
]
}); });
formatter.expect_format_violation() // Formatter returns a string based on the violation
.times(2) // Both should be sent mock_formatter
.returning(|_| "Violation message".to_string()); .expect_format_violation()
.returning(|violation| {
notifier.expect_send_notification() let level = if violation.is_critical() {
.times(2) // Both should be sent "critical"
.returning(|_| Ok(())); } else {
"warning"
};
format!("{} {}", level, violation.get_metric_name())
});
mock_formatter
.expect_format_resolution()
.returning(|metric_name| format!("resolved {}", metric_name));
let mut service = new( // Collect notifications sent
Box::new(notifier), let notifications = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
Box::new(monitor), let notifications_clone = notifications.clone();
Box::new(formatter), mock_notifier
); .expect_send_notification()
.returning(move |message| {
assert!(service.alert_on_threshold_violation().is_ok()); notifications_clone
} .lock()
.unwrap()
#[test] .push(message.to_string());
fn test_alert_resolution() { Ok(())
let mut notifier = MockForSendingNotification::new();
let mut monitor = MockForMonitoringSystem::new();
let mut formatter = MockForFormattingMessage::new();
// First call: critical violation
// Second call: no violations (resolution)
monitor.expect_check_thresholds()
.times(2)
.returning_st({
let mut call_count = 0;
move || {
call_count += 1;
if call_count == 1 {
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new();
violation.expect_get_metric_name().return_const("cpu_usage".to_string());
violation.expect_is_critical().return_const(true);
vec![Box::new(violation)]
} else {
vec![] // No violations - resolved
}
}
}); });
formatter.expect_format_violation() let mut alert_service = alert_service::new(
.times(1) // Only for the initial violation Box::new(mock_notifier),
.returning(|_| "CPU usage is critical".to_string()); Box::new(mock_monitor),
Box::new(mock_formatter),
// Two notifications: initial critical + resolution
notifier.expect_send_notification()
.times(2)
.withf(|msg| {
msg.contains("CPU usage is critical") || msg.contains("cpu_usage has been resolved")
})
.returning(|_| Ok(()));
let mut service = new(
Box::new(notifier),
Box::new(monitor),
Box::new(formatter),
); );
// First call: critical should be sent // First run
assert!(service.alert_on_threshold_violation().is_ok()); alert_service.alert_on_threshold_violation().unwrap();
// Second run
// Second call: resolution should be sent alert_service.alert_on_threshold_violation().unwrap();
assert!(service.alert_on_threshold_violation().is_ok());
}
#[rstest] // Hashmap is not ordered, so we need to compare sets
#[case(true, true, 1, "same critical level twice")] let sent = notifications.lock().unwrap();
#[case(false, false, 1, "same warning level twice")] let sent_set: HashSet<_> = sent.iter().collect();
fn test_same_level_alerts_are_not_sent_twice( let expected_set: HashSet<_> = expected_notifications.iter().collect();
#[case] is_critical: bool, assert_eq!(sent_set, expected_set);
#[case] _second_is_critical: bool, // Same as first for these test cases
#[case] expected_notifications: usize,
#[case] _description: &str,
) {
let mut notifier = MockForSendingNotification::new();
let mut monitor = MockForMonitoringSystem::new();
let mut formatter = MockForFormattingMessage::new();
// Monitor should return the same violation twice
monitor.expect_check_thresholds()
.times(2)
.returning(move || {
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new();
violation.expect_get_metric_name().return_const("cpu_usage".to_string());
violation.expect_is_critical().return_const(is_critical);
vec![Box::new(violation)]
});
formatter.expect_format_violation()
.times(expected_notifications)
.returning(|_| "CPU usage violation".to_string());
notifier.expect_send_notification()
.times(expected_notifications)
.returning(|_| Ok(()));
let mut service = new(
Box::new(notifier),
Box::new(monitor),
Box::new(formatter),
);
assert!(service.alert_on_threshold_violation().is_ok());
assert!(service.alert_on_threshold_violation().is_ok());
}
#[rstest]
#[case(false, true, 2, "escalation from warning to critical")]
#[case(true, false, 2, "improvement from critical to warning")] // Now sends alert
#[case(false, false, 1, "warning then warning again")]
#[case(true, true, 1, "critical then critical again")]
fn test_alert_level_changes(
#[case] first_is_critical: bool,
#[case] second_is_critical: bool,
#[case] expected_notifications: usize,
#[case] _description: &str,
) {
let mut notifier = MockForSendingNotification::new();
let mut monitor = MockForMonitoringSystem::new();
let mut formatter = MockForFormattingMessage::new();
// Monitor returns different levels on consecutive calls
monitor.expect_check_thresholds()
.times(2)
.returning_st({
let mut call_count = 0;
move || {
call_count += 1;
let mut violation = crate::app::ports::driven::MockForGettingViolationData::new();
violation.expect_get_metric_name().return_const("cpu_usage".to_string());
if call_count == 1 {
violation.expect_is_critical().return_const(first_is_critical);
} else {
violation.expect_is_critical().return_const(second_is_critical);
}
vec![Box::new(violation)]
}
});
formatter.expect_format_violation()
.times(expected_notifications)
.returning(|_| "CPU usage violation".to_string());
notifier.expect_send_notification()
.times(expected_notifications)
.returning(|_| Ok(()));
let mut service = new(
Box::new(notifier),
Box::new(monitor),
Box::new(formatter),
);
assert!(service.alert_on_threshold_violation().is_ok());
assert!(service.alert_on_threshold_violation().is_ok());
} }
} }