refactor(system_monitor): move metrics collection and alerting to dedicated modules
* node_notifier doesn't need sysinfo anymore. * implement AlertFormatter in node_notifier and its own file. * implement SystemMonitor, ResourceThreshold, ThresholdViolation, and Severity in system_monitor. * implement parametrized tests for SystemMonitor.
This commit is contained in:
238
system_monitor/Cargo.lock
generated
Normal file
238
system_monitor/Cargo.lock
generated
Normal file
@@ -0,0 +1,238 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.174"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-foundation"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-kit"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.35.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c3ffa3e4ff2b324a57f7aeb3c349656c7b127c3c189520251a648102a92496e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
"ntapi",
|
||||
"objc2-core-foundation",
|
||||
"objc2-io-kit",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "system_monitor"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"sysinfo",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.61.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893"
|
||||
dependencies = [
|
||||
"windows-collections",
|
||||
"windows-core",
|
||||
"windows-future",
|
||||
"windows-link",
|
||||
"windows-numerics",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-collections"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
|
||||
dependencies = [
|
||||
"windows-implement",
|
||||
"windows-interface",
|
||||
"windows-link",
|
||||
"windows-result",
|
||||
"windows-strings",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-future"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-link",
|
||||
"windows-threading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-implement"
|
||||
version = "0.60.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-interface"
|
||||
version = "0.59.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
|
||||
|
||||
[[package]]
|
||||
name = "windows-numerics"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1"
|
||||
dependencies = [
|
||||
"windows-core",
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-strings"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-threading"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
7
system_monitor/Cargo.toml
Normal file
7
system_monitor/Cargo.toml
Normal file
@@ -0,0 +1,7 @@
|
||||
[package]
|
||||
name = "system_monitor"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
sysinfo = "0.35.2"
|
||||
40
system_monitor/src/get_info.rs
Normal file
40
system_monitor/src/get_info.rs
Normal file
@@ -0,0 +1,40 @@
|
||||
use sysinfo::{Disks, System};
|
||||
|
||||
|
||||
pub fn get_cpu_usage(sys: &System, _: &Disks) -> f32 {
|
||||
sys.global_cpu_usage()
|
||||
}
|
||||
|
||||
pub fn get_memory_usage(sys: &System, _: &Disks) -> f32 {
|
||||
let total_memory = sys.total_memory() as f32;
|
||||
let used_memory = sys.used_memory() as f32;
|
||||
|
||||
used_memory / total_memory * 100.0
|
||||
}
|
||||
|
||||
pub fn get_swap_usage(sys: &System, _: &Disks) -> f32 {
|
||||
let total_swap = sys.total_swap() as f32;
|
||||
let used_swap = sys.used_swap() as f32;
|
||||
|
||||
used_swap / total_swap * 100.0
|
||||
}
|
||||
|
||||
pub fn get_disk_usage(_: &System, disks: &Disks) -> f32 {
|
||||
let mut total_used_space = 0.0;
|
||||
let mut total_space = 0.0;
|
||||
|
||||
for disk in disks {
|
||||
if let Some(mount_point) = disk.mount_point().to_str() {
|
||||
if mount_point == "/" || mount_point == "/home" {
|
||||
total_used_space += disk.total_space() as f32 - disk.available_space() as f32;
|
||||
total_space += disk.total_space() as f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if total_space == 0.0 {
|
||||
return 0.0; // Avoid division by zero
|
||||
}
|
||||
|
||||
(total_used_space / total_space) * 100.0
|
||||
}
|
||||
194
system_monitor/src/lib.rs
Normal file
194
system_monitor/src/lib.rs
Normal file
@@ -0,0 +1,194 @@
|
||||
use crate::resource_threshold::ResourceThreshold;
|
||||
use std::collections::HashMap;
|
||||
use sysinfo::{Disks, System};
|
||||
|
||||
mod get_info;
|
||||
pub mod resource_threshold;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ThresholdViolation {
|
||||
pub metric_name: String,
|
||||
pub current_value: f32,
|
||||
pub threshold: f32,
|
||||
pub severity: Severity,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Copy)]
|
||||
pub enum Severity {
|
||||
Warning,
|
||||
Critical,
|
||||
}
|
||||
|
||||
pub struct SystemMonitor {
|
||||
pub system: System,
|
||||
pub disks: Disks,
|
||||
pub thresholds: Vec<ResourceThreshold>,
|
||||
}
|
||||
|
||||
impl SystemMonitor {
|
||||
pub fn new(thresholds: Vec<ResourceThreshold>) -> Self {
|
||||
let mut system_monitor = SystemMonitor {
|
||||
system: System::new_all(),
|
||||
disks: Disks::new_with_refreshed_list(),
|
||||
thresholds,
|
||||
};
|
||||
system_monitor.refresh();
|
||||
|
||||
system_monitor
|
||||
}
|
||||
|
||||
pub fn refresh(&mut self) {
|
||||
self.system.refresh_all();
|
||||
self.disks.refresh(true);
|
||||
}
|
||||
|
||||
pub fn check_thresholds(&self) -> Vec<ThresholdViolation> {
|
||||
let mut violations = Vec::new();
|
||||
|
||||
for threshold in &self.thresholds {
|
||||
let current_value = threshold.get_value(&self.system, &self.disks);
|
||||
|
||||
// Check critical threshold first (highest priority)
|
||||
if let Some(critical_threshold) = threshold.critical_threshold {
|
||||
if current_value > critical_threshold {
|
||||
violations.push(ThresholdViolation {
|
||||
metric_name: threshold.name.clone(),
|
||||
current_value,
|
||||
threshold: critical_threshold,
|
||||
severity: Severity::Critical,
|
||||
});
|
||||
continue; // Skip warning check if critical
|
||||
}
|
||||
}
|
||||
|
||||
// Check warning threshold
|
||||
if let Some(warning_threshold) = threshold.warning_threshold {
|
||||
if current_value > warning_threshold {
|
||||
violations.push(ThresholdViolation {
|
||||
metric_name: threshold.name.clone(),
|
||||
current_value,
|
||||
threshold: warning_threshold,
|
||||
severity: Severity::Warning,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
violations
|
||||
}
|
||||
|
||||
/// Get current system metrics based on configured thresholds
|
||||
pub fn get_metrics(&self) -> HashMap<String, f32> {
|
||||
let mut metrics = HashMap::new();
|
||||
|
||||
for threshold in &self.thresholds {
|
||||
let value = threshold.get_value(&self.system, &self.disks);
|
||||
metrics.insert(threshold.name.clone(), value);
|
||||
}
|
||||
|
||||
metrics
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SystemMonitor {
|
||||
fn default() -> Self {
|
||||
Self::new(resource_threshold::get_default_resource_thresholds())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
struct TestCase {
|
||||
name: &'static str,
|
||||
value: f32,
|
||||
warning_threshold: Option<f32>,
|
||||
critical_threshold: Option<f32>,
|
||||
expected_violation: Option<Severity>,
|
||||
}
|
||||
|
||||
fn create_test_case() -> Vec<TestCase> {
|
||||
vec![
|
||||
TestCase {
|
||||
name: "Everything is fine",
|
||||
value: 1.0,
|
||||
warning_threshold: Some(2.0),
|
||||
critical_threshold: Some(3.0),
|
||||
expected_violation: None,
|
||||
},
|
||||
TestCase {
|
||||
name: "Warning threshold triggered",
|
||||
value: 2.5,
|
||||
warning_threshold: Some(2.0),
|
||||
critical_threshold: Some(3.0),
|
||||
expected_violation: Some(Severity::Warning),
|
||||
},
|
||||
TestCase {
|
||||
name: "Critical threshold triggered",
|
||||
value: 3.5,
|
||||
warning_threshold: Some(1.0),
|
||||
critical_threshold: Some(2.0),
|
||||
expected_violation: Some(Severity::Critical),
|
||||
},
|
||||
TestCase {
|
||||
name: "Warning threshold triggered with critical empty",
|
||||
value: 3.5,
|
||||
warning_threshold: Some(3.0),
|
||||
critical_threshold: None,
|
||||
expected_violation: Some(Severity::Warning),
|
||||
},
|
||||
TestCase {
|
||||
name: "Critical threshold triggered with warning empty",
|
||||
value: 3.5,
|
||||
warning_threshold: None,
|
||||
critical_threshold: Some(3.0),
|
||||
expected_violation: Some(Severity::Critical),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_thresholds() {
|
||||
// Iterate through each test case
|
||||
for test_case in create_test_case() {
|
||||
let mut monitor = SystemMonitor::default();
|
||||
|
||||
let mut threshold = ResourceThreshold::new(
|
||||
test_case.name.to_string(),
|
||||
move |_, _| test_case.value
|
||||
);
|
||||
|
||||
// Apply thresholds conditionally
|
||||
if let Some(warning) = test_case.warning_threshold {
|
||||
threshold = threshold.with_warning_threshold(warning);
|
||||
}
|
||||
if let Some(critical) = test_case.critical_threshold {
|
||||
threshold = threshold.with_critical_threshold(critical);
|
||||
}
|
||||
|
||||
monitor.thresholds = vec![threshold];
|
||||
|
||||
let violations = monitor.check_thresholds();
|
||||
if let Some(expected_severity) = test_case.expected_violation {
|
||||
assert_eq!(
|
||||
violations.len(),
|
||||
1,
|
||||
"Expected one violation for test case: {}",
|
||||
test_case.name
|
||||
);
|
||||
assert_eq!(
|
||||
violations[0].severity, expected_severity,
|
||||
"Unexpected severity for test case: {}",
|
||||
test_case.name
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
violations.is_empty(),
|
||||
"Expected no violations for test case: {}",
|
||||
test_case.name
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
62
system_monitor/src/resource_threshold.rs
Normal file
62
system_monitor/src/resource_threshold.rs
Normal file
@@ -0,0 +1,62 @@
|
||||
use crate::get_info;
|
||||
use sysinfo::{Disks, System};
|
||||
|
||||
pub struct ResourceThreshold {
|
||||
pub name: String,
|
||||
pub get_value_fn: Box<dyn Fn(&System, &Disks) -> f32>,
|
||||
pub warning_threshold: Option<f32>,
|
||||
pub critical_threshold: Option<f32>,
|
||||
}
|
||||
|
||||
impl ResourceThreshold {
|
||||
pub fn new<F>(name: String, get_value_fn: F) -> Self
|
||||
where
|
||||
F: Fn(&System, &Disks) -> f32 + 'static,
|
||||
{
|
||||
Self {
|
||||
name,
|
||||
get_value_fn: Box::new(get_value_fn),
|
||||
warning_threshold: None,
|
||||
critical_threshold: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_warning_threshold(mut self, warning_threshold: f32) -> Self {
|
||||
self.warning_threshold = Some(warning_threshold);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_critical_threshold(mut self, critical_threshold: f32) -> Self {
|
||||
self.critical_threshold = Some(critical_threshold);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn get_value(&self, sys: &System, disks: &Disks) -> f32 {
|
||||
(self.get_value_fn)(sys, disks)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_default_resource_thresholds() -> Vec<ResourceThreshold> {
|
||||
vec![
|
||||
ResourceThreshold::new("CPU Usage".to_string(), |sys, disks| {
|
||||
get_info::get_cpu_usage(sys, disks)
|
||||
})
|
||||
.with_warning_threshold(80.0)
|
||||
.with_critical_threshold(95.0),
|
||||
ResourceThreshold::new("Memory Usage".to_string(), |sys, disks| {
|
||||
get_info::get_memory_usage(sys, disks)
|
||||
})
|
||||
.with_warning_threshold(85.0)
|
||||
.with_critical_threshold(95.0),
|
||||
ResourceThreshold::new("Swap Usage".to_string(), |sys, disks| {
|
||||
get_info::get_swap_usage(sys, disks)
|
||||
})
|
||||
.with_warning_threshold(1.0)
|
||||
.with_critical_threshold(25.0),
|
||||
ResourceThreshold::new("Disk Usage".to_string(), |sys, disks| {
|
||||
get_info::get_disk_usage(sys, disks)
|
||||
})
|
||||
.with_warning_threshold(80.0)
|
||||
.with_critical_threshold(90.0),
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user