feat: ✨ S.M.A.R.T status report

bin/core/src/alert/discord.rs

@@ -117,12 +117,23 @@ pub async fn send_alert(
       path,
       used_gb,
       total_gb,
+      healthy,
+      temperature,
     } => {
       let region = fmt_region(region);
       let link = resource_link(ResourceTargetVariant::Server, id);
       let percentage = 100.0 * used_gb / total_gb;
+      let mut extra = String::new();
+      if *healthy == Some(false) {
+        extra.push_str("\nSMART Health: **FAILED** ❌");
+      }
+      if let Some(temp) = temperature
+        && *temp > 0
+      {
+        extra.push_str(&format!("\nTemperature: **{temp}°C**"));
+      }
       format!(
-        "{level} | **{name}**{region} disk usage at **{percentage:.1}%** 💿\nmount point: `{path:?}`\nusing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**\n{link}"
+        "{level} | **{name}**{region} disk alert at **{percentage:.1}%** 💿\nmount point: `{path:?}`\nusing **{used_gb:.1} GiB** / **{total_gb:.1} GiB**{extra}\n{link}"
       )
     }
     AlertData::ContainerStateChange {

bin/core/src/alert/mod.rs

@@ -348,12 +348,23 @@ fn standard_alert_content(alert: &Alert) -> String {
       path,
       used_gb,
       total_gb,
+      healthy,
+      temperature,
     } => {
       let region = fmt_region(region);
       let link = resource_link(ResourceTargetVariant::Server, id);
       let percentage = 100.0 * used_gb / total_gb;
+      let mut extra = String::new();
+      if *healthy == Some(false) {
+        extra.push_str("\nSMART Health: FAILED ❌");
+      }
+      if let Some(temp) = temperature
+        && *temp > 0
+      {
+        extra.push_str(&format!("\nTemperature: {temp}°C"));
+      }
       format!(
-        "{level} | {name}{region} disk usage at {percentage:.1}%💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB\n{link}",
+        "{level} | {name}{region} disk alert at {percentage:.1}% 💿\nmount point: {path:?}\nusing {used_gb:.1} GiB / {total_gb:.1} GiB{extra}\n{link}",
       )
     }
     AlertData::ContainerStateChange {

bin/core/src/alert/slack.rs

@@ -228,49 +228,37 @@ pub async fn send_alert(
       path,
       used_gb,
       total_gb,
+      healthy,
+      temperature,
     } => {
       let region = fmt_region(region);
       let percentage = 100.0 * used_gb / total_gb;
-      match alert.level {
-        SeverityLevel::Ok => {
-          let text = format!(
-            "{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path:?}* 💿"
-          );
-          let blocks = vec![
-            Block::header(level),
-            Block::section(format!(
-              "*{name}*{region} disk usage at *{percentage:.1}%* 💿"
-            )),
-            Block::section(format!(
-              "mount point: {path:?} | using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*"
-            )),
-            Block::section(resource_link(
-              ResourceTargetVariant::Server,
-              id,
-            )),
-          ];
-          (text, blocks.into())
-        }
-        _ => {
-          let text = format!(
-            "{level} | *{name}*{region} disk usage at *{percentage:.1}%* | mount point: *{path:?}* 💿"
-          );
-          let blocks = vec![
-            Block::header(level),
-            Block::section(format!(
-              "*{name}*{region} disk usage at *{percentage:.1}%* 💿"
-            )),
-            Block::section(format!(
-              "mount point: {path:?} | using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*"
-            )),
-            Block::section(resource_link(
-              ResourceTargetVariant::Server,
-              id,
-            )),
-          ];
-          (text, blocks.into())
-        }
+      let mut extra = String::new();
+      if *healthy == Some(false) {
+        extra.push_str("\nSMART Health: *FAILED* ❌");
       }
+      if let Some(temp) = temperature
+        && *temp > 0
+      {
+        extra.push_str(&format!("\nTemperature: *{temp}°C*"));
+      }
+      let text = format!(
+        "{level} | *{name}*{region} disk alert at *{percentage:.1}%* | mount point: *{path:?}* 💿"
+      );
+      let blocks = vec![
+        Block::header(level),
+        Block::section(format!(
+          "*{name}*{region} disk alert at *{percentage:.1}%* 💿"
+        )),
+        Block::section(format!(
+          "mount point: {path:?} | using *{used_gb:.1} GiB* / *{total_gb:.1} GiB*{extra}"
+        )),
+        Block::section(resource_link(
+          ResourceTargetVariant::Server,
+          id,
+        )),
+      ];
+      (text, blocks.into())
     }
     AlertData::ContainerStateChange {
       name,

bin/core/src/monitor/alert/server.rs

@@ -531,6 +531,8 @@ pub async fn alert_servers(
                   .map(|d| d.total_gb)
                   .unwrap_or_default(),
                 used_gb: disk.map(|d| d.used_gb).unwrap_or_default(),
+                healthy: disk.and_then(|d| d.healthy),
+                temperature: disk.map(|d| d.temperature),
               },
             };
             alerts_to_open
@@ -557,6 +559,8 @@ pub async fn alert_servers(
               path: path.to_owned(),
               total_gb: disk.map(|d| d.total_gb).unwrap_or_default(),
               used_gb: disk.map(|d| d.used_gb).unwrap_or_default(),
+              healthy: disk.and_then(|d| d.healthy),
+              temperature: disk.map(|d| d.temperature),
             };
             alerts_to_update
               .push((alert, server.config.send_disk_alerts));
@@ -584,6 +588,8 @@ pub async fn alert_servers(
             path: path.to_owned(),
             total_gb: disk.map(|d| d.total_gb).unwrap_or_default(),
             used_gb: disk.map(|d| d.used_gb).unwrap_or_default(),
+            healthy: disk.and_then(|d| d.healthy),
+            temperature: disk.map(|d| d.temperature),
           };
           alerts_to_close
             .push((alert, server.config.send_disk_alerts))

bin/core/src/monitor/helpers.rs

@@ -157,6 +157,8 @@ fn get_server_health(
     mount,
     used_gb,
     total_gb,
+    healthy,
+    temperature,
     ..
   } in disks
   {
@@ -171,6 +173,22 @@ fn get_server_health(
     {
       state.should_close_alert = true;
     };
+
+    if healthy == &Some(false) {
+      state.level = SeverityLevel::Critical;
+      state.should_close_alert = false;
+    }
+
+    if *temperature >= 60 {
+      state.level = SeverityLevel::Critical;
+      state.should_close_alert = false;
+    } else if *temperature >= 50 {
+      if state.level < SeverityLevel::Warning {
+        state.level = SeverityLevel::Warning;
+      }
+      state.should_close_alert = false;
+    }
+
     health.disks.insert(mount.clone(), state);
   }
 

bin/periphery/debian-deps.sh

@@ -3,7 +3,7 @@
 ## Periphery deps installer
 
 apt-get update
-apt-get install -y git curl wget ca-certificates
+apt-get install -y git curl wget ca-certificates smartmontools util-linux
 
 install -m 0755 -d /etc/apt/keyrings
 curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc

bin/periphery/src/stats/disk.rs

@@ -0,0 +1,153 @@
+use std::process::Command;
+
+use serde::Deserialize;
+
+#[derive(Deserialize, Debug, PartialEq, Eq, Default)]
+pub struct SmartStatus {
+  pub passed: bool,
+}
+
+#[derive(Deserialize, Debug, PartialEq, Eq, Default)]
+pub struct PowerOnTime {
+  #[serde(default)]
+  pub hours: u64,
+}
+
+#[derive(Deserialize, Debug, PartialEq, Eq, Default)]
+pub struct Temperature {
+  #[serde(default)]
+  pub current: u64,
+}
+
+#[derive(Deserialize, Debug, PartialEq, Eq)]
+pub struct SmartReport {
+  pub smart_status: SmartStatus,
+  #[serde(default)]
+  pub power_on_time: PowerOnTime,
+  #[serde(default)]
+  pub temperature: Temperature,
+}
+
+/// Parse smartctl JSON output string to extract SMART report
+fn parse_smart_report(json_str: &str) -> Option<SmartReport> {
+  serde_json::from_str(json_str).ok()
+}
+
+/// given a device path it will try to get the SMART data about a disk
+pub fn get_smart_data(device_path: &str) -> Option<SmartReport> {
+  let output = Command::new("smartctl")
+    .args(["-a", "-j", device_path])
+    .output()
+    .ok()?;
+  if output.status.success() || !output.stdout.is_empty() {
+    let json_str = String::from_utf8_lossy(&output.stdout);
+    return parse_smart_report(&json_str);
+  }
+  None
+}
+
+#[derive(Deserialize, Debug)]
+pub struct LsblkOutput {
+  pub blockdevices: Vec<LsblkDevice>,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct LsblkDevice {
+  pub name: String,
+  /// 'type' is a reserved keyword in Rust, so rename it
+  #[serde(rename = "type")]
+  pub device_type: String,
+  /// Leaf nodes will not have children, so default to empty Vec
+  #[serde(default)]
+  pub children: Vec<LsblkDevice>,
+}
+
+impl LsblkDevice {
+  /// Recursively traverse the tree to find the physical disk name ('type' == "disk")
+  pub fn find_physical_disk(&self) -> Option<String> {
+    if self.device_type == "disk" {
+      return Some(self.name.clone());
+    }
+    for child in &self.children {
+      if let Some(disk_name) = child.find_physical_disk() {
+        return Some(disk_name);
+      }
+    }
+    None
+  }
+}
+
+pub fn volume_to_device_mapper(mapper: &str) -> Option<String> {
+  let output = Command::new("lsblk")
+    .args(["-s", "-J", mapper])
+    .output()
+    .ok()?;
+  if !output.status.success() || output.stdout.is_empty() {
+    return None;
+  }
+  let output: Option<LsblkOutput> =
+    serde_json::from_str(&String::from_utf8_lossy(&output.stdout))
+      .ok()?;
+  let output = output.unwrap();
+  let disk_name = output
+    .blockdevices
+    .iter()
+    .find_map(|dev| dev.find_physical_disk())?;
+
+  Some(format!("/dev/{disk_name}"))
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_parse_healthy_drive_full_metrics() {
+    let json = r#"{
+      "smart_status": { "passed": true },
+      "power_on_time": { "hours": 14200 },
+      "temperature": { "current": 34 }
+    }"#;
+    assert_eq!(
+      parse_smart_report(json),
+      Some(SmartReport {
+        smart_status: SmartStatus { passed: true },
+        power_on_time: PowerOnTime { hours: 14200 },
+        temperature: Temperature { current: 34 },
+      })
+    );
+  }
+
+  #[test]
+  fn test_parse_missing_optional_fields() {
+    let json = r#"{
+      "smart_status": { "passed": true }
+    }"#;
+    assert_eq!(
+      parse_smart_report(json),
+      Some(SmartReport {
+        smart_status: SmartStatus { passed: true },
+        power_on_time: PowerOnTime { hours: 0 },
+        temperature: Temperature { current: 0 },
+      })
+    );
+  }
+
+  #[test]
+  fn test_parse_failing_drive() {
+    let json = r#"{
+      "smart_status": { "passed": false },
+      "power_on_time": { "hours": 50000 },
+      "temperature": { "current": 55 }
+    }"#;
+    let report = parse_smart_report(json).unwrap();
+    assert!(!report.smart_status.passed);
+  }
+
+  #[test]
+  fn test_parse_unsupported_or_invalid() {
+    let json = r#"{ "device": { "name": "/dev/loop0" } }"#;
+    assert_eq!(parse_smart_report(json), None);
+    assert_eq!(parse_smart_report("invalid json"), None);
+  }
+}

bin/periphery/src/stats/mod.rs

@@ -7,8 +7,13 @@ use komodo_client::entities::stats::{
 };
 use sysinfo::{ProcessRefreshKind, ProcessesToUpdate, System};
 
-use crate::{config::periphery_config, state::stats_client};
+use crate::{
+  config::periphery_config,
+  state::stats_client,
+  stats::disk::{get_smart_data, volume_to_device_mapper},
+};
 
+mod disk;
 mod mem;
 
 /// This should be called before starting the server in main.rs.
@@ -146,11 +151,26 @@ impl StatsClient {
           disk.file_system().to_string_lossy().to_string();
         let disk_total = disk.total_space() as f64 / BYTES_PER_GB;
         let disk_free = disk.available_space() as f64 / BYTES_PER_GB;
+        let real_path =
+          volume_to_device_mapper(&disk.name().to_string_lossy())
+            .unwrap_or(disk.name().to_string_lossy().to_string());
+        let mut healthy = None;
+        let mut power_on_hours = 0;
+        let mut temperature = 0;
+        if let Some(smart_data) = get_smart_data(&real_path) {
+          healthy = Some(smart_data.smart_status.passed);
+          power_on_hours = smart_data.power_on_time.hours;
+          temperature = smart_data.temperature.current;
+        }
+
         SingleDiskUsage {
           mount: disk.mount_point().to_owned(),
           used_gb: disk_total - disk_free,
           total_gb: disk_total,
           file_system,
+          healthy,
+          power_on_hours,
+          temperature,
         }
       })
       .collect()