from __future__ import annotations import json import os import re from checks.base import Check from checks.zfs import ZFSCheck, _normalize_device from state import make_key class SmartCheck(Check): def run(self) -> None: disks = self._collect_disks() self.log.info("--- SMART: %d Disks ---", len(disks)) for disk in disks: self._check_disk(disk) def _collect_disks(self) -> list[str]: """Collect all unique disks: pool members + extra_disks from config.""" seen: set[str] = set() result: list[str] = [] zfs = ZFSCheck(self.config, self.state, self.alerter) for pool in self.config.zfs.pools: for disk in zfs.get_pool_disks(pool): if disk not in seen: seen.add(disk) result.append(disk) for extra in self.config.smart.extra_disks: dev = _normalize_device(extra) if os.path.exists(dev) and dev not in seen: seen.add(dev) result.append(dev) return result def _check_disk(self, disk: str) -> None: if not os.path.exists(disk): self.log.warning("SMART: %s nicht vorhanden", disk) return if "nvme" in disk: self._check_nvme(disk) else: self._check_sata(disk) # ── SATA ────────────────────────────────────────────────── def _check_sata(self, disk: str) -> None: key_fail = make_key("sm_fail", disk) key_issues = make_key("sm_iss", disk) key_delta = make_key("sm_dlt", disk) raw = self._run( ["smartctl", "-j", "-A", "-H", disk], timeout=self.config.smart.timeout, ) if not raw: self.log.warning("SMART: %s nicht lesbar", disk) return try: d = json.loads(raw) except json.JSONDecodeError: self.log.warning("SMART: JSON-Parsing fehlgeschlagen für %s", disk) return passed = d.get("smart_status", {}).get("passed") attrs = {a["id"]: a for a in d.get("ata_smart_attributes", {}).get("table", [])} def attr_raw(aid: int) -> int: return attrs.get(aid, {}).get("raw", {}).get("value", 0) reallocated = attr_raw(5) pending = attr_raw(197) uncorr = attr_raw(198) temp_val = d.get("temperature", {}).get("current") if temp_val is None: temp_val = attrs.get(194, attrs.get(190, {})).get("raw", {}).get("value", 0) temp = int(temp_val) if temp_val else 0 # Overall SMART status if passed is False: if self.alerter.should_alert(key_fail, "CRITICAL"): self.alerter.send_alert( "CRITICAL", f"SMART FAILED: {disk}", f"Disk {disk} meldet SMART FAILED!\n\n" f"Disk ist kurz vor dem Ausfall – sofort ersetzen!\n" f"smartctl -a {disk}", key_fail, ) return else: if self.alerter.clear_alert(key_fail): self.alerter.send_alert( "INFO", f"SMART wieder OK: {disk}", f"Disk {disk} meldet wieder PASSED ✅", ) # Schwellwert-Checks cfg = self.config.smart issues: list[str] = [] if reallocated > cfg.reallocated_max: issues.append(f"⚠️ Reallocated Sectors: {reallocated} (max {cfg.reallocated_max})") if pending > cfg.pending_max: issues.append(f"⚠️ Pending Sectors: {pending}") if uncorr > cfg.uncorrectable_max: issues.append(f"⚠️ Uncorrectable Errors: {uncorr}") if 0 < temp > cfg.temp_max: issues.append(f"🌡️ Temperatur: {temp}°C (max {cfg.temp_max}°C)") if issues: if self.alerter.should_alert(key_issues, "WARNING"): self.alerter.send_alert( "WARNING", f"SMART Warnung: {disk}", f"Disk {disk}:\n\n" + "\n".join(issues) + f"\n\nsmartctl -a {disk}", key_issues, ) else: if self.alerter.clear_alert(key_issues): self.alerter.send_alert( "INFO", f"SMART OK: {disk}", f"Disk {disk}: alle Werte wieder im grünen Bereich ✅", ) # Delta-Erkennung prev_r: int = self.state.get(f"{key_delta}_r", -1) prev_p: int = self.state.get(f"{key_delta}_p", -1) if prev_r >= 0 and reallocated > prev_r: self.alerter.send_alert( "WARNING", f"SMART verschlechtert: {disk}", f"Disk {disk}: Reallocated Sectors gestiegen!\n" f"{prev_r} → {reallocated}", ) if prev_p >= 0 and pending > prev_p: self.alerter.send_alert( "CRITICAL", f"SMART verschlechtert: {disk}", f"Disk {disk}: Pending Sectors gestiegen!\n" f"{prev_p} → {pending}\n\n" f"⚠️ Scrub + Backup sofort prüfen!", ) self.state.set(f"{key_delta}_r", reallocated) self.state.set(f"{key_delta}_p", pending) self.log.info( "SATA %s: reallocated=%d pending=%d uncorr=%d temp=%d°C", disk, reallocated, pending, uncorr, temp, ) # ── NVMe ────────────────────────────────────────────────── def _check_nvme(self, disk: str) -> None: key_crit = make_key("nv_crit", disk) key_spare = make_key("nv_spare", disk) key_temp = make_key("nv_temp", disk) key_media = make_key("nv_med", disk) raw = self._run( ["smartctl", "-j", "-a", disk], timeout=self.config.smart.timeout, ) if not raw: self.log.warning("SMART NVMe: %s nicht lesbar", disk) return try: d = json.loads(raw) except json.JSONDecodeError: self.log.warning("SMART: JSON-Parsing fehlgeschlagen für %s", disk) return log_page = d.get("nvme_smart_health_information_log", {}) crit_warn = log_page.get("critical_warning", 0) media_errors = log_page.get("media_errors", 0) spare = log_page.get("available_spare", 100) spare_thresh = log_page.get("available_spare_threshold", 10) pct_used = log_page.get("percentage_used", 0) temp = int(d.get("temperature", {}).get("current", 0) or 0) # Critical Warning Bits if crit_warn != 0: if self.alerter.should_alert(key_crit, "CRITICAL"): self.alerter.send_alert( "CRITICAL", f"NVMe Critical Warning: {disk}", f"NVMe {disk}: Warning Flag 0x{crit_warn:02x}\n\n" f"Bit 0: Spare under threshold\n" f"Bit 1: Temperatur over limit\n" f"Bit 2: Reliability degraded\n" f"Bit 3: Read-Only gesetzt\n" f"Bit 4: Backup Device Failed\n\n" f"smartctl -a {disk}", key_crit, ) else: if self.alerter.clear_alert(key_crit): self.alerter.send_alert( "INFO", f"NVMe Warning behoben: {disk}", f"NVMe {disk}: Critical Warning Flag wieder 0 ✅", ) # Available Spare if spare <= spare_thresh: if self.alerter.should_alert(key_spare, "WARNING"): self.alerter.send_alert( "WARNING", f"NVMe Spare niedrig: {disk}", f"NVMe {disk}: Available Spare {spare}% " f"(Threshold: {spare_thresh}%)\n" f"Percentage Used: {pct_used}%\n\n" f"NVMe nähert sich dem Lebensende.", key_spare, ) else: self.alerter.clear_alert(key_spare) # Temperatur if 0 < temp > self.config.smart.temp_max: if self.alerter.should_alert(key_temp, "WARNING"): self.alerter.send_alert( "WARNING", f"NVMe Temperatur: {disk}", f"NVMe {disk}: {temp}°C " f"(max {self.config.smart.temp_max}°C)", key_temp, ) else: self.alerter.clear_alert(key_temp) # Media Errors (Delta) prev_media: int = self.state.get(f"{key_media}_prev", -1) if prev_media >= 0 and media_errors > prev_media: self.alerter.send_alert( "WARNING", f"NVMe Media Errors gestiegen: {disk}", f"NVMe {disk}: Media Errors " f"{prev_media} → {media_errors}", ) self.state.set(f"{key_media}_prev", media_errors) self.log.info( "NVMe %s: crit=0x%02x media=%d spare=%d%% pct_used=%d%% temp=%d°C", disk, crit_warn, media_errors, spare, pct_used, temp, )