MrYoshii
c756195530
Kompletter Neustart als Python-Projekt. Portiert von zfs-monitor Bash v3, saubere Architektur: - config.py: TOML-Config mit dataclasses - state.py: zentrales State-Management (JSON statt Einzeldateien) - notify.py: Telegram + Persistent Alerting als wiederverwendbare Klasse - checks/zfs.py: ZFS Pool-Health, I/O-Fehler, Kapazität - checks/smart.py: SATA + NVMe SMART-Monitoring - checks/nut.py: NUT/USV-Monitoring (APC Back-UPS RS 1500G)
257 lines
9.7 KiB
Python
257 lines
9.7 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
|
||
from checks.base import Check
|
||
from checks.zfs import ZFSCheck, _normalize_device
|
||
from state import make_key
|
||
|
||
|
||
class SmartCheck(Check):
|
||
def run(self) -> None:
|
||
disks = self._collect_disks()
|
||
self.log.info("--- SMART: %d Disks ---", len(disks))
|
||
for disk in disks:
|
||
self._check_disk(disk)
|
||
|
||
def _collect_disks(self) -> list[str]:
|
||
"""Collect all unique disks: pool members + extra_disks from config."""
|
||
seen: set[str] = set()
|
||
result: list[str] = []
|
||
|
||
zfs = ZFSCheck(self.config, self.state, self.alerter)
|
||
for pool in self.config.zfs.pools:
|
||
for disk in zfs.get_pool_disks(pool):
|
||
if disk not in seen:
|
||
seen.add(disk)
|
||
result.append(disk)
|
||
|
||
for extra in self.config.smart.extra_disks:
|
||
dev = _normalize_device(extra)
|
||
if os.path.exists(dev) and dev not in seen:
|
||
seen.add(dev)
|
||
result.append(dev)
|
||
|
||
return result
|
||
|
||
def _check_disk(self, disk: str) -> None:
|
||
if not os.path.exists(disk):
|
||
self.log.warning("SMART: %s nicht vorhanden", disk)
|
||
return
|
||
if "nvme" in disk:
|
||
self._check_nvme(disk)
|
||
else:
|
||
self._check_sata(disk)
|
||
|
||
# ── SATA ──────────────────────────────────────────────────
|
||
|
||
def _check_sata(self, disk: str) -> None:
|
||
key_fail = make_key("sm_fail", disk)
|
||
key_issues = make_key("sm_iss", disk)
|
||
key_delta = make_key("sm_dlt", disk)
|
||
|
||
raw = self._run(
|
||
["smartctl", "-j", "-A", "-H", disk],
|
||
timeout=self.config.smart.timeout,
|
||
)
|
||
if not raw:
|
||
self.log.warning("SMART: %s nicht lesbar", disk)
|
||
return
|
||
|
||
try:
|
||
d = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
self.log.warning("SMART: JSON-Parsing fehlgeschlagen für %s", disk)
|
||
return
|
||
|
||
passed = d.get("smart_status", {}).get("passed")
|
||
attrs = {a["id"]: a for a in d.get("ata_smart_attributes", {}).get("table", [])}
|
||
|
||
def attr_raw(aid: int) -> int:
|
||
return attrs.get(aid, {}).get("raw", {}).get("value", 0)
|
||
|
||
reallocated = attr_raw(5)
|
||
pending = attr_raw(197)
|
||
uncorr = attr_raw(198)
|
||
temp_val = d.get("temperature", {}).get("current")
|
||
if temp_val is None:
|
||
temp_val = attrs.get(194, attrs.get(190, {})).get("raw", {}).get("value", 0)
|
||
temp = int(temp_val) if temp_val else 0
|
||
|
||
# Overall SMART status
|
||
if passed is False:
|
||
if self.alerter.should_alert(key_fail, "CRITICAL"):
|
||
self.alerter.send_alert(
|
||
"CRITICAL",
|
||
f"SMART FAILED: {disk}",
|
||
f"<b>Disk <code>{disk}</code> meldet SMART FAILED!</b>\n\n"
|
||
f"Disk ist kurz vor dem Ausfall – sofort ersetzen!\n"
|
||
f"<code>smartctl -a {disk}</code>",
|
||
key_fail,
|
||
)
|
||
return
|
||
else:
|
||
if self.alerter.clear_alert(key_fail):
|
||
self.alerter.send_alert(
|
||
"INFO",
|
||
f"SMART wieder OK: {disk}",
|
||
f"Disk <code>{disk}</code> meldet wieder PASSED ✅",
|
||
)
|
||
|
||
# Schwellwert-Checks
|
||
cfg = self.config.smart
|
||
issues: list[str] = []
|
||
if reallocated > cfg.reallocated_max:
|
||
issues.append(f"⚠️ Reallocated Sectors: <b>{reallocated}</b> (max {cfg.reallocated_max})")
|
||
if pending > cfg.pending_max:
|
||
issues.append(f"⚠️ Pending Sectors: <b>{pending}</b>")
|
||
if uncorr > cfg.uncorrectable_max:
|
||
issues.append(f"⚠️ Uncorrectable Errors: <b>{uncorr}</b>")
|
||
if 0 < temp > cfg.temp_max:
|
||
issues.append(f"🌡️ Temperatur: <b>{temp}°C</b> (max {cfg.temp_max}°C)")
|
||
|
||
if issues:
|
||
if self.alerter.should_alert(key_issues, "WARNING"):
|
||
self.alerter.send_alert(
|
||
"WARNING",
|
||
f"SMART Warnung: {disk}",
|
||
f"Disk <code>{disk}</code>:\n\n"
|
||
+ "\n".join(issues)
|
||
+ f"\n\n<code>smartctl -a {disk}</code>",
|
||
key_issues,
|
||
)
|
||
else:
|
||
if self.alerter.clear_alert(key_issues):
|
||
self.alerter.send_alert(
|
||
"INFO",
|
||
f"SMART OK: {disk}",
|
||
f"Disk <code>{disk}</code>: alle Werte wieder im grünen Bereich ✅",
|
||
)
|
||
|
||
# Delta-Erkennung
|
||
prev_r: int = self.state.get(f"{key_delta}_r", -1)
|
||
prev_p: int = self.state.get(f"{key_delta}_p", -1)
|
||
|
||
if prev_r >= 0 and reallocated > prev_r:
|
||
self.alerter.send_alert(
|
||
"WARNING",
|
||
f"SMART verschlechtert: {disk}",
|
||
f"Disk <code>{disk}</code>: Reallocated Sectors gestiegen!\n"
|
||
f"<b>{prev_r} → {reallocated}</b>",
|
||
)
|
||
if prev_p >= 0 and pending > prev_p:
|
||
self.alerter.send_alert(
|
||
"CRITICAL",
|
||
f"SMART verschlechtert: {disk}",
|
||
f"Disk <code>{disk}</code>: Pending Sectors gestiegen!\n"
|
||
f"<b>{prev_p} → {pending}</b>\n\n"
|
||
f"⚠️ Scrub + Backup sofort prüfen!",
|
||
)
|
||
|
||
self.state.set(f"{key_delta}_r", reallocated)
|
||
self.state.set(f"{key_delta}_p", pending)
|
||
|
||
self.log.info(
|
||
"SATA %s: reallocated=%d pending=%d uncorr=%d temp=%d°C",
|
||
disk, reallocated, pending, uncorr, temp,
|
||
)
|
||
|
||
# ── NVMe ──────────────────────────────────────────────────
|
||
|
||
def _check_nvme(self, disk: str) -> None:
|
||
key_crit = make_key("nv_crit", disk)
|
||
key_spare = make_key("nv_spare", disk)
|
||
key_temp = make_key("nv_temp", disk)
|
||
key_media = make_key("nv_med", disk)
|
||
|
||
raw = self._run(
|
||
["smartctl", "-j", "-a", disk],
|
||
timeout=self.config.smart.timeout,
|
||
)
|
||
if not raw:
|
||
self.log.warning("SMART NVMe: %s nicht lesbar", disk)
|
||
return
|
||
|
||
try:
|
||
d = json.loads(raw)
|
||
except json.JSONDecodeError:
|
||
self.log.warning("SMART: JSON-Parsing fehlgeschlagen für %s", disk)
|
||
return
|
||
|
||
log_page = d.get("nvme_smart_health_information_log", {})
|
||
crit_warn = log_page.get("critical_warning", 0)
|
||
media_errors = log_page.get("media_errors", 0)
|
||
spare = log_page.get("available_spare", 100)
|
||
spare_thresh = log_page.get("available_spare_threshold", 10)
|
||
pct_used = log_page.get("percentage_used", 0)
|
||
temp = int(d.get("temperature", {}).get("current", 0) or 0)
|
||
|
||
# Critical Warning Bits
|
||
if crit_warn != 0:
|
||
if self.alerter.should_alert(key_crit, "CRITICAL"):
|
||
self.alerter.send_alert(
|
||
"CRITICAL",
|
||
f"NVMe Critical Warning: {disk}",
|
||
f"NVMe <code>{disk}</code>: Warning Flag <b>0x{crit_warn:02x}</b>\n\n"
|
||
f"Bit 0: Spare under threshold\n"
|
||
f"Bit 1: Temperatur over limit\n"
|
||
f"Bit 2: Reliability degraded\n"
|
||
f"Bit 3: Read-Only gesetzt\n"
|
||
f"Bit 4: Backup Device Failed\n\n"
|
||
f"<code>smartctl -a {disk}</code>",
|
||
key_crit,
|
||
)
|
||
else:
|
||
if self.alerter.clear_alert(key_crit):
|
||
self.alerter.send_alert(
|
||
"INFO",
|
||
f"NVMe Warning behoben: {disk}",
|
||
f"NVMe <code>{disk}</code>: Critical Warning Flag wieder 0 ✅",
|
||
)
|
||
|
||
# Available Spare
|
||
if spare <= spare_thresh:
|
||
if self.alerter.should_alert(key_spare, "WARNING"):
|
||
self.alerter.send_alert(
|
||
"WARNING",
|
||
f"NVMe Spare niedrig: {disk}",
|
||
f"NVMe <code>{disk}</code>: Available Spare <b>{spare}%</b> "
|
||
f"(Threshold: {spare_thresh}%)\n"
|
||
f"Percentage Used: {pct_used}%\n\n"
|
||
f"NVMe nähert sich dem Lebensende.",
|
||
key_spare,
|
||
)
|
||
else:
|
||
self.alerter.clear_alert(key_spare)
|
||
|
||
# Temperatur
|
||
if 0 < temp > self.config.smart.temp_max:
|
||
if self.alerter.should_alert(key_temp, "WARNING"):
|
||
self.alerter.send_alert(
|
||
"WARNING",
|
||
f"NVMe Temperatur: {disk}",
|
||
f"NVMe <code>{disk}</code>: <b>{temp}°C</b> "
|
||
f"(max {self.config.smart.temp_max}°C)",
|
||
key_temp,
|
||
)
|
||
else:
|
||
self.alerter.clear_alert(key_temp)
|
||
|
||
# Media Errors (Delta)
|
||
prev_media: int = self.state.get(f"{key_media}_prev", -1)
|
||
if prev_media >= 0 and media_errors > prev_media:
|
||
self.alerter.send_alert(
|
||
"WARNING",
|
||
f"NVMe Media Errors gestiegen: {disk}",
|
||
f"NVMe <code>{disk}</code>: Media Errors "
|
||
f"<b>{prev_media} → {media_errors}</b>",
|
||
)
|
||
self.state.set(f"{key_media}_prev", media_errors)
|
||
|
||
self.log.info(
|
||
"NVMe %s: crit=0x%02x media=%d spare=%d%% pct_used=%d%% temp=%d°C",
|
||
disk, crit_warn, media_errors, spare, pct_used, temp,
|
||
)
|