Initial commit: server-monitor v1 (Python)
Kompletter Neustart als Python-Projekt. Portiert von zfs-monitor Bash v3, saubere Architektur: - config.py: TOML-Config mit dataclasses - state.py: zentrales State-Management (JSON statt Einzeldateien) - notify.py: Telegram + Persistent Alerting als wiederverwendbare Klasse - checks/zfs.py: ZFS Pool-Health, I/O-Fehler, Kapazität - checks/smart.py: SATA + NVMe SMART-Monitoring - checks/nut.py: NUT/USV-Monitoring (APC Back-UPS RS 1500G)
This commit is contained in:
+256
@@ -0,0 +1,256 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
from checks.base import Check
|
||||
from checks.zfs import ZFSCheck, _normalize_device
|
||||
from state import make_key
|
||||
|
||||
|
||||
class SmartCheck(Check):
|
||||
def run(self) -> None:
|
||||
disks = self._collect_disks()
|
||||
self.log.info("--- SMART: %d Disks ---", len(disks))
|
||||
for disk in disks:
|
||||
self._check_disk(disk)
|
||||
|
||||
def _collect_disks(self) -> list[str]:
|
||||
"""Collect all unique disks: pool members + extra_disks from config."""
|
||||
seen: set[str] = set()
|
||||
result: list[str] = []
|
||||
|
||||
zfs = ZFSCheck(self.config, self.state, self.alerter)
|
||||
for pool in self.config.zfs.pools:
|
||||
for disk in zfs.get_pool_disks(pool):
|
||||
if disk not in seen:
|
||||
seen.add(disk)
|
||||
result.append(disk)
|
||||
|
||||
for extra in self.config.smart.extra_disks:
|
||||
dev = _normalize_device(extra)
|
||||
if os.path.exists(dev) and dev not in seen:
|
||||
seen.add(dev)
|
||||
result.append(dev)
|
||||
|
||||
return result
|
||||
|
||||
def _check_disk(self, disk: str) -> None:
|
||||
if not os.path.exists(disk):
|
||||
self.log.warning("SMART: %s nicht vorhanden", disk)
|
||||
return
|
||||
if "nvme" in disk:
|
||||
self._check_nvme(disk)
|
||||
else:
|
||||
self._check_sata(disk)
|
||||
|
||||
# ── SATA ──────────────────────────────────────────────────
|
||||
|
||||
def _check_sata(self, disk: str) -> None:
|
||||
key_fail = make_key("sm_fail", disk)
|
||||
key_issues = make_key("sm_iss", disk)
|
||||
key_delta = make_key("sm_dlt", disk)
|
||||
|
||||
raw = self._run(
|
||||
["smartctl", "-j", "-A", "-H", disk],
|
||||
timeout=self.config.smart.timeout,
|
||||
)
|
||||
if not raw:
|
||||
self.log.warning("SMART: %s nicht lesbar", disk)
|
||||
return
|
||||
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
self.log.warning("SMART: JSON-Parsing fehlgeschlagen für %s", disk)
|
||||
return
|
||||
|
||||
passed = d.get("smart_status", {}).get("passed")
|
||||
attrs = {a["id"]: a for a in d.get("ata_smart_attributes", {}).get("table", [])}
|
||||
|
||||
def attr_raw(aid: int) -> int:
|
||||
return attrs.get(aid, {}).get("raw", {}).get("value", 0)
|
||||
|
||||
reallocated = attr_raw(5)
|
||||
pending = attr_raw(197)
|
||||
uncorr = attr_raw(198)
|
||||
temp_val = d.get("temperature", {}).get("current")
|
||||
if temp_val is None:
|
||||
temp_val = attrs.get(194, attrs.get(190, {})).get("raw", {}).get("value", 0)
|
||||
temp = int(temp_val) if temp_val else 0
|
||||
|
||||
# Overall SMART status
|
||||
if passed is False:
|
||||
if self.alerter.should_alert(key_fail, "CRITICAL"):
|
||||
self.alerter.send_alert(
|
||||
"CRITICAL",
|
||||
f"SMART FAILED: {disk}",
|
||||
f"<b>Disk <code>{disk}</code> meldet SMART FAILED!</b>\n\n"
|
||||
f"Disk ist kurz vor dem Ausfall – sofort ersetzen!\n"
|
||||
f"<code>smartctl -a {disk}</code>",
|
||||
key_fail,
|
||||
)
|
||||
return
|
||||
else:
|
||||
if self.alerter.clear_alert(key_fail):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"SMART wieder OK: {disk}",
|
||||
f"Disk <code>{disk}</code> meldet wieder PASSED ✅",
|
||||
)
|
||||
|
||||
# Schwellwert-Checks
|
||||
cfg = self.config.smart
|
||||
issues: list[str] = []
|
||||
if reallocated > cfg.reallocated_max:
|
||||
issues.append(f"⚠️ Reallocated Sectors: <b>{reallocated}</b> (max {cfg.reallocated_max})")
|
||||
if pending > cfg.pending_max:
|
||||
issues.append(f"⚠️ Pending Sectors: <b>{pending}</b>")
|
||||
if uncorr > cfg.uncorrectable_max:
|
||||
issues.append(f"⚠️ Uncorrectable Errors: <b>{uncorr}</b>")
|
||||
if 0 < temp > cfg.temp_max:
|
||||
issues.append(f"🌡️ Temperatur: <b>{temp}°C</b> (max {cfg.temp_max}°C)")
|
||||
|
||||
if issues:
|
||||
if self.alerter.should_alert(key_issues, "WARNING"):
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"SMART Warnung: {disk}",
|
||||
f"Disk <code>{disk}</code>:\n\n"
|
||||
+ "\n".join(issues)
|
||||
+ f"\n\n<code>smartctl -a {disk}</code>",
|
||||
key_issues,
|
||||
)
|
||||
else:
|
||||
if self.alerter.clear_alert(key_issues):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"SMART OK: {disk}",
|
||||
f"Disk <code>{disk}</code>: alle Werte wieder im grünen Bereich ✅",
|
||||
)
|
||||
|
||||
# Delta-Erkennung
|
||||
prev_r: int = self.state.get(f"{key_delta}_r", -1)
|
||||
prev_p: int = self.state.get(f"{key_delta}_p", -1)
|
||||
|
||||
if prev_r >= 0 and reallocated > prev_r:
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"SMART verschlechtert: {disk}",
|
||||
f"Disk <code>{disk}</code>: Reallocated Sectors gestiegen!\n"
|
||||
f"<b>{prev_r} → {reallocated}</b>",
|
||||
)
|
||||
if prev_p >= 0 and pending > prev_p:
|
||||
self.alerter.send_alert(
|
||||
"CRITICAL",
|
||||
f"SMART verschlechtert: {disk}",
|
||||
f"Disk <code>{disk}</code>: Pending Sectors gestiegen!\n"
|
||||
f"<b>{prev_p} → {pending}</b>\n\n"
|
||||
f"⚠️ Scrub + Backup sofort prüfen!",
|
||||
)
|
||||
|
||||
self.state.set(f"{key_delta}_r", reallocated)
|
||||
self.state.set(f"{key_delta}_p", pending)
|
||||
|
||||
self.log.info(
|
||||
"SATA %s: reallocated=%d pending=%d uncorr=%d temp=%d°C",
|
||||
disk, reallocated, pending, uncorr, temp,
|
||||
)
|
||||
|
||||
# ── NVMe ──────────────────────────────────────────────────
|
||||
|
||||
def _check_nvme(self, disk: str) -> None:
|
||||
key_crit = make_key("nv_crit", disk)
|
||||
key_spare = make_key("nv_spare", disk)
|
||||
key_temp = make_key("nv_temp", disk)
|
||||
key_media = make_key("nv_med", disk)
|
||||
|
||||
raw = self._run(
|
||||
["smartctl", "-j", "-a", disk],
|
||||
timeout=self.config.smart.timeout,
|
||||
)
|
||||
if not raw:
|
||||
self.log.warning("SMART NVMe: %s nicht lesbar", disk)
|
||||
return
|
||||
|
||||
try:
|
||||
d = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
self.log.warning("SMART: JSON-Parsing fehlgeschlagen für %s", disk)
|
||||
return
|
||||
|
||||
log_page = d.get("nvme_smart_health_information_log", {})
|
||||
crit_warn = log_page.get("critical_warning", 0)
|
||||
media_errors = log_page.get("media_errors", 0)
|
||||
spare = log_page.get("available_spare", 100)
|
||||
spare_thresh = log_page.get("available_spare_threshold", 10)
|
||||
pct_used = log_page.get("percentage_used", 0)
|
||||
temp = int(d.get("temperature", {}).get("current", 0) or 0)
|
||||
|
||||
# Critical Warning Bits
|
||||
if crit_warn != 0:
|
||||
if self.alerter.should_alert(key_crit, "CRITICAL"):
|
||||
self.alerter.send_alert(
|
||||
"CRITICAL",
|
||||
f"NVMe Critical Warning: {disk}",
|
||||
f"NVMe <code>{disk}</code>: Warning Flag <b>0x{crit_warn:02x}</b>\n\n"
|
||||
f"Bit 0: Spare under threshold\n"
|
||||
f"Bit 1: Temperatur over limit\n"
|
||||
f"Bit 2: Reliability degraded\n"
|
||||
f"Bit 3: Read-Only gesetzt\n"
|
||||
f"Bit 4: Backup Device Failed\n\n"
|
||||
f"<code>smartctl -a {disk}</code>",
|
||||
key_crit,
|
||||
)
|
||||
else:
|
||||
if self.alerter.clear_alert(key_crit):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"NVMe Warning behoben: {disk}",
|
||||
f"NVMe <code>{disk}</code>: Critical Warning Flag wieder 0 ✅",
|
||||
)
|
||||
|
||||
# Available Spare
|
||||
if spare <= spare_thresh:
|
||||
if self.alerter.should_alert(key_spare, "WARNING"):
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"NVMe Spare niedrig: {disk}",
|
||||
f"NVMe <code>{disk}</code>: Available Spare <b>{spare}%</b> "
|
||||
f"(Threshold: {spare_thresh}%)\n"
|
||||
f"Percentage Used: {pct_used}%\n\n"
|
||||
f"NVMe nähert sich dem Lebensende.",
|
||||
key_spare,
|
||||
)
|
||||
else:
|
||||
self.alerter.clear_alert(key_spare)
|
||||
|
||||
# Temperatur
|
||||
if 0 < temp > self.config.smart.temp_max:
|
||||
if self.alerter.should_alert(key_temp, "WARNING"):
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"NVMe Temperatur: {disk}",
|
||||
f"NVMe <code>{disk}</code>: <b>{temp}°C</b> "
|
||||
f"(max {self.config.smart.temp_max}°C)",
|
||||
key_temp,
|
||||
)
|
||||
else:
|
||||
self.alerter.clear_alert(key_temp)
|
||||
|
||||
# Media Errors (Delta)
|
||||
prev_media: int = self.state.get(f"{key_media}_prev", -1)
|
||||
if prev_media >= 0 and media_errors > prev_media:
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"NVMe Media Errors gestiegen: {disk}",
|
||||
f"NVMe <code>{disk}</code>: Media Errors "
|
||||
f"<b>{prev_media} → {media_errors}</b>",
|
||||
)
|
||||
self.state.set(f"{key_media}_prev", media_errors)
|
||||
|
||||
self.log.info(
|
||||
"NVMe %s: crit=0x%02x media=%d spare=%d%% pct_used=%d%% temp=%d°C",
|
||||
disk, crit_warn, media_errors, spare, pct_used, temp,
|
||||
)
|
||||
Reference in New Issue
Block a user