Initial commit: server-monitor v1 (Python)
Kompletter Neustart als Python-Projekt. Portiert von zfs-monitor Bash v3, saubere Architektur: - config.py: TOML-Config mit dataclasses - state.py: zentrales State-Management (JSON statt Einzeldateien) - notify.py: Telegram + Persistent Alerting als wiederverwendbare Klasse - checks/zfs.py: ZFS Pool-Health, I/O-Fehler, Kapazität - checks/smart.py: SATA + NVMe SMART-Monitoring - checks/nut.py: NUT/USV-Monitoring (APC Back-UPS RS 1500G)
This commit is contained in:
+182
@@ -0,0 +1,182 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from checks.base import Check
|
||||
from state import make_key
|
||||
|
||||
|
||||
class ZFSCheck(Check):
|
||||
def run(self) -> None:
|
||||
for pool in self.config.zfs.pools:
|
||||
self.log.info("--- Pool: %s ---", pool)
|
||||
self._check_pool(pool)
|
||||
|
||||
def get_pool_disks(self, pool: str) -> list[str]:
|
||||
"""Return normalized block device paths for all disks in a pool."""
|
||||
out = self._run(
|
||||
["zpool", "status", "-P", pool],
|
||||
timeout=self.config.zfs.timeout,
|
||||
)
|
||||
if not out:
|
||||
return []
|
||||
|
||||
disks: list[str] = []
|
||||
for line in out.splitlines():
|
||||
m = re.match(r"\s+(/dev/\S+)", line)
|
||||
if m:
|
||||
dev = _normalize_device(m.group(1))
|
||||
if dev not in disks:
|
||||
disks.append(dev)
|
||||
return disks
|
||||
|
||||
# ── Pool checks ───────────────────────────────────────────
|
||||
|
||||
def _check_pool(self, pool: str) -> None:
|
||||
key_missing = make_key("pool_miss", pool)
|
||||
key_health = make_key("pool_hlth", pool)
|
||||
key_errors = make_key("pool_err", pool)
|
||||
key_cap = make_key("pool_cap", pool)
|
||||
|
||||
# Erreichbarkeit
|
||||
out = self._run(
|
||||
["zpool", "list", pool],
|
||||
timeout=self.config.zfs.timeout,
|
||||
)
|
||||
if out is None:
|
||||
if self.alerter.should_alert(key_missing, "CRITICAL"):
|
||||
self.alerter.send_alert(
|
||||
"CRITICAL",
|
||||
f"Pool verschwunden: {pool}",
|
||||
f"Pool <code>{pool}</code> ist nicht mehr importiert!\n\n"
|
||||
f"<code>zpool import\nzpool status</code>",
|
||||
key_missing,
|
||||
)
|
||||
return
|
||||
else:
|
||||
if self.alerter.clear_alert(key_missing):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"Pool wieder erreichbar: {pool}",
|
||||
f"Pool <code>{pool}</code> ist wieder da ✅",
|
||||
)
|
||||
|
||||
# Health
|
||||
health = (
|
||||
self._run(
|
||||
["zpool", "list", "-H", "-o", "health", pool],
|
||||
timeout=self.config.zfs.timeout,
|
||||
) or ""
|
||||
).strip()
|
||||
|
||||
if health != "ONLINE":
|
||||
self.state.set(f"{key_health}_last", health)
|
||||
if self.alerter.should_alert(key_health, "CRITICAL"):
|
||||
details = self._pool_status_summary(pool)
|
||||
self.alerter.send_alert(
|
||||
"CRITICAL",
|
||||
f"Pool {pool}: {health}!",
|
||||
f"Pool <code>{pool}</code> ist <b>{health}</b>!\n\n"
|
||||
f"<pre>{details}</pre>\n"
|
||||
f"<code>zpool status -v {pool}</code>",
|
||||
key_health,
|
||||
)
|
||||
else:
|
||||
self.state.set(f"{key_health}_last", "ONLINE")
|
||||
if self.alerter.clear_alert(key_health):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"Pool {pool}: wieder ONLINE ✅",
|
||||
f"Pool <code>{pool}</code> ist wieder gesund.",
|
||||
)
|
||||
|
||||
# I/O Errors
|
||||
err_lines = self._parse_io_errors(pool)
|
||||
if err_lines:
|
||||
if self.alerter.should_alert(key_errors, "WARNING"):
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"Pool {pool}: I/O Fehler!",
|
||||
f"Pool <code>{pool}</code> hat Fehler auf Disks:\n\n"
|
||||
f"<pre>{err_lines}</pre>\n"
|
||||
f"→ <code>zpool scrub {pool}</code>",
|
||||
key_errors,
|
||||
)
|
||||
else:
|
||||
if self.alerter.clear_alert(key_errors):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"Pool {pool}: I/O Fehler behoben ✅",
|
||||
f"Keine I/O Fehler mehr in Pool <code>{pool}</code>.",
|
||||
)
|
||||
|
||||
# Kapazität
|
||||
cap_str = (
|
||||
self._run(
|
||||
["zpool", "list", "-H", "-o", "cap", pool],
|
||||
timeout=self.config.zfs.timeout,
|
||||
) or "0"
|
||||
).strip().rstrip("%")
|
||||
cap = int(cap_str) if cap_str.isdigit() else 0
|
||||
|
||||
if cap >= self.config.zfs.crit_pct:
|
||||
if self.alerter.should_alert(key_cap, "CRITICAL"):
|
||||
self.alerter.send_alert(
|
||||
"CRITICAL",
|
||||
f"Pool {pool}: {cap}% voll!",
|
||||
f"Pool <code>{pool}</code> ist zu <b>{cap}%</b> voll!\n"
|
||||
f"Sofort Platz schaffen – ZFS braucht freien Raum!",
|
||||
key_cap,
|
||||
)
|
||||
elif cap >= self.config.zfs.warn_pct:
|
||||
if self.alerter.should_alert(key_cap, "WARNING"):
|
||||
self.alerter.send_alert(
|
||||
"WARNING",
|
||||
f"Pool {pool}: {cap}% voll",
|
||||
f"Pool <code>{pool}</code> ist zu <b>{cap}%</b> voll.",
|
||||
key_cap,
|
||||
)
|
||||
else:
|
||||
if self.alerter.clear_alert(key_cap):
|
||||
self.alerter.send_alert(
|
||||
"INFO",
|
||||
f"Pool {pool}: Füllstand OK",
|
||||
f"Pool <code>{pool}</code> ist wieder unter "
|
||||
f"{self.config.zfs.warn_pct}% ✅",
|
||||
)
|
||||
|
||||
self.log.info("Pool %s: health=%s cap=%d%%", pool, health, cap)
|
||||
|
||||
def _pool_status_summary(self, pool: str) -> str:
|
||||
out = self._run(["zpool", "status", pool], timeout=self.config.zfs.timeout) or ""
|
||||
lines = [
|
||||
l for l in out.splitlines()
|
||||
if re.search(r"state:|status:|action:|DEGRADED|FAULTED|REMOVED|UNAVAIL", l)
|
||||
]
|
||||
return "\n".join(lines[:10])
|
||||
|
||||
def _parse_io_errors(self, pool: str) -> str:
|
||||
out = self._run(
|
||||
["zpool", "status", "-P", pool],
|
||||
timeout=self.config.zfs.timeout,
|
||||
) or ""
|
||||
lines = []
|
||||
for line in out.splitlines():
|
||||
m = re.match(r"\s+(/dev/\S+)\s+\S+\s+(\d+)\s+(\d+)\s+(\d+)", line)
|
||||
if m:
|
||||
dev, r, w, c = m.group(1), int(m.group(2)), int(m.group(3)), int(m.group(4))
|
||||
if r or w or c:
|
||||
lines.append(f"{dev} R:{r} W:{w} C:{c}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── Device Normalisierung ─────────────────────────────────────
|
||||
|
||||
def _normalize_device(dev: str) -> str:
|
||||
import os
|
||||
resolved = os.path.realpath(dev)
|
||||
# nvme0n1p1 → nvme0n1
|
||||
resolved = re.sub(r"(nvme\d+n\d+)p\d+$", r"\1", resolved)
|
||||
# sda1 → sda
|
||||
resolved = re.sub(r"(sd[a-z]+)\d+$", r"\1", resolved)
|
||||
return resolved
|
||||
Reference in New Issue
Block a user