264 lines
9.7 KiB
Python
264 lines
9.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Compose and send the backup status notification email.
|
|
|
|
This module builds a structured plain-text email and delivers it using
|
|
``ns8-sendmail``, the NS8 system mail sender available on the cluster leader.
|
|
|
|
Why ``ns8-sendmail`` instead of smtplib
|
|
-----------------------------------------
|
|
NS8 manages SMTP relay configuration centrally in the cluster. Using
|
|
``ns8-sendmail`` means the email is sent through whatever relay the
|
|
administrator has configured (internal Postfix, external SMTP relay, etc.)
|
|
without duplicating that configuration in this tool. Direct smtplib calls
|
|
would require re-reading and re-implementing NS8's relay settings.
|
|
|
|
Email structure
|
|
---------------
|
|
The email is plain text with three sections:
|
|
|
|
1. SUMMARY - Overall outcome (SUCCESS / PARTIAL / REPO_FAILURE),
|
|
timestamp, and list of evaluated backup plan IDs.
|
|
|
|
2. MODULE STATUS TABLE - One row per backup module showing module_id,
|
|
backup_id, result, and any error message.
|
|
Absent on SUCCESS to keep the email concise.
|
|
|
|
3. REPOSITORY DIAGNOSTICS - Per-destination restic check results.
|
|
Absent on SUCCESS (repo check is skipped).
|
|
|
|
Subject line format
|
|
--------------------
|
|
[ns8-backup] SUCCESS - all 4 modules backed up successfully
|
|
[ns8-backup] PARTIAL - 1/4 modules failed
|
|
[ns8-backup] REPO_FAILURE - no backup status found (possible repo issue)
|
|
"""
|
|
|
|
import logging
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
from typing import Optional
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Subject builder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_subject(correlation: dict) -> str:
|
|
"""Build a concise email subject line from the correlation outcome.
|
|
|
|
Args:
|
|
correlation: Output dict from ``correlate_backup_status()``.
|
|
|
|
Returns:
|
|
Subject string starting with ``[ns8-backup]``.
|
|
"""
|
|
outcome = correlation["outcome"]
|
|
total = correlation["total"]
|
|
failed = correlation["failed"]
|
|
succeeded = correlation["succeeded"]
|
|
|
|
if outcome == "SUCCESS":
|
|
return f"[ns8-backup] SUCCESS - all {total} module(s) backed up successfully"
|
|
elif outcome == "PARTIAL":
|
|
return f"[ns8-backup] PARTIAL - {failed}/{total} module(s) failed"
|
|
else:
|
|
note = correlation.get("note", "")
|
|
return f"[ns8-backup] REPO_FAILURE - {note or 'possible repository issue'}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Body builder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _build_body(
|
|
alerts: list,
|
|
correlation: dict,
|
|
repo_status: Optional[dict],
|
|
) -> str:
|
|
"""Build the plain-text email body.
|
|
|
|
Args:
|
|
alerts: Raw Alertmanager alert list from the webhook payload.
|
|
correlation: Output dict from ``correlate_backup_status()``.
|
|
repo_status: Output dict from ``check_repositories()``, or None if
|
|
the repo check was skipped (i.e. outcome == SUCCESS).
|
|
|
|
Returns:
|
|
Multi-line string suitable for direct use as the email body.
|
|
"""
|
|
outcome = correlation["outcome"]
|
|
backup_ids = correlation.get("backup_ids", [])
|
|
modules = correlation.get("modules", [])
|
|
failed_mods = correlation.get("failed_modules", [])
|
|
note = correlation.get("note", "")
|
|
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
|
|
lines = []
|
|
|
|
# ------------------------------------------------------------------
|
|
# Section 1: SUMMARY
|
|
# ------------------------------------------------------------------
|
|
lines.append("=" * 60)
|
|
lines.append("NS8 BACKUP MONITOR - STATUS REPORT")
|
|
lines.append("=" * 60)
|
|
lines.append(f"Timestamp : {now}")
|
|
lines.append(f"Outcome : {outcome}")
|
|
lines.append(f"Plan IDs : {', '.join(backup_ids) if backup_ids else 'unknown'}")
|
|
lines.append(f"Total : {correlation['total']} module(s)")
|
|
lines.append(f"Succeeded : {correlation['succeeded']}")
|
|
lines.append(f"Failed : {correlation['failed']}")
|
|
if note:
|
|
lines.append(f"Note : {note}")
|
|
lines.append("")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Section 2: MODULE STATUS TABLE
|
|
# Shown on PARTIAL and REPO_FAILURE to list which modules failed.
|
|
# Omitted on SUCCESS to keep the email concise.
|
|
# ------------------------------------------------------------------
|
|
if outcome != "SUCCESS" and modules:
|
|
lines.append("-" * 60)
|
|
lines.append("MODULE STATUS")
|
|
lines.append("-" * 60)
|
|
# Fixed-width columns for plain-text readability.
|
|
header = f"{'Module':<20} {'Plan':>4} {'Result':<10} Error"
|
|
lines.append(header)
|
|
lines.append("-" * 60)
|
|
for m in modules:
|
|
result_str = m["result"].upper()
|
|
error_str = m["error"][:60] if m["error"] else "-"
|
|
lines.append(
|
|
f"{m['module_id']:<20} {m['backup_id']:>4} {result_str:<10} {error_str}"
|
|
)
|
|
lines.append("")
|
|
|
|
# ------------------------------------------------------------------
|
|
# Section 3: REPOSITORY DIAGNOSTICS
|
|
# Shown only when the repo check was run (non-SUCCESS outcomes).
|
|
# The repo check is skipped on SUCCESS to avoid unnecessary restic
|
|
# network calls, so repo_status is None in that case.
|
|
# ------------------------------------------------------------------
|
|
if repo_status:
|
|
lines.append("-" * 60)
|
|
lines.append("REPOSITORY DIAGNOSTICS")
|
|
lines.append("-" * 60)
|
|
lines.append(f"Summary: {repo_status['summary']}")
|
|
lines.append("")
|
|
for dest in repo_status.get("destinations", []):
|
|
lines.append(f" Repo {dest['repo_id']}: {dest['status']}")
|
|
if dest.get("error"):
|
|
# Indent error detail under the repo line.
|
|
for err_line in dest["error"].splitlines()[:3]:
|
|
lines.append(f" {err_line}")
|
|
lines.append("")
|
|
|
|
lines.append("-" * 60)
|
|
lines.append("Sent by ns8-backup-monitor")
|
|
lines.append("https://github.com/lelekaos/ns8-backup-monitor")
|
|
lines.append("-" * 60)
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Delivery
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _send_via_ns8_sendmail(
|
|
config: dict,
|
|
subject: str,
|
|
body: str,
|
|
) -> bool:
|
|
"""Deliver the email through ``ns8-sendmail``.
|
|
|
|
``ns8-sendmail`` reads SMTP relay settings from the NS8 cluster
|
|
configuration, so no SMTP credentials are needed here.
|
|
|
|
Args:
|
|
config: Parsed configuration dictionary. Reads:
|
|
``notification.mail_to`` - recipient address or list.
|
|
``notification.mail_from`` - sender address.
|
|
subject: Email subject string.
|
|
body: Plain-text email body.
|
|
|
|
Returns:
|
|
True if ``ns8-sendmail`` exited with code 0, False otherwise.
|
|
"""
|
|
mail_to = config.get("notification", {}).get("mail_to", "")
|
|
mail_from = config.get("notification", {}).get(
|
|
"mail_from", "ns8-backup-monitor@localhost"
|
|
)
|
|
|
|
# ``mail_to`` may be a string or a list in the YAML config.
|
|
if isinstance(mail_to, list):
|
|
recipients = mail_to
|
|
else:
|
|
# Split on comma for inline multi-recipient strings.
|
|
recipients = [r.strip() for r in mail_to.split(",") if r.strip()]
|
|
|
|
if not recipients:
|
|
log.error("No mail_to recipients configured; skipping notification")
|
|
return False
|
|
|
|
success = True
|
|
for recipient in recipients:
|
|
try:
|
|
result = subprocess.run(
|
|
["ns8-sendmail", "--from", mail_from, "--to", recipient,
|
|
"--subject", subject],
|
|
input=body,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
if result.returncode == 0:
|
|
log.info("Notification sent to %s", recipient)
|
|
else:
|
|
log.error(
|
|
"ns8-sendmail failed for %s (exit %d): %s",
|
|
recipient, result.returncode, result.stderr.strip()
|
|
)
|
|
success = False
|
|
except FileNotFoundError:
|
|
log.error(
|
|
"ns8-sendmail not found in PATH - "
|
|
"ensure the NS8 mail module is installed on the leader"
|
|
)
|
|
success = False
|
|
except subprocess.TimeoutExpired:
|
|
log.error("ns8-sendmail timed out for %s", recipient)
|
|
success = False
|
|
|
|
return success
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main entry point
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def send_notification(
|
|
config: dict,
|
|
alerts: list,
|
|
correlation: dict,
|
|
repo_status: Optional[dict],
|
|
) -> bool:
|
|
"""Build and send the backup status notification email.
|
|
|
|
Args:
|
|
config: Parsed configuration dictionary.
|
|
alerts: Raw Alertmanager alert list from the webhook payload.
|
|
correlation: Output dict from ``correlate_backup_status()``.
|
|
repo_status: Output dict from ``check_repositories()``, or None
|
|
when the outcome is SUCCESS (repo check skipped).
|
|
|
|
Returns:
|
|
True if the email was delivered successfully, False otherwise.
|
|
"""
|
|
subject = _build_subject(correlation)
|
|
body = _build_body(alerts, correlation, repo_status)
|
|
|
|
log.info("Sending notification: %s", subject)
|
|
return _send_via_ns8_sendmail(config, subject, body)
|