ns8-backup-monitor/ns8_backup_monitor/notifier.py

#!/usr/bin/env python3
"""Compose and send the backup status notification email.

This module builds a structured plain-text email and delivers it using
``ns8-sendmail``, the NS8 system mail sender available on the cluster leader.

Why ``ns8-sendmail`` instead of smtplib
-----------------------------------------
NS8 manages SMTP relay configuration centrally in the cluster. Using
``ns8-sendmail`` means the email is sent through whatever relay the
administrator has configured (internal Postfix, external SMTP relay, etc.)
without duplicating that configuration in this tool. Direct smtplib calls
would require re-reading and re-implementing NS8's relay settings.

Email structure
---------------
The email is plain text with three sections:

  1. SUMMARY     - Overall outcome (SUCCESS / PARTIAL / REPO_FAILURE),
                   timestamp, and list of evaluated backup plan IDs.

  2. MODULE STATUS TABLE - One row per backup module showing module_id,
                           backup_id, result, and any error message.
                           Absent on SUCCESS to keep the email concise.

  3. REPOSITORY DIAGNOSTICS - Per-destination restic check results.
                               Absent on SUCCESS (repo check is skipped).

Subject line format
--------------------
    [ns8-backup] SUCCESS - all 4 modules backed up successfully
    [ns8-backup] PARTIAL - 1/4 modules failed
    [ns8-backup] REPO_FAILURE - no backup status found (possible repo issue)
"""

import logging
import subprocess
from datetime import datetime, timezone
from typing import Optional

log = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Subject builder
# ---------------------------------------------------------------------------

def _build_subject(correlation: dict) -> str:
    """Build a concise email subject line from the correlation outcome.

    Args:
        correlation: Output dict from ``correlate_backup_status()``.

    Returns:
        Subject string starting with ``[ns8-backup]``.
    """
    outcome   = correlation["outcome"]
    total     = correlation["total"]
    failed    = correlation["failed"]
    succeeded = correlation["succeeded"]

    if outcome == "SUCCESS":
        return f"[ns8-backup] SUCCESS - all {total} module(s) backed up successfully"
    elif outcome == "PARTIAL":
        return f"[ns8-backup] PARTIAL - {failed}/{total} module(s) failed"
    else:
        note = correlation.get("note", "")
        return f"[ns8-backup] REPO_FAILURE - {note or 'possible repository issue'}"


# ---------------------------------------------------------------------------
# Body builder
# ---------------------------------------------------------------------------

def _build_body(
    alerts: list,
    correlation: dict,
    repo_status: Optional[dict],
) -> str:
    """Build the plain-text email body.

    Args:
        alerts:      Raw Alertmanager alert list from the webhook payload.
        correlation: Output dict from ``correlate_backup_status()``.
        repo_status: Output dict from ``check_repositories()``, or None if
                     the repo check was skipped (i.e. outcome == SUCCESS).

    Returns:
        Multi-line string suitable for direct use as the email body.
    """
    outcome    = correlation["outcome"]
    backup_ids = correlation.get("backup_ids", [])
    modules    = correlation.get("modules", [])
    failed_mods = correlation.get("failed_modules", [])
    note       = correlation.get("note", "")
    now        = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")

    lines = []

    # ------------------------------------------------------------------
    # Section 1: SUMMARY
    # ------------------------------------------------------------------
    lines.append("=" * 60)
    lines.append("NS8 BACKUP MONITOR - STATUS REPORT")
    lines.append("=" * 60)
    lines.append(f"Timestamp : {now}")
    lines.append(f"Outcome   : {outcome}")
    lines.append(f"Plan IDs  : {', '.join(backup_ids) if backup_ids else 'unknown'}")
    lines.append(f"Total     : {correlation['total']} module(s)")
    lines.append(f"Succeeded : {correlation['succeeded']}")
    lines.append(f"Failed    : {correlation['failed']}")
    if note:
        lines.append(f"Note      : {note}")
    lines.append("")

    # ------------------------------------------------------------------
    # Section 2: MODULE STATUS TABLE
    # Shown on PARTIAL and REPO_FAILURE to list which modules failed.
    # Omitted on SUCCESS to keep the email concise.
    # ------------------------------------------------------------------
    if outcome != "SUCCESS" and modules:
        lines.append("-" * 60)
        lines.append("MODULE STATUS")
        lines.append("-" * 60)
        # Fixed-width columns for plain-text readability.
        header = f"{'Module':<20} {'Plan':>4}  {'Result':<10}  Error"
        lines.append(header)
        lines.append("-" * 60)
        for m in modules:
            result_str = m["result"].upper()
            error_str  = m["error"][:60] if m["error"] else "-"
            lines.append(
                f"{m['module_id']:<20} {m['backup_id']:>4}  {result_str:<10}  {error_str}"
            )
        lines.append("")

    # ------------------------------------------------------------------
    # Section 3: REPOSITORY DIAGNOSTICS
    # Shown only when the repo check was run (non-SUCCESS outcomes).
    # The repo check is skipped on SUCCESS to avoid unnecessary restic
    # network calls, so repo_status is None in that case.
    # ------------------------------------------------------------------
    if repo_status:
        lines.append("-" * 60)
        lines.append("REPOSITORY DIAGNOSTICS")
        lines.append("-" * 60)
        lines.append(f"Summary: {repo_status['summary']}")
        lines.append("")
        for dest in repo_status.get("destinations", []):
            lines.append(f"  Repo {dest['repo_id']}: {dest['status']}")
            if dest.get("error"):
                # Indent error detail under the repo line.
                for err_line in dest["error"].splitlines()[:3]:
                    lines.append(f"    {err_line}")
        lines.append("")

    lines.append("-" * 60)
    lines.append("Sent by ns8-backup-monitor")
    lines.append("https://github.com/lelekaos/ns8-backup-monitor")
    lines.append("-" * 60)

    return "\n".join(lines)


# ---------------------------------------------------------------------------
# Delivery
# ---------------------------------------------------------------------------

def _send_via_ns8_sendmail(
    config: dict,
    subject: str,
    body: str,
) -> bool:
    """Deliver the email through ``ns8-sendmail``.

    ``ns8-sendmail`` reads SMTP relay settings from the NS8 cluster
    configuration, so no SMTP credentials are needed here.

    Args:
        config:  Parsed configuration dictionary. Reads:
                   ``notification.mail_to``   - recipient address or list.
                   ``notification.mail_from`` - sender address.
        subject: Email subject string.
        body:    Plain-text email body.

    Returns:
        True if ``ns8-sendmail`` exited with code 0, False otherwise.
    """
    mail_to   = config.get("notification", {}).get("mail_to", "")
    mail_from = config.get("notification", {}).get(
        "mail_from", "ns8-backup-monitor@localhost"
    )

    # ``mail_to`` may be a string or a list in the YAML config.
    if isinstance(mail_to, list):
        recipients = mail_to
    else:
        # Split on comma for inline multi-recipient strings.
        recipients = [r.strip() for r in mail_to.split(",") if r.strip()]

    if not recipients:
        log.error("No mail_to recipients configured; skipping notification")
        return False

    success = True
    for recipient in recipients:
        try:
            result = subprocess.run(
                ["ns8-sendmail", "--from", mail_from, "--to", recipient,
                 "--subject", subject],
                input=body,
                capture_output=True,
                text=True,
                timeout=30,
            )
            if result.returncode == 0:
                log.info("Notification sent to %s", recipient)
            else:
                log.error(
                    "ns8-sendmail failed for %s (exit %d): %s",
                    recipient, result.returncode, result.stderr.strip()
                )
                success = False
        except FileNotFoundError:
            log.error(
                "ns8-sendmail not found in PATH - "
                "ensure the NS8 mail module is installed on the leader"
            )
            success = False
        except subprocess.TimeoutExpired:
            log.error("ns8-sendmail timed out for %s", recipient)
            success = False

    return success


# ---------------------------------------------------------------------------
# Main entry point
# ---------------------------------------------------------------------------

def send_notification(
    config: dict,
    alerts: list,
    correlation: dict,
    repo_status: Optional[dict],
) -> bool:
    """Build and send the backup status notification email.

    Args:
        config:      Parsed configuration dictionary.
        alerts:      Raw Alertmanager alert list from the webhook payload.
        correlation: Output dict from ``correlate_backup_status()``.
        repo_status: Output dict from ``check_repositories()``, or None
                     when the outcome is SUCCESS (repo check skipped).

    Returns:
        True if the email was delivered successfully, False otherwise.
    """
    subject = _build_subject(correlation)
    body    = _build_body(alerts, correlation, repo_status)

    log.info("Sending notification: %s", subject)
    return _send_via_ns8_sendmail(config, subject, body)