ai-contract-platform/yuqei-infra/tests/test_observability_capacity.py

from __future__ import annotations

import importlib.util
import json
import sys
from pathlib import Path
from types import ModuleType, SimpleNamespace

import pytest


REPO_ROOT = Path(__file__).resolve().parents[2]
GIB = 1024**3


def load_script(name: str, relative_path: str) -> ModuleType:
    path = REPO_ROOT / relative_path
    spec = importlib.util.spec_from_file_location(name, path)
    assert spec and spec.loader
    module = importlib.util.module_from_spec(spec)
    sys.modules[name] = module
    spec.loader.exec_module(module)
    return module


health = load_script(
    "yuqei_observability_capacity",
    "yuqei-infra/observability/check-production-health.py",
)
preflight = load_script(
    "yuqei_observability_preflight",
    "yuqei-infra/deploy/preflight-production.py",
)


@pytest.mark.parametrize(
    ("used_gib", "free_gib", "expected"),
    [
        (79, 21, "pass"),
        (80, 20, "warning"),
        (89, 9, "fail"),
        (90, 10, "fail"),
    ],
)
def test_capacity_check_applies_percent_ceiling_and_free_space_floor(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
    used_gib: int,
    free_gib: int,
    expected: str,
) -> None:
    total_gib = used_gib + free_gib
    monkeypatch.setattr(
        health.shutil,
        "disk_usage",
        lambda _path: SimpleNamespace(total=total_gib * GIB, used=used_gib * GIB, free=free_gib * GIB),
    )

    result = health.check_disk(
        tmp_path,
        key="disk_root",
        warning_percent=80,
        critical_percent=90,
        warning_free_gib=20,
        critical_free_gib=10,
    )

    assert result.status == expected
    assert "warning_percent=80.00" in result.message
    assert "critical_free_gib=10.00" in result.message


def test_required_capacity_mount_fails_closed(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    monkeypatch.setattr(health.os.path, "ismount", lambda _path: False)

    result = health.check_disk(
        tmp_path,
        key="disk_backup",
        warning_percent=80,
        critical_percent=90,
        warning_free_gib=20,
        critical_free_gib=10,
        require_mount=True,
    )

    assert result.status == "fail"
    assert "not a separate mount" in result.message


def test_capacity_percent_excludes_filesystem_reserved_blocks(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    # total includes a reserved-block pool that is unavailable to the service.
    monkeypatch.setattr(
        health.shutil,
        "disk_usage",
        lambda _path: SimpleNamespace(total=100 * GIB, used=80 * GIB, free=10 * GIB),
    )

    result = health.check_disk(
        tmp_path,
        key="disk_root",
        warning_percent=85,
        critical_percent=90,
        warning_free_gib=5,
        critical_free_gib=2,
    )

    assert result.status == "warning"
    assert "used_percent=88.89" in result.message


@pytest.mark.parametrize(
    ("warning_percent", "critical_percent", "warning_free_gib", "critical_free_gib"),
    [
        (90, 80, 20, 10),
        (80, 101, 20, 10),
        (80, 90, 10, 20),
        (-1, 90, 20, 10),
    ],
)
def test_invalid_capacity_threshold_order_fails_closed(
    tmp_path: Path,
    warning_percent: float,
    critical_percent: float,
    warning_free_gib: float,
    critical_free_gib: float,
) -> None:
    result = health.check_disk(
        tmp_path,
        key="disk_root",
        warning_percent=warning_percent,
        critical_percent=critical_percent,
        warning_free_gib=warning_free_gib,
        critical_free_gib=critical_free_gib,
    )

    assert result.status == "fail"
    assert "thresholds must satisfy" in result.message


def _main_args(tmp_path: Path) -> SimpleNamespace:
    return SimpleNamespace(
        edge_base_url="https://api.contracts.example.com",
        contract_api_base="http://127.0.0.1:8100",
        ai_api_base="http://127.0.0.1:8101",
        contract_proxy_key_file=None,
        ai_service_key_file=None,
        tenant_id="default",
        backup_root=tmp_path,
        backup_max_age_hours=30,
        root_disk_path=tmp_path,
        root_disk_warning_percent=80,
        root_disk_critical_percent=90,
        root_disk_warning_free_gib=20,
        root_disk_critical_free_gib=10,
        home_disk_path=tmp_path,
        home_disk_warning_percent=80,
        home_disk_critical_percent=90,
        home_disk_warning_free_gib=20,
        home_disk_critical_free_gib=10,
        home_require_separate_mount=True,
        backup_disk_path=tmp_path,
        backup_disk_warning_percent=80,
        backup_disk_critical_percent=90,
        backup_disk_warning_free_gib=20,
        backup_disk_critical_free_gib=10,
        backup_require_separate_mount=True,
        timeout_seconds=1,
        webhook_url_file=tmp_path / "webhook-url",
        artifact_dir=tmp_path / "artifact",
    )


def _mock_non_capacity_checks(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(
        health,
        "check_json_endpoint",
        lambda key, *_args, **_kwargs: (health.HealthCheck(key, "pass", "ok"), {"status": "ok"}),
    )
    monkeypatch.setattr(
        health,
        "check_schema",
        lambda key, *_args, **_kwargs: (
            health.HealthCheck(key, "pass", "ok"),
            {"checks": [{"name": "provider_runtime", "status": "ok"}]},
        ),
    )
    monkeypatch.setattr(
        health,
        "check_backup_freshness",
        lambda *_args, **_kwargs: health.HealthCheck("backup_freshness", "pass", "ok"),
    )
    monkeypatch.setattr(
        health,
        "check_disk",
        lambda **policy: health.HealthCheck(
            policy["key"],
            "warning" if policy["key"] == "disk_home" else "pass",
            "simulated",
        ),
    )


def test_warning_dispatches_webhook_and_records_acceptance_artifact(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    args = _main_args(tmp_path)
    delivered: list[str] = []
    monkeypatch.setattr(health, "_parse_args", lambda: args)
    _mock_non_capacity_checks(monkeypatch)
    monkeypatch.setattr(
        health,
        "send_webhook",
        lambda _path, summary, **_kwargs: delivered.append(summary["status"]),
    )

    exit_code = health.main()
    summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))

    assert exit_code == 0
    assert delivered == ["warning"]
    assert summary["warning_checks"] == ["disk_home"]
    assert [item["key"] for item in summary["capacity_policy"]] == [
        "disk_root",
        "disk_home",
        "disk_backup",
    ]
    assert summary["alert_delivery"] == {
        "configured": True,
        "required": True,
        "attempted": True,
        "delivered": True,
        "error": None,
    }


def test_webhook_delivery_failure_is_visible_to_systemd(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    args = _main_args(tmp_path)
    monkeypatch.setattr(health, "_parse_args", lambda: args)
    _mock_non_capacity_checks(monkeypatch)

    def fail_delivery(*_args: object, **_kwargs: object) -> None:
        raise TimeoutError("simulated")

    monkeypatch.setattr(health, "send_webhook", fail_delivery)

    exit_code = health.main()
    summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))

    assert exit_code == 2
    assert summary["alert_delivery"]["attempted"] is True
    assert summary["alert_delivery"]["delivered"] is False
    assert summary["alert_delivery"]["error"] == "TimeoutError"


def test_failed_capacity_check_dispatches_webhook_and_exits_one(
    monkeypatch: pytest.MonkeyPatch,
    tmp_path: Path,
) -> None:
    args = _main_args(tmp_path)
    delivered: list[str] = []
    monkeypatch.setattr(health, "_parse_args", lambda: args)
    _mock_non_capacity_checks(monkeypatch)
    monkeypatch.setattr(
        health,
        "check_disk",
        lambda **policy: health.HealthCheck(
            policy["key"],
            "fail" if policy["key"] == "disk_backup" else "pass",
            "simulated",
        ),
    )
    monkeypatch.setattr(
        health,
        "send_webhook",
        lambda _path, summary, **_kwargs: delivered.append(summary["status"]),
    )

    exit_code = health.main()

    assert exit_code == 1
    assert delivered == ["fail"]


def test_healthy_run_does_not_dispatch_webhook(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    args = _main_args(tmp_path)
    monkeypatch.setattr(health, "_parse_args", lambda: args)
    _mock_non_capacity_checks(monkeypatch)
    monkeypatch.setattr(
        health,
        "check_disk",
        lambda **policy: health.HealthCheck(policy["key"], "pass", "simulated"),
    )

    def unexpected_delivery(*_args: object, **_kwargs: object) -> None:
        pytest.fail("healthy runs must not dispatch alerts")

    monkeypatch.setattr(health, "send_webhook", unexpected_delivery)

    exit_code = health.main()
    summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))

    assert exit_code == 0
    assert summary["alert_delivery"]["required"] is False
    assert summary["alert_delivery"]["attempted"] is False


def test_preflight_rejects_missing_or_inverted_capacity_thresholds() -> None:
    values = {
        "ROOT_DISK_WARNING_PERCENT": "80",
        "ROOT_DISK_CRITICAL_PERCENT": "90",
        "ROOT_DISK_WARNING_FREE_GIB": "20",
        "ROOT_DISK_CRITICAL_FREE_GIB": "10",
        "HOME_DISK_WARNING_PERCENT": "95",
        "HOME_DISK_CRITICAL_PERCENT": "90",
        "HOME_DISK_WARNING_FREE_GIB": "20",
        "HOME_DISK_CRITICAL_FREE_GIB": "10",
        "BACKUP_DISK_WARNING_PERCENT": "80",
        "BACKUP_DISK_CRITICAL_PERCENT": "90",
        "BACKUP_DISK_WARNING_FREE_GIB": "20",
    }

    result = preflight._validate_capacity_thresholds(values)

    assert result.status == "fail"
    assert "HOME_DISK" in result.message
    assert "BACKUP_DISK" in result.message


def test_systemd_timer_and_service_cover_all_capacity_paths() -> None:
    unit = (REPO_ROOT / "yuqei-infra/observability/yuqei-production-health.service").read_text(
        encoding="utf-8"
    )
    timer = (REPO_ROOT / "yuqei-infra/observability/yuqei-production-health.timer").read_text(
        encoding="utf-8"
    )

    assert "OnUnitActiveSec=1min" in timer
    assert "Persistent=true" in timer
    assert "ProtectHome=read-only" in unit
    assert "--root-disk-path /" in unit
    assert "--home-disk-path /home" in unit
    assert "--home-require-separate-mount" in unit
    assert "--backup-disk-path ${BACKUP_HOST_DIR}" in unit
    assert "--backup-require-separate-mount" in unit