yuqei
ce7e040e98
Build and Test Quality Gate / Contract Web typecheck and build (pull_request) Successful in 7m28s
Build and Test Quality Gate / Python tests and Contract API migration (pull_request) Failing after 26m12s
Security Quality Gate / npm audit, Bandit, pip-audit, detect-secrets (pull_request) Successful in 6m33s
483 lines
15 KiB
Python
483 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from types import ModuleType, SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
GIB = 1024**3
|
|
|
|
|
|
def load_script(name: str, relative_path: str) -> ModuleType:
|
|
path = REPO_ROOT / relative_path
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
health = load_script(
|
|
"yuqei_observability_capacity",
|
|
"yuqei-infra/observability/check-production-health.py",
|
|
)
|
|
preflight = load_script(
|
|
"yuqei_observability_preflight",
|
|
"yuqei-infra/deploy/preflight-production.py",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("used_gib", "free_gib", "expected"),
|
|
[
|
|
(79, 21, "pass"),
|
|
(80, 20, "warning"),
|
|
(89, 9, "fail"),
|
|
(90, 10, "fail"),
|
|
],
|
|
)
|
|
def test_capacity_check_applies_percent_ceiling_and_free_space_floor(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
used_gib: int,
|
|
free_gib: int,
|
|
expected: str,
|
|
) -> None:
|
|
total_gib = used_gib + free_gib
|
|
monkeypatch.setattr(
|
|
health.shutil,
|
|
"disk_usage",
|
|
lambda _path: SimpleNamespace(total=total_gib * GIB, used=used_gib * GIB, free=free_gib * GIB),
|
|
)
|
|
|
|
result = health.check_disk(
|
|
tmp_path,
|
|
key="disk_root",
|
|
warning_percent=80,
|
|
critical_percent=90,
|
|
warning_free_gib=20,
|
|
critical_free_gib=10,
|
|
)
|
|
|
|
assert result.status == expected
|
|
assert "warning_percent=80.00" in result.message
|
|
assert "critical_free_gib=10.00" in result.message
|
|
|
|
|
|
def test_required_capacity_mount_fails_closed(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
monkeypatch.setattr(health.os.path, "ismount", lambda _path: False)
|
|
|
|
result = health.check_disk(
|
|
tmp_path,
|
|
key="disk_backup",
|
|
warning_percent=80,
|
|
critical_percent=90,
|
|
warning_free_gib=20,
|
|
critical_free_gib=10,
|
|
require_mount=True,
|
|
)
|
|
|
|
assert result.status == "fail"
|
|
assert "not a separate mount" in result.message
|
|
|
|
|
|
def test_capacity_percent_excludes_filesystem_reserved_blocks(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
# total includes a reserved-block pool that is unavailable to the service.
|
|
monkeypatch.setattr(
|
|
health.shutil,
|
|
"disk_usage",
|
|
lambda _path: SimpleNamespace(total=100 * GIB, used=80 * GIB, free=10 * GIB),
|
|
)
|
|
|
|
result = health.check_disk(
|
|
tmp_path,
|
|
key="disk_root",
|
|
warning_percent=85,
|
|
critical_percent=90,
|
|
warning_free_gib=5,
|
|
critical_free_gib=2,
|
|
)
|
|
|
|
assert result.status == "warning"
|
|
assert "used_percent=88.89" in result.message
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("warning_percent", "critical_percent", "warning_free_gib", "critical_free_gib"),
|
|
[
|
|
(90, 80, 20, 10),
|
|
(80, 101, 20, 10),
|
|
(80, 90, 10, 20),
|
|
(-1, 90, 20, 10),
|
|
],
|
|
)
|
|
def test_invalid_capacity_threshold_order_fails_closed(
|
|
tmp_path: Path,
|
|
warning_percent: float,
|
|
critical_percent: float,
|
|
warning_free_gib: float,
|
|
critical_free_gib: float,
|
|
) -> None:
|
|
result = health.check_disk(
|
|
tmp_path,
|
|
key="disk_root",
|
|
warning_percent=warning_percent,
|
|
critical_percent=critical_percent,
|
|
warning_free_gib=warning_free_gib,
|
|
critical_free_gib=critical_free_gib,
|
|
)
|
|
|
|
assert result.status == "fail"
|
|
assert "thresholds must satisfy" in result.message
|
|
|
|
|
|
def _main_args(tmp_path: Path) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
edge_base_url="https://api.contracts.example.com",
|
|
contract_api_base="http://127.0.0.1:8100",
|
|
ai_api_base="http://127.0.0.1:8101",
|
|
contract_proxy_key_file=None,
|
|
ai_service_key_file=None,
|
|
tenant_id="default",
|
|
backup_root=tmp_path,
|
|
backup_max_age_hours=30,
|
|
root_disk_path=tmp_path,
|
|
root_disk_warning_percent=80,
|
|
root_disk_critical_percent=90,
|
|
root_disk_warning_free_gib=20,
|
|
root_disk_critical_free_gib=10,
|
|
home_disk_path=tmp_path,
|
|
home_disk_warning_percent=80,
|
|
home_disk_critical_percent=90,
|
|
home_disk_warning_free_gib=20,
|
|
home_disk_critical_free_gib=10,
|
|
home_require_separate_mount=True,
|
|
backup_disk_path=tmp_path,
|
|
backup_disk_warning_percent=80,
|
|
backup_disk_critical_percent=90,
|
|
backup_disk_warning_free_gib=20,
|
|
backup_disk_critical_free_gib=10,
|
|
backup_require_separate_mount=True,
|
|
timeout_seconds=1,
|
|
webhook_url_file=tmp_path / "webhook-url",
|
|
alert_state_file=None,
|
|
alert_resend_seconds=3600,
|
|
artifact_dir=tmp_path / "artifact",
|
|
)
|
|
|
|
|
|
def _mock_non_capacity_checks(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_json_endpoint",
|
|
lambda key, *_args, **_kwargs: (health.HealthCheck(key, "pass", "ok"), {"status": "ok"}),
|
|
)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_schema",
|
|
lambda key, *_args, **_kwargs: (
|
|
health.HealthCheck(key, "pass", "ok"),
|
|
{"checks": [{"name": "provider_runtime", "status": "ok"}]},
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_backup_freshness",
|
|
lambda *_args, **_kwargs: health.HealthCheck("backup_freshness", "pass", "ok"),
|
|
)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_disk",
|
|
lambda **policy: health.HealthCheck(
|
|
policy["key"],
|
|
"warning" if policy["key"] == "disk_home" else "pass",
|
|
"simulated",
|
|
),
|
|
)
|
|
|
|
|
|
def test_warning_dispatches_webhook_and_records_acceptance_artifact(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
delivered: list[str] = []
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"send_webhook",
|
|
lambda _path, summary, **_kwargs: delivered.append(summary["status"]),
|
|
)
|
|
|
|
exit_code = health.main()
|
|
summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))
|
|
|
|
assert exit_code == 0
|
|
assert delivered == ["warning"]
|
|
assert summary["warning_checks"] == ["disk_home"]
|
|
assert [item["key"] for item in summary["capacity_policy"]] == [
|
|
"disk_root",
|
|
"disk_home",
|
|
"disk_backup",
|
|
]
|
|
assert summary["alert_delivery"] == {
|
|
"configured": True,
|
|
"required": True,
|
|
"attempted": True,
|
|
"delivered": True,
|
|
"error": None,
|
|
"receiver_event_id": None,
|
|
"receiver_received_at": None,
|
|
"receiver_body_sha256": None,
|
|
"suppressed": False,
|
|
"decision": "deduplication_not_configured",
|
|
}
|
|
|
|
|
|
def test_webhook_receipt_is_recorded_in_local_acceptance_artifact(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"send_webhook",
|
|
lambda *_args, **_kwargs: {
|
|
"event_id": "yuqei-alert-0123456789abcdef",
|
|
"received_at": "2026-08-12T02:00:00+00:00",
|
|
"body_sha256": "a" * 64,
|
|
},
|
|
)
|
|
|
|
assert health.main() == 0
|
|
summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))
|
|
assert summary["alert_delivery"]["receiver_event_id"] == "yuqei-alert-0123456789abcdef"
|
|
assert summary["alert_delivery"]["receiver_received_at"] == "2026-08-12T02:00:00+00:00"
|
|
assert summary["alert_delivery"]["receiver_body_sha256"] == "a" * 64
|
|
|
|
|
|
def test_webhook_delivery_failure_is_visible_to_systemd(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
|
|
def fail_delivery(*_args: object, **_kwargs: object) -> None:
|
|
raise TimeoutError("simulated")
|
|
|
|
monkeypatch.setattr(health, "send_webhook", fail_delivery)
|
|
|
|
exit_code = health.main()
|
|
summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))
|
|
|
|
assert exit_code == 2
|
|
assert summary["alert_delivery"]["attempted"] is True
|
|
assert summary["alert_delivery"]["delivered"] is False
|
|
assert summary["alert_delivery"]["error"] == "TimeoutError"
|
|
|
|
|
|
def test_duplicate_alert_is_suppressed_until_resend_window(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
args.alert_state_file = tmp_path / "alert-state.json"
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
delivered: list[str] = []
|
|
monkeypatch.setattr(
|
|
health,
|
|
"send_webhook",
|
|
lambda _path, summary, **_kwargs: delivered.append(summary["status"]) or {},
|
|
)
|
|
|
|
assert health.main() == 0
|
|
assert health.main() == 0
|
|
summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))
|
|
|
|
assert delivered == ["warning"]
|
|
assert summary["alert_delivery"]["attempted"] is False
|
|
assert summary["alert_delivery"]["suppressed"] is True
|
|
assert summary["alert_delivery"]["decision"] == "duplicate_within_resend_window"
|
|
|
|
|
|
def test_alert_state_change_bypasses_deduplication(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
args.alert_state_file = tmp_path / "alert-state.json"
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
delivered: list[str] = []
|
|
monkeypatch.setattr(
|
|
health,
|
|
"send_webhook",
|
|
lambda _path, summary, **_kwargs: delivered.append(summary["status"]) or {},
|
|
)
|
|
|
|
assert health.main() == 0
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_disk",
|
|
lambda **policy: health.HealthCheck(
|
|
policy["key"],
|
|
"fail" if policy["key"] == "disk_root" else "pass",
|
|
"changed",
|
|
),
|
|
)
|
|
assert health.main() == 1
|
|
|
|
assert delivered == ["warning", "fail"]
|
|
|
|
|
|
def test_healthy_run_resets_alert_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
args = _main_args(tmp_path)
|
|
args.alert_state_file = tmp_path / "alert-state.json"
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
monkeypatch.setattr(health, "send_webhook", lambda *_args, **_kwargs: {})
|
|
assert health.main() == 0
|
|
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_disk",
|
|
lambda **policy: health.HealthCheck(policy["key"], "pass", "healthy"),
|
|
)
|
|
assert health.main() == 0
|
|
|
|
state = json.loads(args.alert_state_file.read_text(encoding="utf-8"))
|
|
assert state["fingerprint"] is None
|
|
assert state["delivered_at"] is None
|
|
|
|
|
|
def test_alert_state_write_failure_is_recorded_and_exits_two(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
args.alert_state_file = tmp_path / "alert-state.json"
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
monkeypatch.setattr(health, "send_webhook", lambda *_args, **_kwargs: {})
|
|
monkeypatch.setattr(
|
|
health,
|
|
"write_alert_state",
|
|
lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("read-only filesystem")),
|
|
)
|
|
|
|
assert health.main() == 2
|
|
summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))
|
|
assert summary["alert_delivery"]["delivered"] is True
|
|
assert summary["alert_delivery"]["error"] == "StateWriteError"
|
|
|
|
|
|
def test_failed_capacity_check_dispatches_webhook_and_exits_one(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
args = _main_args(tmp_path)
|
|
delivered: list[str] = []
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_disk",
|
|
lambda **policy: health.HealthCheck(
|
|
policy["key"],
|
|
"fail" if policy["key"] == "disk_backup" else "pass",
|
|
"simulated",
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"send_webhook",
|
|
lambda _path, summary, **_kwargs: delivered.append(summary["status"]),
|
|
)
|
|
|
|
exit_code = health.main()
|
|
|
|
assert exit_code == 1
|
|
assert delivered == ["fail"]
|
|
|
|
|
|
def test_healthy_run_does_not_dispatch_webhook(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
|
args = _main_args(tmp_path)
|
|
monkeypatch.setattr(health, "_parse_args", lambda: args)
|
|
_mock_non_capacity_checks(monkeypatch)
|
|
monkeypatch.setattr(
|
|
health,
|
|
"check_disk",
|
|
lambda **policy: health.HealthCheck(policy["key"], "pass", "simulated"),
|
|
)
|
|
|
|
def unexpected_delivery(*_args: object, **_kwargs: object) -> None:
|
|
pytest.fail("healthy runs must not dispatch alerts")
|
|
|
|
monkeypatch.setattr(health, "send_webhook", unexpected_delivery)
|
|
|
|
exit_code = health.main()
|
|
summary = json.loads((args.artifact_dir / "production-health-summary.json").read_text(encoding="utf-8"))
|
|
|
|
assert exit_code == 0
|
|
assert summary["alert_delivery"]["required"] is False
|
|
assert summary["alert_delivery"]["attempted"] is False
|
|
|
|
|
|
def test_preflight_rejects_missing_or_inverted_capacity_thresholds() -> None:
|
|
values = {
|
|
"ROOT_DISK_WARNING_PERCENT": "80",
|
|
"ROOT_DISK_CRITICAL_PERCENT": "90",
|
|
"ROOT_DISK_WARNING_FREE_GIB": "20",
|
|
"ROOT_DISK_CRITICAL_FREE_GIB": "10",
|
|
"HOME_DISK_WARNING_PERCENT": "95",
|
|
"HOME_DISK_CRITICAL_PERCENT": "90",
|
|
"HOME_DISK_WARNING_FREE_GIB": "20",
|
|
"HOME_DISK_CRITICAL_FREE_GIB": "10",
|
|
"BACKUP_DISK_WARNING_PERCENT": "80",
|
|
"BACKUP_DISK_CRITICAL_PERCENT": "90",
|
|
"BACKUP_DISK_WARNING_FREE_GIB": "20",
|
|
}
|
|
|
|
result = preflight._validate_capacity_thresholds(values)
|
|
|
|
assert result.status == "fail"
|
|
assert "HOME_DISK" in result.message
|
|
assert "BACKUP_DISK" in result.message
|
|
|
|
|
|
def test_systemd_timer_and_service_cover_all_capacity_paths() -> None:
|
|
unit = (REPO_ROOT / "yuqei-infra/observability/yuqei-production-health.service").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
timer = (REPO_ROOT / "yuqei-infra/observability/yuqei-production-health.timer").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
assert "OnUnitActiveSec=1min" in timer
|
|
assert "Persistent=true" in timer
|
|
assert "ProtectHome=read-only" in unit
|
|
assert "--root-disk-path /" in unit
|
|
assert "--home-disk-path /home" in unit
|
|
assert "--home-require-separate-mount" in unit
|
|
assert "--backup-disk-path ${BACKUP_HOST_DIR}" in unit
|
|
assert "--backup-disk-critical-free-gib ${BACKUP_DISK_CRITICAL_FREE_GIB}" in unit
|
|
assert "--backup-require-separate-mount" in unit
|
|
assert "--webhook-url-file ${PRODUCTION_ALERT_WEBHOOK_URL_FILE}" in unit
|
|
assert "--alert-state-file /opt/apps/ai-contract-platform-v2-production/project-docs/qa-artifacts/v2-production-health/alert-state.json" in unit
|
|
assert "--alert-resend-seconds 3600" in unit
|