ai-contract-platform/yuqei-infra/scripts/verify-provider-promotion-nodes.py

#!/usr/bin/env python3
"""Verify Provider promotion state across production nodes without writing state."""

from __future__ import annotations

import argparse
import json
import math
import re
import sys
import time
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable, Literal
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit, urlunsplit
from urllib.request import HTTPRedirectHandler, Request, build_opener


MAX_RESPONSE_BYTES = 1024 * 1024
MAX_SERVICE_KEY_BYTES = 64 * 1024
SAFE_IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
Mode = Literal["canary", "revocation", "kill-switch"]


class _NoRedirectHandler(HTTPRedirectHandler):
    def redirect_request(self, request, fp, code, msg, headers, newurl):  # pragma: no cover
        raise ValueError("readiness redirects are not permitted")


@dataclass(frozen=True)
class NodeObservation:
    node: str
    readiness_status: str | None
    release_allowed: bool
    provider_acceptance_verified: bool
    receipt_id: str | None
    key_id: str | None
    target_release_id: str | None
    target_image_digest: str | None
    provider_problems: str
    kill_switch_active: bool
    environment: str | None = None
    generated_at: str | None = None
    verification_mode: str | None = None
    promotion_diagnostics_consistent: bool = False
    error: str | None = None


Fetcher = Callable[[str], NodeObservation]


def fetch_node_readiness(
    node: str,
    *,
    service_key: str | None,
    timeout_seconds: float,
) -> NodeObservation:
    base_url = normalize_node_url(node, sends_credential=bool(service_key))
    request = Request(
        f"{base_url}/api/v1/quality/release-readiness",
        method="GET",
        headers={
            "Accept": "application/json",
            "Cache-Control": "no-cache, no-store",
            "Pragma": "no-cache",
            "User-Agent": "yuqei-provider-promotion-canary/1",
            **({"X-AI-Platform-Service-Key": service_key} if service_key else {}),
        },
    )
    opener = build_opener(_NoRedirectHandler())
    try:
        with opener.open(request, timeout=timeout_seconds) as response:
            if response.geturl() != request.full_url:
                raise ValueError("readiness redirects are not permitted")
            body = response.read(MAX_RESPONSE_BYTES + 1)
            if len(body) > MAX_RESPONSE_BYTES:
                raise ValueError("readiness response exceeds the size limit")
    except (HTTPError, URLError, OSError, ValueError) as error:
        return _failed_observation(base_url, f"readiness request failed ({type(error).__name__})")
    try:
        payload = json.loads(body.decode("utf-8"))
        if not isinstance(payload, dict):
            raise ValueError
        return parse_readiness_observation(base_url, payload)
    except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError):
        return _failed_observation(base_url, "readiness response is invalid")


def parse_readiness_observation(node: str, payload: dict[str, Any]) -> NodeObservation:
    checks = payload.get("checks")
    if not isinstance(checks, list):
        raise ValueError("readiness checks are missing")
    provider = _check_by_code(checks, "provider_acceptance")
    kill_switch = _check_by_code(checks, "provider_promotion_kill_switch")
    provider_details = provider.get("details") if isinstance(provider.get("details"), dict) else {}
    kill_details = kill_switch.get("details") if isinstance(kill_switch.get("details"), dict) else {}
    diagnostics = payload.get("promotion_diagnostics")
    if not isinstance(diagnostics, dict):
        raise ValueError("promotion diagnostics are missing")
    environment = _optional_string(payload.get("environment"))
    generated_at = _parse_aware_datetime(payload.get("generated_at"), label="generated_at").isoformat()
    verification_mode = _optional_string(diagnostics.get("verification_mode"))
    provider_verified = payload.get("provider_acceptance_verified") is True
    kill_switch_active = kill_details.get("active") is True
    diagnostic_bindings = (
        ("release_id", "target_release_id"),
        ("image_digest", "target_image_digest"),
        ("receipt_id", "receipt_id"),
        ("key_id", "key_id"),
    )
    diagnostics_consistent = (
        diagnostics.get("verified") is provider_verified
        and diagnostics.get("kill_switch_active") is kill_switch_active
        and all(
            provider_details.get(detail_key) is None
            or diagnostics.get(diagnostic_key) == provider_details.get(detail_key)
            for diagnostic_key, detail_key in diagnostic_bindings
        )
    )
    return NodeObservation(
        node=node,
        readiness_status=str(payload.get("status")) if payload.get("status") is not None else None,
        release_allowed=payload.get("release_allowed") is True,
        provider_acceptance_verified=provider_verified,
        receipt_id=_optional_string(provider_details.get("receipt_id")),
        key_id=_optional_string(provider_details.get("key_id")),
        target_release_id=_optional_string(provider_details.get("target_release_id")),
        target_image_digest=_optional_string(provider_details.get("target_image_digest")),
        provider_problems=_optional_string(provider_details.get("problems")) or "",
        kill_switch_active=kill_switch_active,
        environment=environment,
        generated_at=generated_at,
        verification_mode=verification_mode,
        promotion_diagnostics_consistent=diagnostics_consistent,
    )


def verify_canary(
    observations: list[NodeObservation],
    *,
    expected_release_id: str,
    expected_receipt_id: str,
    expected_key_id: str,
    expected_image_digest: str,
    max_observation_age_seconds: float = 120.0,
    now: datetime | None = None,
) -> list[str]:
    problems = verify_observation_provenance(
        observations,
        max_age_seconds=max_observation_age_seconds,
        now=now,
    )
    for item in observations:
        if item.error:
            problems.append(f"{item.node}: {item.error}")
        elif item.kill_switch_active:
            problems.append(f"{item.node}: promotion kill switch is active")
        elif item.readiness_status != "ready" or not item.release_allowed:
            problems.append(f"{item.node}: release readiness is not ready")
        elif not item.provider_acceptance_verified:
            problems.append(f"{item.node}: Provider acceptance is not verified")
        elif item.target_release_id != expected_release_id:
            problems.append(f"{item.node}: release id does not match")
        elif item.receipt_id != expected_receipt_id:
            problems.append(f"{item.node}: receipt id does not match")
        elif item.key_id != expected_key_id:
            problems.append(f"{item.node}: receipt key id does not match")
        elif item.target_image_digest != expected_image_digest:
            problems.append(f"{item.node}: image digest does not match")
    fingerprints = {
        (item.target_release_id, item.target_image_digest, item.receipt_id, item.key_id)
        for item in observations
        if not item.error
    }
    if len(fingerprints) > 1:
        problems.append("nodes do not report one immutable promotion fingerprint")
    return problems


def verify_observation_provenance(
    observations: list[NodeObservation],
    *,
    max_age_seconds: float,
    now: datetime | None = None,
) -> list[str]:
    observed_at = now or datetime.now(UTC)
    if observed_at.tzinfo is None:
        raise ValueError("verification time must include a timezone")
    problems: list[str] = []
    for item in observations:
        if item.error:
            continue
        if item.environment != "production":
            problems.append(f"{item.node}: readiness is not from production")
        if item.verification_mode != "offline_read_only":
            problems.append(f"{item.node}: readiness is not offline read-only verification")
        if not item.promotion_diagnostics_consistent:
            problems.append(f"{item.node}: promotion diagnostics are inconsistent")
        try:
            generated_at = _parse_aware_datetime(item.generated_at, label="generated_at")
        except ValueError:
            problems.append(f"{item.node}: readiness generation time is invalid")
            continue
        age_seconds = (observed_at.astimezone(UTC) - generated_at.astimezone(UTC)).total_seconds()
        if age_seconds < -30:
            problems.append(f"{item.node}: readiness generation time is in the future")
        elif age_seconds > max_age_seconds:
            problems.append(f"{item.node}: readiness response is stale")
    return problems


def wait_for_control(
    nodes: list[str],
    *,
    mode: Literal["revocation", "kill-switch"],
    fetcher: Fetcher,
    deadline_seconds: float,
    poll_interval_seconds: float,
    clock: Callable[[], float] = time.monotonic,
    sleeper: Callable[[float], None] = time.sleep,
) -> tuple[list[NodeObservation], list[str]]:
    deadline = clock() + deadline_seconds
    observations: list[NodeObservation] = []
    while True:
        observations = [fetcher(node) for node in nodes]
        pending = [item.node for item in observations if not _control_observed(item, mode)]
        if not pending:
            return observations, []
        if clock() >= deadline:
            return observations, [f"{node}: {mode} did not propagate within SLA" for node in pending]
        sleeper(min(poll_interval_seconds, max(0.0, deadline - clock())))


def _control_observed(item: NodeObservation, mode: str) -> bool:
    if item.error or item.release_allowed or item.provider_acceptance_verified:
        return False
    if mode == "kill-switch":
        return item.kill_switch_active
    return "revoked" in item.provider_problems.lower()


def normalize_node_url(value: str, *, sends_credential: bool) -> str:
    parsed = urlsplit(value.strip())
    if parsed.scheme.lower() not in {"http", "https"} or not parsed.hostname:
        raise ValueError("node URL must be HTTP or HTTPS")
    if parsed.username or parsed.password or parsed.query or parsed.fragment:
        raise ValueError("node URL must not contain credentials, query, or fragment")
    hostname = parsed.hostname.lower()
    if sends_credential and parsed.scheme.lower() != "https" and hostname not in {"127.0.0.1", "::1", "localhost"}:
        raise ValueError("service authentication requires HTTPS except on loopback")
    port = parsed.port
    default_port = (parsed.scheme.lower() == "https" and port == 443) or (
        parsed.scheme.lower() == "http" and port == 80
    )
    netloc = hostname if port is None or default_port else f"{hostname}:{port}"
    return urlunsplit((parsed.scheme.lower(), netloc, parsed.path.rstrip("/"), "", ""))


def _check_by_code(checks: list[Any], code: str) -> dict[str, Any]:
    matches = [item for item in checks if isinstance(item, dict) and item.get("code") == code]
    if len(matches) != 1:
        raise ValueError(f"readiness check {code} is missing or duplicated")
    return matches[0]


def _optional_string(value: Any) -> str | None:
    return value if isinstance(value, str) and value else None


def _parse_aware_datetime(value: Any, *, label: str) -> datetime:
    if not isinstance(value, str) or not value:
        raise ValueError(f"{label} is missing")
    try:
        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
    except ValueError:
        raise ValueError(f"{label} is invalid") from None
    if parsed.tzinfo is None:
        raise ValueError(f"{label} must include a timezone")
    return parsed


def _failed_observation(node: str, error: str) -> NodeObservation:
    return NodeObservation(
        node=node,
        readiness_status=None,
        release_allowed=False,
        provider_acceptance_verified=False,
        receipt_id=None,
        key_id=None,
        target_release_id=None,
        target_image_digest=None,
        provider_problems="",
        kill_switch_active=False,
        error=error,
    )


def _read_service_key(path: Path | None) -> str | None:
    if path is None:
        return None
    try:
        resolved = path.resolve(strict=True)
        if not resolved.is_file():
            raise ValueError
        with resolved.open("rb") as stream:
            raw = stream.read(MAX_SERVICE_KEY_BYTES + 1)
        if not raw or len(raw) > MAX_SERVICE_KEY_BYTES:
            raise ValueError
        key = raw.decode("utf-8").strip()
        if not key:
            raise ValueError
        return key
    except (OSError, UnicodeDecodeError, ValueError):
        raise ValueError("service key is unavailable or invalid") from None


def _parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--mode", choices=("canary", "revocation", "kill-switch"), default="canary")
    parser.add_argument("--node", action="append", required=True, help="Production AI Platform base URL; repeat for every node.")
    parser.add_argument("--service-key-file", type=Path)
    parser.add_argument("--expected-release-id")
    parser.add_argument("--expected-receipt-id")
    parser.add_argument("--expected-key-id")
    parser.add_argument("--expected-image-digest")
    parser.add_argument("--timeout-seconds", type=float, default=5.0)
    parser.add_argument("--deadline-seconds", type=float, default=300.0)
    parser.add_argument("--poll-interval-seconds", type=float, default=2.0)
    parser.add_argument("--max-observation-age-seconds", type=float, default=120.0)
    return parser.parse_args()


def main() -> int:
    args = _parse_args()
    try:
        nodes = list(dict.fromkeys(normalize_node_url(node, sends_credential=bool(args.service_key_file)) for node in args.node))
        if len(nodes) < 2:
            raise ValueError("multi-node verification requires at least two distinct nodes")
        if not 0.1 <= args.timeout_seconds <= 60 or not math.isfinite(args.timeout_seconds):
            raise ValueError("timeout must be between 0.1 and 60 seconds")
        if not 1 <= args.max_observation_age_seconds <= 600 or not math.isfinite(args.max_observation_age_seconds):
            raise ValueError("maximum observation age must be between 1 and 600 seconds")
        service_key = _read_service_key(args.service_key_file)
        fetcher = lambda node: fetch_node_readiness(
            node,
            service_key=service_key,
            timeout_seconds=args.timeout_seconds,
        )
        if args.mode == "canary":
            expected = (args.expected_release_id, args.expected_receipt_id, args.expected_key_id)
            if any(not value or not SAFE_IDENTIFIER.fullmatch(value) for value in expected):
                raise ValueError("canary requires valid release, receipt, and key identifiers")
            if not args.expected_image_digest or not re.fullmatch(r"sha256:[0-9a-f]{64}", args.expected_image_digest):
                raise ValueError("canary requires a valid target image digest")
            observations = [fetcher(node) for node in nodes]
            problems = verify_canary(
                observations,
                expected_release_id=args.expected_release_id,
                expected_receipt_id=args.expected_receipt_id,
                expected_key_id=args.expected_key_id,
                expected_image_digest=args.expected_image_digest,
                max_observation_age_seconds=args.max_observation_age_seconds,
            )
        else:
            if not 1 <= args.deadline_seconds <= 900 or not math.isfinite(args.deadline_seconds):
                raise ValueError("control propagation deadline must be between 1 and 900 seconds")
            if not 0.1 <= args.poll_interval_seconds <= 30 or not math.isfinite(args.poll_interval_seconds):
                raise ValueError("poll interval must be between 0.1 and 30 seconds")
            observations, problems = wait_for_control(
                nodes,
                mode=args.mode,
                fetcher=fetcher,
                deadline_seconds=args.deadline_seconds,
                poll_interval_seconds=args.poll_interval_seconds,
            )
            problems.extend(
                verify_observation_provenance(
                    observations,
                    max_age_seconds=args.max_observation_age_seconds,
                )
            )
        summary = {
            "mode": args.mode,
            "status": "passed" if not problems else "failed",
            "node_count": len(nodes),
            "observations": [asdict(item) for item in observations],
            "problems": problems,
        }
        print(json.dumps(summary, ensure_ascii=True, sort_keys=True, separators=(",", ":")))
        return 0 if not problems else 1
    except ValueError as error:
        print(json.dumps({"mode": args.mode, "status": "failed", "problems": [str(error)]}, separators=(",", ":")))
        return 2


if __name__ == "__main__":
    raise SystemExit(main())