yuqei/ai-contract-platform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
74da61a9f18d334e8df53e792fef4f1991e4fb70
ai-contract-platform/yuqei-infra/deploy/release-package.py
T

4678 lines
192 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""Verify, plan, and explicitly apply one immutable Yuqei backend package.
The default operation is read-only verification. Install, rollback, and
forward operations are plans unless ``--apply`` and the operation's exact
confirmation string are supplied; those controls are necessary but not
sufficient. Production apply currently fails closed while the declared
release blockers remain unresolved. The executor is deliberately offline: it
never builds or pulls images and it never manages traffic, DNS, volumes, or
database downgrades. Contract Web identity is package-bound but is deployed by
its separate web orchestrator, so no result from this tool claims completion of
the full cross-host release.
"""
from __future__ import annotations
import argparse
import hashlib
import importlib.util
import json
import os
import re
import stat
import subprocess
import sys
import time
from contextlib import contextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Callable, Iterator, Mapping
REPO_ROOT = Path(__file__).resolve().parents[2]
PACKAGE_FILE = "release-package.json"
BASE_COMPOSE_RELATIVE = "yuqei-infra/compose/docker-compose.yml"
PRODUCTION_COMPOSE_RELATIVE = "yuqei-infra/compose/docker-compose.production.yml"
OFFLINE_COMPOSE_RELATIVE = "yuqei-infra/compose/docker-compose.production.offline.yml"
PACKAGE_SCHEMA_V1 = "yuqei.release-package.v1"
PACKAGE_SCHEMA_V2 = "yuqei.release-package.v2"
# Historical fixtures import this name. New installable candidates must opt in
# to v2 explicitly; v1 remains read-only compatibility only.
PACKAGE_SCHEMA = PACKAGE_SCHEMA_V1
CHECKSUM_SCHEMA = "yuqei.release-package-checksums.v1"
SOURCE_MANIFEST_SCHEMA = "yuqei.release-source-manifest.v1"
IMAGE_IDENTITY_SCHEMA_V1 = "yuqei.release-image-identity.v1"
IMAGE_IDENTITY_SCHEMA_V2 = "yuqei.release-image-identity.v2"
# Historical callers and fixtures import this name. Keep it pinned to v1;
# v2 is opt-in and is never inferred from the payload shape.
IMAGE_IDENTITY_SCHEMA = IMAGE_IDENTITY_SCHEMA_V1
BACKUP_PROOF_SCHEMA = "yuqei.release-backup-proof.v1"
COMPATIBILITY_PROOF_SCHEMA = "yuqei.release-rollback-compatibility.v1"
INSTALLABLE_MANIFEST_SCHEMA = "yuqei.installable-release-manifest.v1"
DOCKER_IMPORT_PLAN_SCHEMA = "yuqei.docker-image-import-plan.v1"
PLAN_SCHEMA = "yuqei.release-package-plan.v4"
RECEIPT_SCHEMA = "yuqei.release-package-receipt.v3"
PENDING_SCHEMA = "yuqei.release-package-pending.v3"
POST_APPLY_SCHEMA = "yuqei.release-post-apply-check.v2"
PREFLIGHT_BINDING_SCHEMA = "yuqei.production-preflight-binding.v2"
INFRASTRUCTURE_SOURCE_SET_SCHEMA = "yuqei.infrastructure-image-source-set.v2"
INFRASTRUCTURE_BUNDLE_SCHEMA = "yuqei.infrastructure-image-bundle.v2"
THIRD_PARTY_INPUT_SCHEMA = "yuqei.third-party-materials-input.v1"
THIRD_PARTY_MANIFEST_SCHEMA = "yuqei.third-party-materials.v1"
THIRD_PARTY_VERIFICATION_SCHEMA = "yuqei.third-party-materials-verification.v1"
RELEASE_IMPORT_RESULT_SCHEMA = "yuqei.docker-image-import-result.v2"
RECOVERY_READINESS_RESULT = "PASS_RECOVERY_READINESS"
DOCKER_CLI = "/usr/bin/docker"
LOCAL_DOCKER_HOST = "unix:///var/run/docker.sock"
FIXED_DOCKER_HOME = "/var/empty"
REAL_APPLY_BLOCKERS = (
"infrastructure_docker_import_plan_not_available",
"backup_and_compatibility_external_signatures_not_verified",
)
FIRST_RELEASE_REAL_APPLY_BLOCKERS = (
"infrastructure_docker_import_plan_not_available",
"preinstall_system_and_database_snapshot_evidence_not_verified",
)
LEGACY_PACKAGE_APPLY_BLOCKER = "legacy_package_schema_not_apply_eligible"
THIRD_PARTY_MATERIALS_BLOCKER = "third_party_materials_not_independently_verified"
RECOVERY_STRATEGY_PREDECESSOR = "predecessor_release"
RECOVERY_STRATEGY_FIRST_RELEASE = "first_release_snapshot_restore"
FIRST_RELEASE_REQUIRED_RESTORE_SCOPES = (
"host_runtime_state",
"postgres_consistent_backup",
"object_storage_state",
"release_environment_and_tls_bindings",
)
FIRST_RELEASE_REQUIRED_POSTGRES_DATABASES = ("contract", "ai", "operations")
FIRST_RELEASE_REQUIRED_STORAGE_TARGETS = (
"contract_files_volume",
"minio_object_store",
)
PREDECESSOR_EXTERNAL_EVIDENCE_REQUIREMENTS = (
"externally_signed_backup_proof",
"externally_signed_schema_compatibility_proof",
"approved_predecessor_application_import_plan",
)
FIRST_RELEASE_EXTERNAL_EVIDENCE_REQUIREMENTS = (
"signed_no_prior_successful_release_receipt",
"preinstall_quiesced_system_and_data_snapshot",
"independent_snapshot_restore_test",
"single_named_release_owner_approval",
)
CONFIRMATIONS = {
"install": "I_AUTHORIZE_THIS_EXACT_RELEASE_INSTALL",
"rollback": "I_AUTHORIZE_THIS_EXACT_RELEASE_ROLLBACK",
"forward": "I_AUTHORIZE_THIS_EXACT_RELEASE_FORWARD",
}
APPLICATION_IMAGE_KEYS = (
"CONTRACT_API_IMAGE",
"CONTRACT_WEB_IMAGE",
"AI_PLATFORM_API_IMAGE",
"AI_WORKER_IMAGE",
)
BACKEND_APPLICATION_IMAGE_KEYS = (
"CONTRACT_API_IMAGE",
"AI_PLATFORM_API_IMAGE",
"AI_WORKER_IMAGE",
)
INFRASTRUCTURE_IMAGE_KEYS = (
"POSTGRES_IMAGE",
"POSTGRES_CLIENT_IMAGE",
"REDIS_IMAGE",
"MINIO_IMAGE",
"MINIO_MC_IMAGE",
"NGINX_IMAGE",
"BACKUP_UTILITY_IMAGE",
)
BACKEND_INSTALL_INFRASTRUCTURE_IMAGE_KEYS = (
"POSTGRES_IMAGE",
"POSTGRES_CLIENT_IMAGE",
"REDIS_IMAGE",
"MINIO_IMAGE",
)
DEPLOYMENT_SCOPE = "backend_only"
SEPARATELY_DEPLOYED_IMAGE_KEYS = ("CONTRACT_WEB_IMAGE",)
RUNTIME_SERVICES = (
"contract-api",
"contract-review-worker",
"contract-playbook-governance-worker",
"ai-platform-api",
"worker",
)
INSTALL_SERVICES = (
"postgres",
"postgres-init-databases",
"redis",
"ai-redis",
"minio",
"contract-api-migrate",
"ai-platform-api-migrate",
"ai-platform-prompt-assets-import",
*RUNTIME_SERVICES,
)
V1_REQUIRED_ARTIFACTS = (
"release_manifest",
"checksum_manifest",
"image_identity",
"installable_manifest",
)
V2_REQUIRED_ARTIFACTS = (
*V1_REQUIRED_ARTIFACTS,
"infrastructure_source_set",
"infrastructure_image_bundle",
"third_party_materials_input",
"third_party_materials_manifest",
"third_party_materials_verification_receipt",
)
# Backwards compatible export used by older callers.
REQUIRED_ARTIFACTS = V1_REQUIRED_ARTIFACTS
OPTIONAL_EVIDENCE_ARTIFACTS = ("backup_proof", "compatibility_proof")
IMPORT_ROLE_ENV_KEYS = (
("contract-web", "CONTRACT_WEB_IMAGE"),
("contract-api", "CONTRACT_API_IMAGE"),
("ai-platform-api", "AI_PLATFORM_API_IMAGE"),
("ai-worker", "AI_WORKER_IMAGE"),
)
RELEASE_IMPORT_ROLE_ENV_SERVICES = (
("contract-web", "CONTRACT_WEB_IMAGE", ()),
(
"contract-api",
"CONTRACT_API_IMAGE",
(
"contract-api",
"contract-api-migrate",
"contract-review-worker",
"contract-playbook-governance-worker",
),
),
(
"ai-platform-api",
"AI_PLATFORM_API_IMAGE",
(
"ai-platform-api",
"ai-platform-api-migrate",
"ai-platform-prompt-assets-import",
),
),
("ai-worker", "AI_WORKER_IMAGE", ("worker",)),
("postgres-runtime", "POSTGRES_IMAGE", ("postgres",)),
("postgres-client", "POSTGRES_CLIENT_IMAGE", ("postgres-init-databases",)),
("redis", "REDIS_IMAGE", ("redis", "ai-redis")),
("minio", "MINIO_IMAGE", ("minio",)),
("minio-client", "MINIO_MC_IMAGE", ("minio-backup",)),
("nginx-edge", "NGINX_IMAGE", ("edge",)),
("backup-utility", "BACKUP_UTILITY_IMAGE", ("contract-files-backup",)),
)
LONG_RUNNING_INSTALL_SERVICES = (
"postgres",
"redis",
"ai-redis",
"minio",
*RUNTIME_SERVICES,
)
ONE_SHOT_INSTALL_SERVICES = (
"postgres-init-databases",
"contract-api-migrate",
"ai-platform-api-migrate",
"ai-platform-prompt-assets-import",
)
POST_APPLY_WAIT_TIMEOUT_SECONDS = 300
POST_APPLY_POLL_INTERVAL_SECONDS = 2
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
COMMIT_RE = re.compile(r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")
RELEASE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$")
PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$")
APPROVAL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$")
ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
IMAGE_REFERENCE_RE = re.compile(
r"^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$"
)
IMAGE_DIGEST_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
MAX_JSON_BYTES = 4 * 1024 * 1024
MAX_ENV_BYTES = 1024 * 1024
FORBIDDEN_COMMAND_TOKENS = frozenset(
{
"down",
"rm",
"prune",
"build",
"pull",
"push",
"volume",
"volumes",
"downgrade",
"edge",
"dns",
"--volumes",
"-v",
}
)
class ReleasePackageError(RuntimeError):
"""Raised when a package or requested transition fails closed."""
@dataclass(frozen=True)
class ArtifactPin:
path: str
sha256: str
@dataclass(frozen=True)
class ValidatedPackage:
root: Path
package_schema_version: str
package: dict[str, Any]
package_sha256: str
checksum_manifest: dict[str, Any]
checksum_manifest_sha256: str
release_manifest: dict[str, Any]
release_manifest_sha256: str
image_identity: dict[str, Any]
image_identity_sha256: str
installable_manifest: dict[str, Any]
installable_manifest_sha256: str
installable_root: Path
trusted_build_receipt_sha256: str
docker_import_plan: tuple[dict[str, Any], ...]
offline_image_overrides: dict[str, str]
infrastructure_source_set: dict[str, Any] | None
infrastructure_source_set_sha256: str | None
infrastructure_bundle: dict[str, Any] | None
infrastructure_bundle_sha256: str | None
third_party_materials_input: dict[str, Any] | None
third_party_materials_input_sha256: str | None
third_party_materials_manifest: dict[str, Any] | None
third_party_materials_manifest_sha256: str | None
third_party_verification_receipt: dict[str, Any] | None
third_party_verification_receipt_sha256: str | None
third_party_profile: str | None
backup_proof: dict[str, Any] | None
backup_proof_sha256: str | None
compatibility_proof: dict[str, Any] | None
compatibility_proof_sha256: str | None
application_images: dict[str, dict[str, str]]
infrastructure_images: dict[str, dict[str, str]]
recovery_strategy: str
recovery_policy: dict[str, Any]
rollback_images: dict[str, dict[str, str]] | None
rollback_release_id: str | None
@dataclass(frozen=True)
class ExecutionOutcome:
payload: dict[str, Any]
exit_code: int = 0
replayed: bool = False
@dataclass(frozen=True)
class RuntimeEvidence:
env_file: Path
env_values: dict[str, str]
env_sha256: str
preflight_summary: dict[str, Any]
compose_gate_summary: dict[str, Any]
daemon_identity: dict[str, str]
@dataclass(frozen=True)
class SnapshotInputs:
root: Path
source_env_file: Path
env_file: Path
compose_files: tuple[Path, Path, Path]
docker_archives: dict[str, Path]
tls_files: dict[str, Path]
manifest: dict[str, Any]
_PREFLIGHT_MODULE: Any | None = None
_COMPOSE_GATE_MODULE: Any | None = None
_INSTALLABLE_VALIDATOR_MODULE: Any | None = None
_INFRASTRUCTURE_BUNDLE_MODULE: Any | None = None
_THIRD_PARTY_MATERIALS_MODULE: Any | None = None
_SNAPSHOT_RESTORE_PLANNER_MODULE: Any | None = None
def _exec_module_without_bytecode(spec: Any, module: Any) -> None:
"""Execute a local validator without mutating its source tree."""
previous = sys.dont_write_bytecode
sys.dont_write_bytecode = True
try:
spec.loader.exec_module(module)
finally:
sys.dont_write_bytecode = previous
def _load_local_module(filename: str, module_name: str) -> Any:
path = Path(__file__).resolve().with_name(filename)
spec = importlib.util.spec_from_file_location(module_name, path)
if spec is None or spec.loader is None:
raise ReleasePackageError(f"Cannot load required deployment gate: {filename}")
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
try:
_exec_module_without_bytecode(spec, module)
except Exception as exc:
sys.modules.pop(module_name, None)
raise ReleasePackageError(f"Cannot load required deployment gate: {filename}") from exc
return module
def _load_preflight_module() -> Any:
global _PREFLIGHT_MODULE
if _PREFLIGHT_MODULE is None:
_PREFLIGHT_MODULE = _load_local_module(
"preflight-production.py",
"yuqei_release_package_preflight",
)
return _PREFLIGHT_MODULE
def _load_compose_gate_module() -> Any:
global _COMPOSE_GATE_MODULE
if _COMPOSE_GATE_MODULE is None:
_COMPOSE_GATE_MODULE = _load_local_module(
"verify-production-compose.py",
"yuqei_release_package_compose_gate",
)
return _COMPOSE_GATE_MODULE
def _load_installable_validator_module() -> Any:
global _INSTALLABLE_VALIDATOR_MODULE
if _INSTALLABLE_VALIDATOR_MODULE is None:
path = Path(__file__).resolve().parents[1] / "scripts" / "build-installable-release.py"
spec = importlib.util.spec_from_file_location(
"yuqei_release_package_installable_validator",
path,
)
if spec is None or spec.loader is None:
raise ReleasePackageError("Cannot load the strict installable manifest validator.")
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
try:
_exec_module_without_bytecode(spec, module)
except Exception as exc:
sys.modules.pop(spec.name, None)
raise ReleasePackageError(
"Cannot load the strict installable manifest validator."
) from exc
_INSTALLABLE_VALIDATOR_MODULE = module
return _INSTALLABLE_VALIDATOR_MODULE
def _load_script_module(relative: str, module_name: str, *, label: str) -> Any:
path = Path(__file__).resolve().parents[1] / relative
spec = importlib.util.spec_from_file_location(module_name, path)
if spec is None or spec.loader is None:
raise ReleasePackageError(f"Cannot load the fixed {label}.")
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
try:
_exec_module_without_bytecode(spec, module)
except Exception as exc:
sys.modules.pop(module_name, None)
raise ReleasePackageError(f"Cannot load the fixed {label}.") from exc
return module
def _load_infrastructure_bundle_module() -> Any:
global _INFRASTRUCTURE_BUNDLE_MODULE
if _INFRASTRUCTURE_BUNDLE_MODULE is None:
_INFRASTRUCTURE_BUNDLE_MODULE = _load_script_module(
"scripts/build-infrastructure-image-bundle.py",
"yuqei_release_package_infrastructure_bundle",
label="infrastructure image bundle validator",
)
return _INFRASTRUCTURE_BUNDLE_MODULE
def _load_third_party_materials_module() -> Any:
global _THIRD_PARTY_MATERIALS_MODULE
if _THIRD_PARTY_MATERIALS_MODULE is None:
_THIRD_PARTY_MATERIALS_MODULE = _load_script_module(
"scripts/build-third-party-materials.py",
"yuqei_release_package_third_party_materials",
label="third-party materials validator",
)
return _THIRD_PARTY_MATERIALS_MODULE
def _load_snapshot_restore_planner_module() -> Any:
global _SNAPSHOT_RESTORE_PLANNER_MODULE
if _SNAPSHOT_RESTORE_PLANNER_MODULE is None:
_SNAPSHOT_RESTORE_PLANNER_MODULE = _load_local_module(
"plan-first-release-snapshot-restore.py",
"yuqei_release_package_snapshot_restore_planner",
)
return _SNAPSHOT_RESTORE_PLANNER_MODULE
def canonical_json_bytes(value: object) -> bytes:
try:
return json.dumps(
value,
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
).encode("utf-8")
except (TypeError, ValueError) as exc:
raise ReleasePackageError("Value is not canonical JSON.") from exc
def canonical_sha256(value: object) -> str:
return hashlib.sha256(canonical_json_bytes(value)).hexdigest()
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def confirmation_for(operation: str) -> str:
try:
return CONFIRMATIONS[operation]
except KeyError as exc:
raise ReleasePackageError(f"Unsupported write operation: {operation}") from exc
def _real_apply_blockers(
operation: str,
*,
recovery_strategy: str = RECOVERY_STRATEGY_PREDECESSOR,
package_schema_version: str = PACKAGE_SCHEMA_V1,
infrastructure_ready: bool = False,
third_party_ready: bool = False,
recovery_readiness_ready: bool = False,
) -> list[str]:
if package_schema_version == PACKAGE_SCHEMA_V1:
blockers = [LEGACY_PACKAGE_APPLY_BLOCKER]
blockers.extend(
FIRST_RELEASE_REAL_APPLY_BLOCKERS
if recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else REAL_APPLY_BLOCKERS
)
else:
blockers = []
if not infrastructure_ready:
blockers.append("infrastructure_docker_import_plan_not_available")
if not third_party_ready:
blockers.append(THIRD_PARTY_MATERIALS_BLOCKER)
if recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE:
if not recovery_readiness_ready:
blockers.append(
"preinstall_system_and_database_snapshot_evidence_not_verified"
)
else:
blockers.append(
"backup_and_compatibility_external_signatures_not_verified"
)
if (
operation == "rollback"
and recovery_strategy == RECOVERY_STRATEGY_PREDECESSOR
):
blockers.append("rollback_docker_import_plan_not_available")
return blockers
def _reject_json_constant(value: str) -> None:
raise ReleasePackageError(f"Non-finite JSON value is forbidden: {value}")
def _reject_duplicate_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in pairs:
if key in result:
raise ReleasePackageError(f"Duplicate JSON key is forbidden: {key}")
result[key] = value
return result
def _read_json(path: Path, *, label: str) -> dict[str, Any]:
try:
size = path.stat().st_size
if size <= 0 or size > MAX_JSON_BYTES:
raise ReleasePackageError(f"{label} must be non-empty and no larger than 4 MiB.")
raw = path.read_bytes()
payload = json.loads(
raw.decode("utf-8"),
object_pairs_hook=_reject_duplicate_pairs,
parse_constant=_reject_json_constant,
)
except ReleasePackageError:
raise
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
raise ReleasePackageError(f"{label} is not readable strict UTF-8 JSON.") from exc
if not isinstance(payload, dict):
raise ReleasePackageError(f"{label} must be a JSON object.")
return payload
def _verify_embedded_hash(
payload: Mapping[str, Any],
field: str,
*,
label: str,
) -> None:
expected = payload.get(field)
if not isinstance(expected, str) or not SHA256_RE.fullmatch(expected):
raise ReleasePackageError(f"{label} has no valid {field}.")
unsigned = {key: value for key, value in payload.items() if key != field}
if canonical_sha256(unsigned) != expected:
raise ReleasePackageError(f"{label} canonical payload hash mismatch.")
def _absolute_lexical(path: Path) -> Path:
return Path(os.path.abspath(os.fspath(path)))
def _reject_symlink_components(path: Path, *, label: str) -> None:
absolute = _absolute_lexical(path)
current = Path(absolute.anchor)
for part in absolute.parts[1:]:
current = current / part
try:
if current.exists() and current.is_symlink():
raise ReleasePackageError(f"{label} must not contain symbolic links: {current}")
except OSError as exc:
raise ReleasePackageError(f"Unable to inspect {label} path components.") from exc
def _validated_directory(path: Path, *, label: str) -> Path:
lexical = _absolute_lexical(path)
_reject_symlink_components(lexical, label=label)
try:
resolved = lexical.resolve(strict=True)
except OSError as exc:
raise ReleasePackageError(f"{label} does not exist.") from exc
if lexical.is_symlink() or not resolved.is_dir():
raise ReleasePackageError(f"{label} must be a real directory, not a symbolic link.")
return resolved
def _validated_file(path: Path, *, label: str, max_bytes: int | None = None) -> Path:
lexical = _absolute_lexical(path)
if not lexical.is_absolute(): # pragma: no cover - os.path.abspath guarantees this.
raise ReleasePackageError(f"{label} must be absolute.")
_reject_symlink_components(lexical, label=label)
try:
resolved = lexical.resolve(strict=True)
size = resolved.stat().st_size
except OSError as exc:
raise ReleasePackageError(f"{label} is unavailable.") from exc
if lexical.is_symlink() or not resolved.is_file():
raise ReleasePackageError(f"{label} must be a real regular file, not a symbolic link.")
if max_bytes is not None and (size <= 0 or size > max_bytes):
raise ReleasePackageError(f"{label} has an invalid size.")
return resolved
def _safe_relative_path(value: object, *, label: str) -> str:
if not isinstance(value, str) or not value or "\\" in value or "\x00" in value:
raise ReleasePackageError(f"{label} must use a non-empty canonical POSIX relative path.")
pure = PurePosixPath(value)
if (
pure.is_absolute()
or pure.as_posix() != value
or any(part in {"", ".", ".."} for part in pure.parts)
):
raise ReleasePackageError(f"{label} contains path traversal or is not canonical: {value!r}")
return value
def _artifact_file(root: Path, relative_path: str, *, label: str) -> Path:
relative_path = _safe_relative_path(relative_path, label=label)
lexical = root.joinpath(*PurePosixPath(relative_path).parts)
_reject_symlink_components(lexical, label=label)
try:
resolved = lexical.resolve(strict=True)
resolved.relative_to(root)
except (OSError, ValueError) as exc:
raise ReleasePackageError(f"{label} escapes the release package or is missing.") from exc
if lexical.is_symlink() or not resolved.is_file():
raise ReleasePackageError(f"{label} must be a real regular file, not a symbolic link.")
return resolved
def _artifact_pin(package: Mapping[str, Any], name: str) -> ArtifactPin:
artifacts = package.get("artifacts")
if not isinstance(artifacts, dict):
raise ReleasePackageError("release-package.json has no artifacts object.")
raw = artifacts.get(name)
if not isinstance(raw, dict):
raise ReleasePackageError(f"Required release package artifact is missing: {name}")
if set(raw) != {"path", "sha256"}:
raise ReleasePackageError(f"Artifact {name} fields are missing or unknown.")
path = _safe_relative_path(raw.get("path"), label=f"artifact {name} path")
digest = raw.get("sha256")
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
raise ReleasePackageError(f"Artifact {name} has no valid lowercase SHA-256.")
return ArtifactPin(path=path, sha256=digest)
def _validate_identifier(value: object, pattern: re.Pattern[str], *, label: str) -> str:
if not isinstance(value, str) or not pattern.fullmatch(value):
raise ReleasePackageError(f"{label} is missing or unsafe.")
return value
def _validate_image_map(
value: object,
expected_keys: tuple[str, ...],
*,
label: str,
) -> dict[str, dict[str, str]]:
if not isinstance(value, dict) or set(value) != set(expected_keys):
raise ReleasePackageError(f"{label} must contain exactly: {', '.join(expected_keys)}")
result: dict[str, dict[str, str]] = {}
for key in expected_keys:
raw = value[key]
if not isinstance(raw, dict):
raise ReleasePackageError(f"{label}.{key} must be an object.")
reference = raw.get("reference")
digest = raw.get("digest")
if not isinstance(reference, str) or not IMAGE_REFERENCE_RE.fullmatch(reference):
raise ReleasePackageError(f"{label}.{key} is not an immutable safe image reference.")
if not isinstance(digest, str) or not IMAGE_DIGEST_RE.fullmatch(digest):
raise ReleasePackageError(f"{label}.{key} has no valid image digest.")
if reference.rsplit("@", 1)[1] != digest:
raise ReleasePackageError(f"{label}.{key} reference and digest do not match.")
result[key] = {"reference": reference, "digest": digest}
return result
def _validate_image_identity_recovery(
image_identity: Mapping[str, Any],
*,
application_images: Mapping[str, Mapping[str, str]],
) -> tuple[
str,
dict[str, Any],
dict[str, dict[str, str]] | None,
str | None,
]:
schema = image_identity.get("schema_version")
if schema == IMAGE_IDENTITY_SCHEMA_V1:
if "recovery" in image_identity:
raise ReleasePackageError(
"Image identity v1 must not mix the v2 recovery contract with legacy rollback fields."
)
rollback = image_identity.get("rollback")
if not isinstance(rollback, dict) or set(rollback) != {
"target_release_id",
"evidence_status",
"application_images",
}:
raise ReleasePackageError("Image identity has no complete rollback candidate.")
if rollback.get("evidence_status") != "external_approval_missing":
raise ReleasePackageError(
"Rollback image identity must remain unapproved until external evidence exists."
)
rollback_release_id = _validate_identifier(
rollback.get("target_release_id"),
RELEASE_ID_RE,
label="rollback target release_id",
)
rollback_images = _validate_image_map(
rollback.get("application_images"),
APPLICATION_IMAGE_KEYS,
label="rollback.application_images",
)
if rollback_images == application_images:
raise ReleasePackageError(
"Rollback image identity must differ from the install image identity."
)
return (
RECOVERY_STRATEGY_PREDECESSOR,
{
"strategy": RECOVERY_STRATEGY_PREDECESSOR,
"evidence_status": "external_approval_missing",
"predecessor_package_required": True,
"database_downgrade_allowed": False,
"target_release_id": rollback_release_id,
},
rollback_images,
rollback_release_id,
)
if schema != IMAGE_IDENTITY_SCHEMA_V2:
raise ReleasePackageError("Unsupported image identity schema.")
expected_identity_fields = {
"schema_version",
"release_id",
"source_commit",
"source_manifest_sha256",
"application_images",
"infrastructure_images",
"recovery",
"identity_payload_sha256",
}
if set(image_identity) != expected_identity_fields:
raise ReleasePackageError(
"Image identity v2 fields are missing or unknown; legacy rollback fields are forbidden."
)
recovery = image_identity.get("recovery")
if not isinstance(recovery, dict):
raise ReleasePackageError("Image identity v2 has no recovery contract.")
strategy = recovery.get("strategy")
if strategy == RECOVERY_STRATEGY_PREDECESSOR:
expected_recovery_fields = {
"strategy",
"target_release_id",
"evidence_status",
"application_images",
}
if set(recovery) != expected_recovery_fields:
raise ReleasePackageError(
"Predecessor recovery fields are missing or unknown."
)
if recovery.get("evidence_status") != "external_approval_missing":
raise ReleasePackageError(
"Predecessor recovery must remain unapproved until external evidence exists."
)
rollback_release_id = _validate_identifier(
recovery.get("target_release_id"),
RELEASE_ID_RE,
label="recovery target release_id",
)
rollback_images = _validate_image_map(
recovery.get("application_images"),
APPLICATION_IMAGE_KEYS,
label="recovery.application_images",
)
if rollback_images == application_images:
raise ReleasePackageError(
"Predecessor recovery image identity must differ from the install image identity."
)
return (
strategy,
{
"strategy": strategy,
"evidence_status": "external_approval_missing",
"predecessor_package_required": True,
"database_downgrade_allowed": False,
"target_release_id": rollback_release_id,
},
rollback_images,
rollback_release_id,
)
if strategy == RECOVERY_STRATEGY_FIRST_RELEASE:
expected_recovery_fields = {
"strategy",
"evidence_status",
"first_release_only",
"predecessor_package_required",
"database_downgrade_allowed",
"required_restore_scopes",
"required_postgres_databases",
"required_storage_targets",
}
if set(recovery) != expected_recovery_fields:
raise ReleasePackageError(
"First-release snapshot recovery fields are missing or unknown; a predecessor is forbidden."
)
if (
recovery.get("evidence_status") != "runtime_signed_evidence_required"
or recovery.get("first_release_only") is not True
or recovery.get("predecessor_package_required") is not False
or recovery.get("database_downgrade_allowed") is not False
or recovery.get("required_restore_scopes")
!= list(FIRST_RELEASE_REQUIRED_RESTORE_SCOPES)
or recovery.get("required_postgres_databases")
!= list(FIRST_RELEASE_REQUIRED_POSTGRES_DATABASES)
or recovery.get("required_storage_targets")
!= list(FIRST_RELEASE_REQUIRED_STORAGE_TARGETS)
):
raise ReleasePackageError(
"First-release snapshot recovery must declare the fixed fail-closed evidence and restore scope."
)
return strategy, dict(recovery), None, None
raise ReleasePackageError("Image identity v2 has an unsupported recovery strategy.")
def _recovery_contract(package: ValidatedPackage) -> dict[str, Any]:
first_release = package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
return {
"strategy": package.recovery_strategy,
"predecessor_package_required": not first_release,
"rollback_plan_available": not first_release,
"forward_plan_available": not first_release,
"rollback_target_release_id": package.rollback_release_id,
"required_restore_scopes": (
list(FIRST_RELEASE_REQUIRED_RESTORE_SCOPES) if first_release else []
),
"required_postgres_databases": (
list(FIRST_RELEASE_REQUIRED_POSTGRES_DATABASES) if first_release else []
),
"required_storage_targets": (
list(FIRST_RELEASE_REQUIRED_STORAGE_TARGETS) if first_release else []
),
"external_evidence_requirements": list(
FIRST_RELEASE_EXTERNAL_EVIDENCE_REQUIREMENTS
if first_release
else PREDECESSOR_EXTERNAL_EVIDENCE_REQUIREMENTS
),
"external_restore_executor_required": first_release,
"post_restore_current_release_operation": "install" if first_release else "forward",
}
def _assert_operation_supported_for_recovery(
package: ValidatedPackage,
*,
operation: str,
) -> None:
if (
package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
and operation in {"rollback", "forward"}
):
raise ReleasePackageError(
"First-release snapshot recovery forbids package rollback/forward; use the "
"externally approved snapshot-restore planner, then run a new full install."
)
def _validate_installable_manifest(
root: Path,
payload: Mapping[str, Any],
*,
source_commit: str,
application_images: Mapping[str, Mapping[str, str]],
) -> tuple[dict[str, Any], ...]:
if payload.get("schema_version") != INSTALLABLE_MANIFEST_SCHEMA:
raise ReleasePackageError("Unsupported installable release manifest schema.")
_verify_embedded_hash(
payload,
"manifest_payload_sha256",
label="installable release manifest",
)
candidate = payload.get("candidate")
if not isinstance(candidate, dict) or candidate.get("commit") != source_commit:
raise ReleasePackageError("Installable manifest is not bound to the release source commit.")
images = payload.get("images")
expected_roles = [role for role, _key in IMPORT_ROLE_ENV_KEYS]
if (
not isinstance(images, list)
or any(not isinstance(item, dict) for item in images)
or [item.get("role") for item in images] != expected_roles
):
raise ReleasePackageError("Installable manifest must contain the four application roles in fixed order.")
images_by_role = {item["role"]: item for item in images}
raw_plan = payload.get("docker_import_plan")
if not isinstance(raw_plan, dict) or raw_plan.get("schema_version") != DOCKER_IMPORT_PLAN_SCHEMA:
raise ReleasePackageError("Installable manifest has no supported docker_import_plan.")
if raw_plan.get("implicit_preloaded_images_allowed") is not False:
raise ReleasePackageError("Docker import plan must forbid implicit preloaded images.")
raw_imports = raw_plan.get("images")
if not isinstance(raw_imports, list) or len(raw_imports) != len(IMPORT_ROLE_ENV_KEYS):
raise ReleasePackageError("Docker import plan must contain exactly four application archives.")
normalized: list[dict[str, Any]] = []
seen_paths: set[str] = set()
seen_ids: set[str] = set()
for sequence, ((expected_role, expected_key), raw) in enumerate(
zip(IMPORT_ROLE_ENV_KEYS, raw_imports, strict=True),
start=1,
):
if not isinstance(raw, dict):
raise ReleasePackageError("Docker import plan entries must be objects.")
if (
raw.get("sequence") != sequence
or raw.get("role") != expected_role
or raw.get("environment_key") != expected_key
):
raise ReleasePackageError("Docker import plan role, sequence, or environment key drifted.")
image = images_by_role[expected_role]
if image.get("environment_key") != expected_key:
raise ReleasePackageError("Installable image environment key drifted.")
archive = raw.get("archive")
docker_archive = image.get("docker_archive")
if not isinstance(archive, dict) or archive != docker_archive:
raise ReleasePackageError("Docker import plan archive is not bound to its image record.")
if set(archive) != {"path", "sha256", "size_bytes"}:
raise ReleasePackageError("Docker archive record has unexpected or missing fields.")
relative = _safe_relative_path(archive.get("path"), label="Docker archive path")
digest = archive.get("sha256")
size = archive.get("size_bytes")
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
raise ReleasePackageError("Docker archive record has an invalid SHA-256.")
if not isinstance(size, int) or isinstance(size, bool) or size <= 0:
raise ReleasePackageError("Docker archive record has an invalid size.")
if relative in seen_paths:
raise ReleasePackageError("Docker import plan reuses an archive path.")
seen_paths.add(relative)
archive_path = _artifact_file(root, relative, label=f"{expected_role} Docker archive")
archive_stat = archive_path.stat()
if archive_stat.st_nlink != 1:
raise ReleasePackageError(f"{expected_role} Docker archive must not be hard linked.")
if archive_stat.st_size != size or file_sha256(archive_path) != digest:
raise ReleasePackageError(f"{expected_role} Docker archive size or SHA-256 mismatch.")
config_digest = image.get("config_digest")
manifest_digest = image.get("manifest_digest")
registry_reference = image.get("registry_reference")
expected_tag = raw.get("expected_repo_tag")
if (
not isinstance(expected_tag, str)
or not expected_tag
or len(expected_tag) > 512
or any(character.isspace() or ord(character) < 32 for character in expected_tag)
):
raise ReleasePackageError("Docker import plan has an unsafe expected repository tag.")
if not isinstance(config_digest, str) or not IMAGE_DIGEST_RE.fullmatch(config_digest):
raise ReleasePackageError("Installable image has an invalid config digest.")
if config_digest in seen_ids:
raise ReleasePackageError("Docker import plan reuses a config-based image ID.")
seen_ids.add(config_digest)
if not isinstance(manifest_digest, str) or not IMAGE_DIGEST_RE.fullmatch(manifest_digest):
raise ReleasePackageError("Installable image has an invalid OCI manifest digest.")
if not isinstance(registry_reference, str) or not IMAGE_REFERENCE_RE.fullmatch(
registry_reference
):
raise ReleasePackageError("Installable image has an invalid online registry reference.")
if (
raw.get("command_argv")
!= [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"image",
"load",
"--input",
relative,
]
or raw.get("expected_loaded_image_id") != config_digest
or raw.get("expected_oci_manifest_digest") != manifest_digest
or raw.get("offline_compose_override") != config_digest
or raw.get("registry_reference_for_online_release") != registry_reference
or raw.get("repo_digests_expected_after_load") != []
or image.get("offline_image_id") != config_digest
or image.get("image_ref") != expected_tag
):
raise ReleasePackageError("Docker import plan does not match the installable image identity.")
identity = application_images[expected_key]
if identity["reference"] != registry_reference or identity["digest"] != manifest_digest:
raise ReleasePackageError(
f"Docker import plan online identity does not match application_images.{expected_key}."
)
normalized.append(
{
"sequence": sequence,
"image_class": "application",
"role": expected_role,
"environment_key": expected_key,
"services": list(RELEASE_IMPORT_ROLE_ENV_SERVICES[sequence - 1][2]),
"archive_path": relative,
"archive_sha256": digest,
"archive_size_bytes": size,
"expected_repo_tag": expected_tag,
"expected_loaded_image_id": config_digest,
"expected_oci_manifest_digest": manifest_digest,
"expected_platform": "linux/amd64",
"tag_command_argv": [],
"offline_compose_override": config_digest,
"registry_reference_for_online_release": registry_reference,
}
)
return tuple(normalized)
def _validate_checksum_manifest(
root: Path,
payload: Mapping[str, Any],
*,
package_file_path: str,
checksum_file_path: str,
) -> dict[str, dict[str, Any]]:
if payload.get("schema_version") != CHECKSUM_SCHEMA:
raise ReleasePackageError("Unsupported checksum manifest schema.")
_verify_embedded_hash(payload, "manifest_payload_sha256", label="checksum manifest")
raw_items = payload.get("files")
if not isinstance(raw_items, list) or not raw_items:
raise ReleasePackageError("Checksum manifest must list package artifacts.")
records: dict[str, dict[str, Any]] = {}
casefolded: set[str] = set()
observed_order: list[str] = []
for raw in raw_items:
if not isinstance(raw, dict):
raise ReleasePackageError("Checksum manifest file entries must be objects.")
relative = _safe_relative_path(raw.get("path"), label="checksum entry path")
digest = raw.get("sha256")
size = raw.get("size_bytes")
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
raise ReleasePackageError(f"Checksum entry has invalid SHA-256: {relative}")
if not isinstance(size, int) or isinstance(size, bool) or size < 0:
raise ReleasePackageError(f"Checksum entry has invalid size: {relative}")
if relative in records or relative.casefold() in casefolded:
raise ReleasePackageError(f"Checksum manifest has a duplicate path: {relative}")
if relative in {package_file_path, checksum_file_path}:
raise ReleasePackageError("The package and checksum manifests must not checksum themselves.")
records[relative] = {"sha256": digest, "size_bytes": size}
casefolded.add(relative.casefold())
observed_order.append(relative)
if observed_order != sorted(observed_order, key=lambda item: item.encode("utf-8")):
raise ReleasePackageError("Checksum manifest file entries must be bytewise path sorted.")
for relative, record in records.items():
target = _artifact_file(root, relative, label=f"checksummed artifact {relative}")
if target.stat().st_size != record["size_bytes"]:
raise ReleasePackageError(f"Artifact size mismatch: {relative}")
if file_sha256(target) != record["sha256"]:
raise ReleasePackageError(f"Artifact checksum mismatch: {relative}")
actual_files: set[str] = set()
for path in root.rglob("*"):
if path.is_symlink():
raise ReleasePackageError(f"Release package contains a symbolic link: {path}")
if path.is_file():
actual_files.add(path.relative_to(root).as_posix())
elif not path.is_dir():
raise ReleasePackageError(f"Release package contains a non-regular entry: {path}")
expected_files = {package_file_path, checksum_file_path, *records}
if actual_files != expected_files:
missing = sorted(expected_files - actual_files)
unexpected = sorted(actual_files - expected_files)
raise ReleasePackageError(
"Release package file set differs from its checksum manifest: "
f"missing={missing}, unexpected={unexpected}"
)
return records
def _validate_source_manifest(payload: Mapping[str, Any]) -> str:
if payload.get("schema_version") != SOURCE_MANIFEST_SCHEMA:
raise ReleasePackageError("Unsupported release source manifest schema.")
_verify_embedded_hash(payload, "manifest_payload_sha256", label="release source manifest")
candidate = payload.get("candidate")
if not isinstance(candidate, dict):
raise ReleasePackageError("Release source manifest has no candidate identity.")
commit = _validate_identifier(candidate.get("commit"), COMMIT_RE, label="source commit")
if candidate.get("worktree_was_clean") is not True:
raise ReleasePackageError("Release source manifest is not from a clean candidate.")
files = payload.get("files")
if not isinstance(files, dict) or not isinstance(files.get("items"), list):
raise ReleasePackageError("Release source manifest has no file inventory.")
if files.get("count") != len(files["items"]):
raise ReleasePackageError("Release source manifest file count is inconsistent.")
return commit
def _source_candidate_tag(payload: Mapping[str, Any]) -> str:
candidate = payload.get("candidate")
tag = candidate.get("tag") if isinstance(candidate, Mapping) else None
if not isinstance(tag, str) or not tag.strip() or len(tag) > 255:
raise ReleasePackageError("Release source manifest has no valid candidate tag.")
return tag
def _validate_source_bound_tool(
source_manifest: Mapping[str, Any],
repo_root: Path,
relative: str,
*,
loaded_file: Path,
) -> None:
by_path: dict[str, Mapping[str, Any]] = {}
for raw in source_manifest["files"]["items"]:
if isinstance(raw, Mapping) and isinstance(raw.get("path"), str):
by_path[raw["path"]] = raw
record = by_path.get(relative)
if record is None:
raise ReleasePackageError(
f"Release source manifest does not bind required validator: {relative}"
)
expected = record.get("sha256")
if not isinstance(expected, str) or not SHA256_RE.fullmatch(expected):
raise ReleasePackageError(f"Release source validator hash is invalid: {relative}")
repository_copy = _artifact_file(
_validated_directory(repo_root, label="repository root"),
relative,
label=f"source-bound validator {relative}",
)
executable_copy = _validated_file(loaded_file, label=f"loaded validator {relative}")
if file_sha256(repository_copy) != expected or file_sha256(executable_copy) != expected:
raise ReleasePackageError(
f"Loaded validator differs from the frozen source manifest: {relative}"
)
def _validate_infrastructure_bundle(
*,
package_root: Path,
artifact_root: Path,
source_set_path: Path,
bundle_path: Path,
installable_manifest: Mapping[str, Any],
source_commit: str,
candidate_tag: str,
infrastructure_images: Mapping[str, Mapping[str, str]],
application_plan: tuple[dict[str, Any], ...],
) -> tuple[dict[str, Any], dict[str, Any], tuple[dict[str, Any], ...], dict[str, str]]:
source_set = _read_json(source_set_path, label="infrastructure source set")
included_bundle = _read_json(bundle_path, label="infrastructure image bundle")
module = _load_infrastructure_bundle_module()
try:
rebuilt = module.build_bundle(
artifact_root=artifact_root,
source_set=source_set,
application_manifest=installable_manifest,
)
except Exception as exc:
if isinstance(exc, ReleasePackageError):
raise
raise ReleasePackageError(
f"Infrastructure image bundle validation failed: {exc}"
) from exc
if rebuilt != included_bundle:
raise ReleasePackageError(
"Infrastructure image bundle differs from independent deterministic recomputation."
)
if (
included_bundle.get("schema_version") != INFRASTRUCTURE_BUNDLE_SCHEMA
or source_set.get("schema_version") != INFRASTRUCTURE_SOURCE_SET_SCHEMA
or included_bundle.get("candidate")
!= {"commit": source_commit, "tag": candidate_tag}
):
raise ReleasePackageError("Infrastructure bundle candidate identity is invalid.")
raw_imports = included_bundle.get("docker_import_plan", {}).get("images")
raw_images = included_bundle.get("infrastructure_images")
if (
not isinstance(raw_imports, list)
or not isinstance(raw_images, list)
or len(raw_imports) != 7
or len(raw_images) != 7
):
raise ReleasePackageError("Infrastructure bundle must contain exactly seven images.")
normalized = [dict(item) for item in application_plan]
for offset, ((role, environment_key, services), raw_import, raw_image) in enumerate(
zip(RELEASE_IMPORT_ROLE_ENV_SERVICES[4:], raw_imports, raw_images, strict=True),
start=5,
):
if not isinstance(raw_import, Mapping) or not isinstance(raw_image, Mapping):
raise ReleasePackageError("Infrastructure image records must be objects.")
if (
raw_import.get("sequence") != offset - 4
or raw_import.get("role") != role
or raw_import.get("environment_key") != environment_key
or raw_image.get("role") != role
or raw_image.get("environment_key") != environment_key
or raw_image.get("services") != list(services)
):
raise ReleasePackageError("Infrastructure role, order, or service mapping drifted.")
archive = raw_import.get("archive")
if not isinstance(archive, Mapping) or set(archive) != {
"path",
"sha256",
"size_bytes",
}:
raise ReleasePackageError("Infrastructure Docker archive record is invalid.")
relative = _safe_relative_path(
archive.get("path"), label=f"{role} Docker archive path"
)
archive_path = _artifact_file(
artifact_root, relative, label=f"{role} Docker archive"
)
if (
archive_path.stat().st_nlink != 1
or archive_path.stat().st_size != archive.get("size_bytes")
or file_sha256(archive_path) != archive.get("sha256")
):
raise ReleasePackageError(f"{role} Docker archive is not immutable and bound.")
manifest_digest = raw_image.get("manifest_digest")
config_digest = raw_image.get("config_digest")
canonical_reference = raw_image.get("canonical_reference")
expected_tag = raw_image.get("expected_repo_tag")
identity = infrastructure_images[environment_key]
expected_tag_command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"image",
"tag",
config_digest,
expected_tag,
]
if (
not isinstance(manifest_digest, str)
or not IMAGE_DIGEST_RE.fullmatch(manifest_digest)
or not isinstance(config_digest, str)
or not IMAGE_DIGEST_RE.fullmatch(config_digest)
or not isinstance(canonical_reference, str)
or not IMAGE_REFERENCE_RE.fullmatch(canonical_reference)
or identity.get("reference") != canonical_reference
or identity.get("digest") != manifest_digest
or raw_import.get("expected_loaded_image_id") != config_digest
or raw_import.get("expected_manifest_digest") != manifest_digest
or raw_import.get("expected_platform") != "linux/amd64"
or raw_import.get("offline_compose_override") != config_digest
or raw_import.get("tag_command_argv") != expected_tag_command
):
raise ReleasePackageError(
f"Infrastructure import plan does not match image identity: {role}"
)
normalized.append(
{
"sequence": offset,
"image_class": "infrastructure",
"role": role,
"environment_key": environment_key,
"services": list(services),
"archive_path": relative,
"archive_package_path": archive_path.relative_to(package_root).as_posix(),
"archive_sha256": archive["sha256"],
"archive_size_bytes": archive["size_bytes"],
"expected_repo_tag": expected_tag,
"expected_loaded_image_id": config_digest,
"expected_oci_manifest_digest": manifest_digest,
"expected_platform": "linux/amd64",
"tag_command_argv": expected_tag_command,
"offline_compose_override": config_digest,
"registry_reference_for_online_release": canonical_reference,
}
)
if len(normalized) != len(RELEASE_IMPORT_ROLE_ENV_SERVICES):
raise ReleasePackageError("Release import plan does not cover exactly eleven images.")
for field in (
"role",
"environment_key",
"archive_package_path",
"expected_loaded_image_id",
"expected_oci_manifest_digest",
):
values = [str(item[field]).casefold() for item in normalized]
if len(values) != len(set(values)):
raise ReleasePackageError(f"Release import plan repeats or collides on {field}.")
overrides = included_bundle.get("offline_compose_overrides")
expected_overrides = {
item["environment_key"]: item["expected_loaded_image_id"]
for item in normalized
}
if overrides != expected_overrides:
raise ReleasePackageError("Infrastructure bundle offline override map is incomplete.")
return source_set, included_bundle, tuple(normalized), expected_overrides
def _validate_third_party_materials(
*,
input_path: Path,
manifest_path: Path,
receipt_path: Path,
source_commit: str,
candidate_tag: str,
source_manifest_sha256: str,
release_import_plan: tuple[dict[str, Any], ...],
installable_manifest: Mapping[str, Any],
infrastructure_bundle: Mapping[str, Any],
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], str]:
materials_input = _read_json(input_path, label="third-party materials input")
if materials_input.get("schema_version") != THIRD_PARTY_INPUT_SCHEMA:
raise ReleasePackageError("Unsupported third-party materials input schema.")
profile = materials_input.get("profile")
if profile not in {"internal-hosted", "redistributable"}:
raise ReleasePackageError("Third-party materials profile is invalid.")
if manifest_path.name != "materials-manifest.json":
raise ReleasePackageError("Third-party materials manifest must use its fixed name.")
module = _load_third_party_materials_module()
try:
recomputed_receipt = module.verify(input_path, manifest_path.parent, profile)
except Exception as exc:
raise ReleasePackageError(
f"Third-party materials independent verification failed: {exc}"
) from exc
materials_manifest = _read_json(manifest_path, label="third-party materials manifest")
included_receipt = _read_json(
receipt_path, label="third-party materials verification receipt"
)
if included_receipt != recomputed_receipt:
raise ReleasePackageError(
"Third-party verification receipt differs from independent recomputation."
)
if (
materials_manifest.get("schema_version") != THIRD_PARTY_MANIFEST_SCHEMA
or materials_manifest.get("result") != "PASS_TECHNICAL_MATERIALS_COMPLETE"
or materials_manifest.get("distribution_clearance_claimed") is not False
or included_receipt.get("schema_version") != THIRD_PARTY_VERIFICATION_SCHEMA
or included_receipt.get("result") != "PASS_INDEPENDENT_RECOMPUTATION"
or included_receipt.get("unresolved_occurrence_count") != 0
or included_receipt.get("distribution_clearance_claimed") is not False
):
raise ReleasePackageError(
"Third-party materials are incomplete or make an unsafe clearance claim."
)
candidate = materials_manifest.get("candidate")
if not isinstance(candidate, Mapping) or any(
candidate.get(key) != expected
for key, expected in (
("commit", source_commit),
("tag", candidate_tag),
("source_manifest_sha256", source_manifest_sha256),
)
):
raise ReleasePackageError("Third-party materials bind a different candidate.")
raw_images = candidate.get("images")
if not isinstance(raw_images, list) or len(raw_images) != 11:
raise ReleasePackageError("Third-party materials must bind all eleven images.")
expected_images = {
item["role"]: (
item["expected_oci_manifest_digest"],
item["expected_loaded_image_id"],
)
for item in release_import_plan
}
def report_fingerprint(raw: object, *, label: str) -> tuple[str, int]:
if not isinstance(raw, Mapping):
raise ReleasePackageError(f"{label} record is missing.")
digest = raw.get("sha256")
size = raw.get("size_bytes")
if not isinstance(digest, str) or not SHA256_RE.fullmatch(digest):
raise ReleasePackageError(f"{label} record has an invalid SHA-256.")
if not isinstance(size, int) or isinstance(size, bool) or size <= 0:
raise ReleasePackageError(f"{label} record has an invalid size.")
return digest, size
formal_reports: dict[str, dict[str, tuple[str, int]]] = {}
formal_image_groups = (
("application", installable_manifest.get("images")),
("infrastructure", infrastructure_bundle.get("infrastructure_images")),
)
for image_class, raw_group in formal_image_groups:
if not isinstance(raw_group, list):
raise ReleasePackageError(
f"Formal {image_class} image security-report inventory is missing."
)
for raw_image in raw_group:
if not isinstance(raw_image, Mapping):
raise ReleasePackageError(
f"Formal {image_class} image security-report entry is invalid."
)
role = raw_image.get("role")
if not isinstance(role, str) or role not in expected_images:
raise ReleasePackageError(
f"Formal {image_class} image security-report role is invalid."
)
if role in formal_reports:
raise ReleasePackageError(
f"Formal image security-report inventory repeats role: {role}"
)
expected_manifest_digest, expected_config_digest = expected_images[role]
if (
raw_image.get("manifest_digest") != expected_manifest_digest
or raw_image.get("config_digest") != expected_config_digest
):
raise ReleasePackageError(
f"Formal image security reports bind a different image: {role}"
)
formal_reports[role] = {
"sbom": report_fingerprint(
raw_image.get("sbom"), label=f"formal {role} SBOM"
),
"trivy_license": report_fingerprint(
raw_image.get("license_scan"),
label=f"formal {role} Trivy license report",
),
}
if set(formal_reports) != set(expected_images):
raise ReleasePackageError(
"Formal image security reports do not cover exactly all eleven release images."
)
observed_images: dict[str, tuple[object, object]] = {}
for item in raw_images:
if not isinstance(item, Mapping) or not isinstance(item.get("role"), str):
raise ReleasePackageError("Third-party image identity is invalid.")
role = item["role"]
if role in observed_images:
raise ReleasePackageError(f"Third-party image role is repeated: {role}")
observed_images[role] = (
item.get("manifest_digest"),
item.get("config_digest"),
)
expected_reports = formal_reports.get(role)
if expected_reports is None:
raise ReleasePackageError(
f"Third-party image role has no formal security reports: {role}"
)
for report_kind in ("sbom", "trivy_license"):
observed_report = report_fingerprint(
item.get(report_kind),
label=f"third-party {role} {report_kind}",
)
if observed_report != expected_reports[report_kind]:
raise ReleasePackageError(
"Third-party materials use a different raw image report than the "
f"formal build gate: {role} {report_kind}."
)
if observed_images != expected_images:
raise ReleasePackageError("Third-party image identities differ from the release plan.")
return materials_input, materials_manifest, included_receipt, profile
def _validate_compose_inputs(source_manifest: Mapping[str, Any], repo_root: Path) -> None:
root = _validated_directory(repo_root, label="repository root")
raw_items = source_manifest["files"]["items"]
by_path: dict[str, Mapping[str, Any]] = {}
for item in raw_items:
if not isinstance(item, dict) or not isinstance(item.get("path"), str):
raise ReleasePackageError("Release source manifest contains an invalid file record.")
path = _safe_relative_path(item["path"], label="release source file path")
if path in by_path:
raise ReleasePackageError(f"Duplicate release source file path: {path}")
by_path[path] = item
for relative in (
BASE_COMPOSE_RELATIVE,
PRODUCTION_COMPOSE_RELATIVE,
OFFLINE_COMPOSE_RELATIVE,
):
item = by_path.get(relative)
if not isinstance(item, Mapping):
raise ReleasePackageError(f"Release source manifest does not bind Compose input: {relative}")
expected = item.get("sha256")
if not isinstance(expected, str) or not SHA256_RE.fullmatch(expected):
raise ReleasePackageError(f"Release source manifest has an invalid Compose hash: {relative}")
target = _artifact_file(root, relative, label=f"repository Compose input {relative}")
if file_sha256(target) != expected:
raise ReleasePackageError(f"Repository Compose input differs from release manifest: {relative}")
def _validate_backup_proof(
payload: Mapping[str, Any],
*,
release_id: str,
rollback_release_id: str,
) -> None:
if payload.get("schema_version") != BACKUP_PROOF_SCHEMA:
raise ReleasePackageError("Unsupported backup proof schema.")
_verify_embedded_hash(payload, "proof_payload_sha256", label="backup proof")
if payload.get("release_id") != release_id or payload.get("rollback_target_release_id") != rollback_release_id:
raise ReleasePackageError("Backup proof is not bound to this release transition.")
if payload.get("status") != "unverified" or payload.get("restore_tested") is not False:
raise ReleasePackageError(
"Self-hashed backup material must remain explicitly unverified."
)
backup_id = payload.get("backup_id")
if backup_id is not None:
_validate_identifier(backup_id, APPROVAL_RE, label="backup proof backup_id")
backup_manifest_sha = payload.get("backup_manifest_sha256")
if backup_manifest_sha is not None and (
not isinstance(backup_manifest_sha, str)
or not SHA256_RE.fullmatch(backup_manifest_sha)
):
raise ReleasePackageError("Backup proof has an invalid backup manifest SHA-256.")
def _validate_compatibility_proof(
payload: Mapping[str, Any],
*,
release_id: str,
rollback_release_id: str,
rollback_images: Mapping[str, Mapping[str, str]],
) -> None:
if payload.get("schema_version") != COMPATIBILITY_PROOF_SCHEMA:
raise ReleasePackageError("Unsupported rollback compatibility proof schema.")
_verify_embedded_hash(payload, "proof_payload_sha256", label="rollback compatibility proof")
if payload.get("from_release_id") != release_id or payload.get("to_release_id") != rollback_release_id:
raise ReleasePackageError("Compatibility proof is not bound to this release transition.")
if payload.get("status") != "unverified":
raise ReleasePackageError(
"Self-hashed compatibility material must remain explicitly unverified."
)
if payload.get("approval_id") is not None:
raise ReleasePackageError(
"Untrusted compatibility material must not claim an approval identifier."
)
if payload.get("database_downgrade_required") is not False:
raise ReleasePackageError("Database downgrade is forbidden.")
if payload.get("current_schema_is_backward_compatible") not in {None, False}:
raise ReleasePackageError(
"Untrusted compatibility material must not claim schema compatibility."
)
approved = payload.get("approved_application_digests")
if approved is not None and approved != {} and approved != []:
raise ReleasePackageError(
"Untrusted compatibility material must not claim approved image digests."
)
def validate_release_package(
package_dir: Path,
*,
expected_package_sha256: str,
repo_root: Path = REPO_ROOT,
) -> ValidatedPackage:
if not SHA256_RE.fullmatch(expected_package_sha256):
raise ReleasePackageError("--expected-package-sha256 must be a lowercase SHA-256.")
root = _validated_directory(package_dir, label="release package directory")
package_path = _artifact_file(root, PACKAGE_FILE, label="release-package.json")
package_sha = file_sha256(package_path)
if package_sha != expected_package_sha256:
raise ReleasePackageError("release-package.json does not match the externally approved SHA-256.")
package = _read_json(package_path, label="release-package.json")
package_schema = package.get("schema_version")
if package_schema not in {PACKAGE_SCHEMA_V1, PACKAGE_SCHEMA_V2}:
raise ReleasePackageError("Unsupported release package schema.")
_verify_embedded_hash(package, "package_payload_sha256", label="release-package.json")
expected_package_fields = {
"schema_version",
"release_id",
"compose_project_name",
"deployment_scope",
"deployed_service_set",
"separately_deployed_artifacts",
"trusted_build_receipt_sha256",
"artifacts",
"package_payload_sha256",
}
if set(package) != expected_package_fields:
raise ReleasePackageError("release-package.json fields are missing or unknown.")
trusted_build_receipt_sha256 = package.get("trusted_build_receipt_sha256")
if not isinstance(trusted_build_receipt_sha256, str) or not SHA256_RE.fullmatch(
trusted_build_receipt_sha256
):
raise ReleasePackageError(
"release-package.json has no trusted build receipt raw SHA-256."
)
release_id = _validate_identifier(package.get("release_id"), RELEASE_ID_RE, label="release_id")
_validate_identifier(package.get("compose_project_name"), PROJECT_RE, label="compose_project_name")
if package.get("deployment_scope") != DEPLOYMENT_SCOPE:
raise ReleasePackageError("release-package.json must declare deployment_scope=backend_only.")
if package.get("deployed_service_set") != list(INSTALL_SERVICES):
raise ReleasePackageError(
"release-package.json deployed_service_set must equal the fixed backend install allowlist."
)
if package.get("separately_deployed_artifacts") != list(SEPARATELY_DEPLOYED_IMAGE_KEYS):
raise ReleasePackageError(
"release-package.json must identify Contract Web as a separately deployed artifact."
)
artifacts = package.get("artifacts")
artifact_names = set(artifacts) if isinstance(artifacts, dict) else set()
required_artifacts = (
V2_REQUIRED_ARTIFACTS
if package_schema == PACKAGE_SCHEMA_V2
else V1_REQUIRED_ARTIFACTS
)
if (
not isinstance(artifacts, dict)
or not set(required_artifacts).issubset(artifact_names)
or not artifact_names.issubset(
set(required_artifacts + OPTIONAL_EVIDENCE_ARTIFACTS)
)
):
raise ReleasePackageError(
"release-package.json artifact closure is incomplete or contains unknown pins."
)
pins = {name: _artifact_pin(package, name) for name in artifacts}
if len({pin.path.casefold() for pin in pins.values()}) != len(pins):
raise ReleasePackageError("Release package artifact paths must be unique.")
checksum_path = _artifact_file(root, pins["checksum_manifest"].path, label="checksum manifest")
checksum_sha = file_sha256(checksum_path)
if checksum_sha != pins["checksum_manifest"].sha256:
raise ReleasePackageError("Checksum manifest SHA-256 does not match release-package.json.")
checksum_payload = _read_json(checksum_path, label="checksum manifest")
if checksum_payload.get("release_id") != release_id:
raise ReleasePackageError("Checksum manifest release_id mismatch.")
checksum_records = _validate_checksum_manifest(
root,
checksum_payload,
package_file_path=PACKAGE_FILE,
checksum_file_path=pins["checksum_manifest"].path,
)
for name, pin in pins.items():
if name == "checksum_manifest":
continue
record = checksum_records.get(pin.path)
if record is None or record["sha256"] != pin.sha256:
raise ReleasePackageError(f"Checksum manifest does not bind artifact {name}.")
release_path = _artifact_file(root, pins["release_manifest"].path, label="release source manifest")
release_sha = file_sha256(release_path)
if release_sha != pins["release_manifest"].sha256:
raise ReleasePackageError("Release source manifest SHA-256 mismatch.")
release_manifest = _read_json(release_path, label="release source manifest")
source_commit = _validate_source_manifest(release_manifest)
_validate_compose_inputs(release_manifest, repo_root)
identity_path = _artifact_file(root, pins["image_identity"].path, label="image identity")
identity_sha = file_sha256(identity_path)
if identity_sha != pins["image_identity"].sha256:
raise ReleasePackageError("Image identity SHA-256 mismatch.")
image_identity = _read_json(identity_path, label="image identity")
_verify_embedded_hash(image_identity, "identity_payload_sha256", label="image identity")
if (
package_schema == PACKAGE_SCHEMA_V2
and image_identity.get("schema_version") != IMAGE_IDENTITY_SCHEMA_V2
):
raise ReleasePackageError("Release package v2 requires image identity v2.")
if (
image_identity.get("release_id") != release_id
or image_identity.get("source_commit") != source_commit
or image_identity.get("source_manifest_sha256") != release_sha
):
raise ReleasePackageError("Image identity is not bound to the release manifest and source commit.")
application_images = _validate_image_map(
image_identity.get("application_images"),
APPLICATION_IMAGE_KEYS,
label="application_images",
)
infrastructure_images = _validate_image_map(
image_identity.get("infrastructure_images"),
INFRASTRUCTURE_IMAGE_KEYS,
label="infrastructure_images",
)
(
recovery_strategy,
recovery_policy,
rollback_images,
rollback_release_id,
) = _validate_image_identity_recovery(
image_identity,
application_images=application_images,
)
installable_path = _artifact_file(
root,
pins["installable_manifest"].path,
label="installable release manifest",
)
installable_sha = file_sha256(installable_path)
if installable_sha != pins["installable_manifest"].sha256:
raise ReleasePackageError("Installable release manifest SHA-256 mismatch.")
installable_validator = _load_installable_validator_module()
try:
installable_manifest = installable_validator.validate_installable_release_manifest(
artifact_root=installable_path.parent,
manifest_path=installable_path,
expected_receipt_sha256=trusted_build_receipt_sha256,
repo=_validated_directory(repo_root, label="repository root"),
)
except installable_validator.ReleaseValidationError as exc:
raise ReleasePackageError(f"Strict installable manifest validation failed: {exc}") from exc
bundle_prefix = installable_path.parent.relative_to(root)
for record in installable_manifest.get("files", []):
if not isinstance(record, dict):
raise ReleasePackageError("Installable manifest file inventory is invalid.")
relative = _safe_relative_path(
record.get("path"),
label="installable manifest file path",
)
package_relative = (bundle_prefix / PurePosixPath(relative)).as_posix()
outer = checksum_records.get(package_relative)
if outer != {
"sha256": record.get("sha256"),
"size_bytes": record.get("size_bytes"),
}:
raise ReleasePackageError(
f"Outer checksum manifest does not bind installable file: {package_relative}"
)
receipt_record = installable_manifest.get("build_receipt")
if (
not isinstance(receipt_record, dict)
or receipt_record.get("sha256") != trusted_build_receipt_sha256
):
raise ReleasePackageError(
"Installable manifest is not bound to the trusted build receipt raw SHA-256."
)
build_receipt_path = _artifact_file(
installable_path.parent,
str(receipt_record["path"]),
label="trusted build receipt",
)
build_receipt = _read_json(build_receipt_path, label="trusted build receipt")
receipt_candidate = build_receipt.get("candidate")
source_record = (
receipt_candidate.get("source_manifest")
if isinstance(receipt_candidate, dict)
else None
)
if not isinstance(source_record, dict):
raise ReleasePackageError("Trusted build receipt has no source manifest record.")
receipt_source_path = _artifact_file(
installable_path.parent,
str(source_record.get("path")),
label="trusted build receipt source manifest",
)
if receipt_source_path != release_path or source_record.get("sha256") != release_sha:
raise ReleasePackageError(
"Release package source manifest is not the build receipt source manifest."
)
application_import_plan = tuple(
{
**item,
"archive_package_path": (
bundle_prefix / PurePosixPath(item["archive_path"])
).as_posix(),
}
for item in _validate_installable_manifest(
installable_path.parent,
installable_manifest,
source_commit=source_commit,
application_images=application_images,
)
)
docker_import_plan = application_import_plan
offline_image_overrides = {
item["environment_key"]: item["offline_compose_override"]
for item in application_import_plan
}
infrastructure_source_set: dict[str, Any] | None = None
infrastructure_source_set_sha: str | None = None
infrastructure_bundle: dict[str, Any] | None = None
infrastructure_bundle_sha: str | None = None
third_party_input: dict[str, Any] | None = None
third_party_input_sha: str | None = None
third_party_manifest: dict[str, Any] | None = None
third_party_manifest_sha: str | None = None
third_party_receipt: dict[str, Any] | None = None
third_party_receipt_sha: str | None = None
third_party_profile: str | None = None
if package_schema == PACKAGE_SCHEMA_V2:
infrastructure_module = _load_infrastructure_bundle_module()
third_party_module = _load_third_party_materials_module()
snapshot_module = _load_snapshot_restore_planner_module()
for relative, loaded in (
(
"yuqei-infra/scripts/build-infrastructure-image-bundle.py",
Path(infrastructure_module.__file__),
),
(
"yuqei-infra/scripts/build-third-party-materials.py",
Path(third_party_module.__file__),
),
(
"yuqei-infra/release-trust/third-party-materials-baseline.json",
Path(third_party_module.REPOSITORY_TRUST_BASELINE_PATH),
),
(
"yuqei-infra/deploy/plan-first-release-snapshot-restore.py",
Path(snapshot_module.__file__),
),
):
_validate_source_bound_tool(
release_manifest,
repo_root,
relative,
loaded_file=loaded,
)
candidate = installable_manifest.get("candidate")
candidate_tag = candidate.get("tag") if isinstance(candidate, Mapping) else None
if not isinstance(candidate_tag, str) or not candidate_tag:
raise ReleasePackageError("Installable manifest candidate tag is invalid.")
infrastructure_source_path = _artifact_file(
root,
pins["infrastructure_source_set"].path,
label="infrastructure source set",
)
infrastructure_bundle_path = _artifact_file(
root,
pins["infrastructure_image_bundle"].path,
label="infrastructure image bundle",
)
infrastructure_source_set_sha = file_sha256(infrastructure_source_path)
infrastructure_bundle_sha = file_sha256(infrastructure_bundle_path)
(
infrastructure_source_set,
infrastructure_bundle,
docker_import_plan,
offline_image_overrides,
) = _validate_infrastructure_bundle(
package_root=root,
artifact_root=infrastructure_source_path.parent,
source_set_path=infrastructure_source_path,
bundle_path=infrastructure_bundle_path,
installable_manifest=installable_manifest,
source_commit=source_commit,
candidate_tag=candidate_tag,
infrastructure_images=infrastructure_images,
application_plan=application_import_plan,
)
third_party_input_path = _artifact_file(
root,
pins["third_party_materials_input"].path,
label="third-party materials input",
)
third_party_manifest_path = _artifact_file(
root,
pins["third_party_materials_manifest"].path,
label="third-party materials manifest",
)
third_party_receipt_path = _artifact_file(
root,
pins["third_party_materials_verification_receipt"].path,
label="third-party verification receipt",
)
third_party_input_sha = file_sha256(third_party_input_path)
third_party_manifest_sha = file_sha256(third_party_manifest_path)
third_party_receipt_sha = file_sha256(third_party_receipt_path)
(
third_party_input,
third_party_manifest,
third_party_receipt,
third_party_profile,
) = _validate_third_party_materials(
input_path=third_party_input_path,
manifest_path=third_party_manifest_path,
receipt_path=third_party_receipt_path,
source_commit=source_commit,
candidate_tag=candidate_tag,
source_manifest_sha256=release_sha,
release_import_plan=docker_import_plan,
installable_manifest=installable_manifest,
infrastructure_bundle=infrastructure_bundle,
)
backup_proof: dict[str, Any] | None = None
backup_sha: str | None = None
if recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE and (
"backup_proof" in pins or "compatibility_proof" in pins
):
raise ReleasePackageError(
"First-release snapshot recovery forbids predecessor backup/compatibility artifacts; "
"runtime signed snapshot evidence is required."
)
if "backup_proof" in pins:
if rollback_release_id is None:
raise ReleasePackageError("Backup proof requires a predecessor release target.")
backup_path = _artifact_file(root, pins["backup_proof"].path, label="backup proof")
backup_sha = file_sha256(backup_path)
if backup_sha != pins["backup_proof"].sha256:
raise ReleasePackageError("Backup proof SHA-256 mismatch.")
backup_proof = _read_json(backup_path, label="backup proof")
_validate_backup_proof(
backup_proof,
release_id=release_id,
rollback_release_id=rollback_release_id,
)
compatibility_proof: dict[str, Any] | None = None
compatibility_sha: str | None = None
if "compatibility_proof" in pins:
if rollback_release_id is None or rollback_images is None:
raise ReleasePackageError(
"Rollback compatibility proof requires a predecessor release target."
)
compatibility_path = _artifact_file(
root, pins["compatibility_proof"].path, label="rollback compatibility proof"
)
compatibility_sha = file_sha256(compatibility_path)
if compatibility_sha != pins["compatibility_proof"].sha256:
raise ReleasePackageError("Rollback compatibility proof SHA-256 mismatch.")
compatibility_proof = _read_json(
compatibility_path,
label="rollback compatibility proof",
)
_validate_compatibility_proof(
compatibility_proof,
release_id=release_id,
rollback_release_id=rollback_release_id,
rollback_images=rollback_images,
)
return ValidatedPackage(
root=root,
package_schema_version=package_schema,
package=package,
package_sha256=package_sha,
checksum_manifest=checksum_payload,
checksum_manifest_sha256=checksum_sha,
release_manifest=release_manifest,
release_manifest_sha256=release_sha,
image_identity=image_identity,
image_identity_sha256=identity_sha,
installable_manifest=installable_manifest,
installable_manifest_sha256=installable_sha,
installable_root=installable_path.parent,
trusted_build_receipt_sha256=trusted_build_receipt_sha256,
docker_import_plan=docker_import_plan,
offline_image_overrides=offline_image_overrides,
infrastructure_source_set=infrastructure_source_set,
infrastructure_source_set_sha256=infrastructure_source_set_sha,
infrastructure_bundle=infrastructure_bundle,
infrastructure_bundle_sha256=infrastructure_bundle_sha,
third_party_materials_input=third_party_input,
third_party_materials_input_sha256=third_party_input_sha,
third_party_materials_manifest=third_party_manifest,
third_party_materials_manifest_sha256=third_party_manifest_sha,
third_party_verification_receipt=third_party_receipt,
third_party_verification_receipt_sha256=third_party_receipt_sha,
third_party_profile=third_party_profile,
backup_proof=backup_proof,
backup_proof_sha256=backup_sha,
compatibility_proof=compatibility_proof,
compatibility_proof_sha256=compatibility_sha,
application_images=application_images,
infrastructure_images=infrastructure_images,
recovery_strategy=recovery_strategy,
recovery_policy=recovery_policy,
rollback_images=rollback_images,
rollback_release_id=rollback_release_id,
)
def parse_env_file(path: Path) -> dict[str, str]:
compose_gate = _load_compose_gate_module()
try:
_target, values = compose_gate.parse_strict_env(path)
except compose_gate.ComposeGateError as exc:
raise ReleasePackageError(str(exc)) from exc
return values
def _fixed_runtime_environment(overrides: Mapping[str, str] | None = None) -> dict[str, str]:
environment = {
"HOME": FIXED_DOCKER_HOME,
"LANG": "C.UTF-8",
"LC_ALL": "C.UTF-8",
}
if overrides:
reserved = {
"PATH",
"DOCKER_HOST",
"DOCKER_CONTEXT",
"DOCKER_CONFIG",
"COMPOSE_FILE",
"COMPOSE_PATH_SEPARATOR",
"COMPOSE_PROFILES",
"COMPOSE_PROJECT_NAME",
}
conflict = sorted(reserved.intersection(overrides))
if conflict:
raise ReleasePackageError(
f"Runtime overrides contain forbidden ambient-control keys: {', '.join(conflict)}"
)
environment.update(overrides)
return environment
def _run_production_preflight(env_values: dict[str, str]) -> dict[str, Any]:
preflight = _load_preflight_module()
try:
environment_checks = preflight.validate_environment(dict(env_values))
file_checks = preflight.validate_files(dict(env_values))
except Exception as exc:
raise ReleasePackageError("Production preflight raised while validating protected inputs.") from exc
def bound_record(check: Any) -> dict[str, Any]:
record: dict[str, Any] = {"key": check.key, "status": check.status}
details = getattr(check, "details", None)
if details:
record["details"] = details
return record
environment_records = [bound_record(check) for check in environment_checks]
file_records = [bound_record(check) for check in file_checks]
failed = [
f"environment:{record['key']}"
for record in environment_records
if record["status"] == "fail"
] + [
f"files:{record['key']}"
for record in file_records
if record["status"] == "fail"
]
if failed:
raise ReleasePackageError(
"Production preflight failed before any state write: " + ", ".join(failed)
)
summary: dict[str, Any] = {
"schema_version": PREFLIGHT_BINDING_SCHEMA,
"status": "passed",
"environment_checks": environment_records,
"file_checks": file_records,
}
summary["summary_sha256"] = canonical_sha256(summary)
return summary
def _tls_material_binding(preflight_summary: Mapping[str, Any]) -> dict[str, str]:
records = preflight_summary.get("file_checks")
if not isinstance(records, list):
raise ReleasePackageError("Production preflight has no TLS file-check inventory.")
matches = [
record
for record in records
if isinstance(record, dict) and record.get("key") == "tls_material_binding"
]
if len(matches) != 1 or matches[0].get("status") != "pass":
raise ReleasePackageError("Production preflight has no unique passing TLS material binding.")
details = matches[0].get("details")
expected_fields = {
"certificate_sha256",
"private_key_sha256",
"trusted_ca_sha256",
"server_name",
}
if not isinstance(details, dict) or set(details) != expected_fields:
raise ReleasePackageError("Production preflight TLS material binding is incomplete.")
for field in (
"certificate_sha256",
"private_key_sha256",
"trusted_ca_sha256",
):
value = details.get(field)
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
raise ReleasePackageError(f"Production preflight has an invalid TLS {field}.")
server_name = details.get("server_name")
if (
not isinstance(server_name, str)
or not server_name
or server_name != server_name.strip()
or any(ord(character) < 32 or ord(character) == 127 for character in server_name)
):
raise ReleasePackageError("Production preflight has an invalid TLS server name.")
return {field: str(details[field]) for field in sorted(expected_fields)}
def _parse_json_text(value: object, *, label: str, max_bytes: int = MAX_JSON_BYTES) -> dict[str, Any]:
raw = _output_bytes(value)
if len(raw) > max_bytes:
raise ReleasePackageError(f"{label} exceeds the maximum allowed size.")
try:
payload = json.loads(
raw.decode("utf-8"),
object_pairs_hook=_reject_duplicate_pairs,
parse_constant=_reject_json_constant,
)
except ReleasePackageError:
raise
except (UnicodeError, ValueError, TypeError) as exc:
raise ReleasePackageError(f"{label} is not strict UTF-8 JSON.") from exc
if not isinstance(payload, dict):
raise ReleasePackageError(f"{label} must be a JSON object.")
return payload
def _docker_daemon_identity(
*,
runner: Callable[..., Any],
repo_root: Path,
) -> dict[str, str]:
command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"info",
"--format",
"{{json .}}",
]
try:
completed = runner(
command,
check=False,
capture_output=True,
text=True,
shell=False,
cwd=str(repo_root),
env=_fixed_runtime_environment(),
timeout=30,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise ReleasePackageError("Cannot query the fixed local Docker daemon identity.") from exc
if int(completed.returncode) != 0:
raise ReleasePackageError("Cannot query the fixed local Docker daemon identity.")
payload = _parse_json_text(completed.stdout, label="Docker daemon identity")
fields = {
"daemon_id": payload.get("ID"),
"daemon_name": payload.get("Name"),
"docker_root_dir": payload.get("DockerRootDir"),
"server_version": payload.get("ServerVersion"),
}
if any(not isinstance(value, str) or not value.strip() for value in fields.values()):
raise ReleasePackageError("Docker daemon identity is incomplete.")
identity: dict[str, str] = {
"docker_cli": str(DOCKER_CLI),
"docker_host": LOCAL_DOCKER_HOST,
**{key: str(value) for key, value in fields.items()},
}
identity["identity_sha256"] = canonical_sha256(identity)
return identity
def _collect_runtime_evidence(
*,
env_file: Path,
repo_root: Path,
inspection_runner: Callable[..., Any],
) -> RuntimeEvidence:
env_target = _validated_file(env_file, label="production env file", max_bytes=MAX_ENV_BYTES)
env_values = parse_env_file(env_target)
preflight_summary = _run_production_preflight(env_values)
compose_gate = _load_compose_gate_module()
try:
compose_report = compose_gate.verify_production_compose(
env_file=env_target,
repo_root=repo_root,
runner=inspection_runner,
)
except compose_gate.ComposeGateError as exc:
raise ReleasePackageError(f"Production Compose merge gate failed: {exc}") from exc
compose_summary = dict(compose_report)
compose_summary["summary_sha256"] = canonical_sha256(compose_summary)
daemon_identity = _docker_daemon_identity(runner=inspection_runner, repo_root=repo_root)
return RuntimeEvidence(
env_file=env_target,
env_values=env_values,
env_sha256=file_sha256(env_target),
preflight_summary=preflight_summary,
compose_gate_summary=compose_summary,
daemon_identity=daemon_identity,
)
def _run_offline_execution_gate(
*,
env_file: Path,
repo_root: Path,
compose_files: tuple[Path, Path, Path],
project_name: str,
offline_image_overrides: Mapping[str, str],
inspection_runner: Callable[..., Any],
) -> dict[str, Any]:
compose_gate = _load_compose_gate_module()
env_values = parse_env_file(env_file)
env_values.update(offline_image_overrides)
child_env = _fixed_runtime_environment(offline_image_overrides)
cases: list[dict[str, Any]] = []
try:
for profile_case in compose_gate.PROFILE_CASES:
command = compose_gate._command(
project_name=project_name,
env_file=env_file,
compose_files=compose_files,
profile_case=profile_case,
)
payload = compose_gate._run_config(
command,
repo_root=repo_root,
runner=inspection_runner,
child_env=child_env,
label=f"offline-execution/{profile_case.name}",
)
cases.append(
compose_gate._validate_model(
payload,
env_values=env_values,
mode="offline",
profile_case=profile_case,
)
)
except compose_gate.ComposeGateError as exc:
raise ReleasePackageError(f"Offline execution Compose gate failed: {exc}") from exc
summary: dict[str, Any] = {
"schema_version": "yuqei.offline-compose-execution-gate.v1",
"status": "passed",
"docker_host": LOCAL_DOCKER_HOST,
"project_name": project_name,
"offline_image_overrides": dict(offline_image_overrides),
"cases": cases,
}
summary["summary_sha256"] = canonical_sha256(summary)
return summary
def _assert_safe_compose_command(command: list[str], *, operation: str) -> None:
if command[:4] != [str(DOCKER_CLI), "--host", LOCAL_DOCKER_HOST, "compose"]:
raise ReleasePackageError("Only the fixed local Docker Compose target is allowed.")
if any("\x00" in item or "\r" in item or "\n" in item for item in command):
raise ReleasePackageError("Deployment argv contains a forbidden control character.")
if command.count("up") != 1 or command.count("--pull") != 1 or "--no-build" not in command:
raise ReleasePackageError("Offline execution requires compose up --pull never --no-build.")
pull_index = command.index("--pull")
if pull_index + 1 >= len(command) or command[pull_index + 1] != "never":
raise ReleasePackageError("Offline execution requires the exact --pull never policy.")
lowered = [item.casefold() for item in command]
for index, token in enumerate(lowered):
if token in FORBIDDEN_COMMAND_TOKENS:
pull_pair = token == "pull" and index > 0 and lowered[index - 1] == "--pull"
if not pull_pair:
raise ReleasePackageError(f"Forbidden deployment command token: {command[index]}")
if "edge" in lowered or any("dns" in item for item in lowered):
raise ReleasePackageError("Automatic edge or DNS operations are forbidden.")
if operation in {"rollback", "forward"} and "--no-deps" not in command:
raise ReleasePackageError("Rollback and forward must switch application services with --no-deps.")
expected_services = RUNTIME_SERVICES if operation in {"rollback", "forward"} else INSTALL_SERVICES
if command[-len(expected_services) :] != list(expected_services):
raise ReleasePackageError("Deployment command service set is not the fixed operation allowlist.")
def _compose_command(
*,
operation: str,
project_name: str,
env_file: Path,
repo_root: Path,
compose_files: tuple[Path, Path, Path] | None = None,
) -> list[str]:
if compose_files is None:
root = _validated_directory(repo_root, label="repository root")
base = _artifact_file(root, BASE_COMPOSE_RELATIVE, label="base Compose file")
production = _artifact_file(
root,
PRODUCTION_COMPOSE_RELATIVE,
label="production Compose file",
)
offline = _artifact_file(
root,
OFFLINE_COMPOSE_RELATIVE,
label="offline Compose override",
)
else:
base, production, offline = compose_files
for label, path in (
("base Compose snapshot", base),
("production Compose snapshot", production),
("offline Compose snapshot", offline),
):
_validated_file(path, label=label, max_bytes=MAX_JSON_BYTES)
command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"compose",
"--project-name",
project_name,
"--env-file",
str(env_file),
"-f",
str(base),
"-f",
str(production),
"-f",
str(offline),
"up",
"-d",
"--pull",
"never",
"--no-build",
]
if operation in {"rollback", "forward"}:
command.append("--no-deps")
command.extend(RUNTIME_SERVICES)
else:
command.extend(INSTALL_SERVICES)
_assert_safe_compose_command(command, operation=operation)
return command
def _assert_fixed_subprocess_argv(command: object) -> list[str]:
if (
not isinstance(command, list)
or any(not isinstance(item, str) for item in command)
or command[:3] != [str(DOCKER_CLI), "--host", LOCAL_DOCKER_HOST]
or any("\x00" in item or "\r" in item or "\n" in item for item in command)
):
raise ReleasePackageError("Fixed runner received an unsafe Docker argv.")
if command[3:] == ["info", "--format", "{{json .}}"]:
return command
if len(command) >= 5 and command[3] == "image":
tail = command[4:]
if (
len(tail) == 3
and tail[:2] == ["load", "--input"]
and Path(tail[2]).is_absolute()
):
return command
if (
len(tail) == 3
and tail[0] == "tag"
and IMAGE_DIGEST_RE.fullmatch(tail[1])
and tail[2]
):
return command
if (
len(tail) == 4
and tail[:2] == ["inspect", "--format"]
and tail[2] in {"{{.Id}}", "{{json .}}"}
and tail[3]
):
return command
raise ReleasePackageError("Fixed runner rejected a Docker image command.")
if len(command) == 7 and command[3:6] == ["inspect", "--format", "{{json .}}"]:
return command
if len(command) < 8 or command[3] != "compose":
raise ReleasePackageError("Fixed runner only permits Docker info, image, inspect, and Compose.")
lowered = [item.casefold() for item in command]
mutating = {"down", "rm", "pull", "build", "push", "run", "exec", "restart", "kill"}
if any(item in mutating for item in lowered):
pull_never = any(
lowered[index] == "pull"
and index > 0
and lowered[index - 1] == "--pull"
for index in range(len(lowered))
)
forbidden = [
item
for index, item in enumerate(lowered)
if item in mutating
and not (item == "pull" and index > 0 and lowered[index - 1] == "--pull")
]
if forbidden or ("pull" in lowered and not pull_never):
raise ReleasePackageError("Fixed runner rejected a mutating Compose command.")
if "config" in command:
if command[-3:] != ["config", "--format", "json"]:
raise ReleasePackageError("Fixed runner permits only Compose config JSON.")
return command
if "up" in command:
operation = "forward" if "--no-deps" in command else "install"
_assert_safe_compose_command(command, operation=operation)
return command
if command[-4:] == ["ps", "--all", "--format", "json"]:
return command
raise ReleasePackageError("Fixed runner rejected an unsupported Compose command.")
def _fixed_subprocess_runner(command: list[str], **kwargs: Any) -> Any:
safe_command = _assert_fixed_subprocess_argv(command)
if (
kwargs.get("shell") is not False
or kwargs.get("check") is not False
or kwargs.get("capture_output") is not True
):
raise ReleasePackageError("Fixed runner requires non-shell captured execution.")
timeout = kwargs.get("timeout")
if (
not isinstance(timeout, (int, float))
or isinstance(timeout, bool)
or timeout <= 0
or timeout > 1800
):
raise ReleasePackageError("Fixed runner timeout is outside the approved bound.")
cwd = kwargs.get("cwd")
if cwd is not None and (not isinstance(cwd, str) or not Path(cwd).is_absolute()):
raise ReleasePackageError("Fixed runner cwd must be an absolute path.")
environment = kwargs.get("env")
allowed_env = {
"HOME",
"LANG",
"LC_ALL",
"AI_PLATFORM_RELEASE_IMAGE_DIGEST",
*(key for _role, key, _services in RELEASE_IMPORT_ROLE_ENV_SERVICES),
}
if (
not isinstance(environment, Mapping)
or not set(environment).issubset(allowed_env)
or environment.get("HOME") != FIXED_DOCKER_HOME
or environment.get("LANG") != "C.UTF-8"
or environment.get("LC_ALL") != "C.UTF-8"
or any(
not isinstance(value, str)
or "\x00" in value
or "\r" in value
or "\n" in value
for value in environment.values()
)
):
raise ReleasePackageError("Fixed runner environment is not the approved clean environment.")
return subprocess.run(safe_command, **kwargs)
def _parse_json_value(value: object, *, label: str) -> Any:
raw = _output_bytes(value)
if len(raw) > MAX_JSON_BYTES:
raise ReleasePackageError(f"{label} exceeds the maximum allowed size.")
try:
return json.loads(
raw.decode("utf-8"),
object_pairs_hook=_reject_duplicate_pairs,
parse_constant=_reject_json_constant,
)
except (UnicodeError, ValueError, TypeError) as exc:
raise ReleasePackageError(f"{label} is not strict UTF-8 JSON.") from exc
def _builtin_post_apply_checker(
*,
plan: Mapping[str, Any],
command_argv: list[str],
runtime_environment: Mapping[str, str],
daemon_identity: Mapping[str, str],
) -> dict[str, Any]:
up_index = command_argv.index("up")
compose_prefix = command_argv[:up_index]
ps_command = [*compose_prefix, "ps", "--all", "--format", "json"]
expected_services = list(plan["deployed_service_set"])
service_image_ids: dict[str, str] = {}
for item in plan["docker_import_plan"]:
for service in item["services"]:
if service not in expected_services:
continue
if service in service_image_ids:
raise ReleasePackageError(
f"Post-apply image plan repeats service binding: {service}."
)
service_image_ids[service] = item["expected_loaded_image_id"]
if set(service_image_ids) != set(expected_services):
raise ReleasePackageError("Post-apply image plan does not cover every deployed service.")
deadline = time.monotonic() + POST_APPLY_WAIT_TIMEOUT_SECONDS
records: list[dict[str, Any]] = []
release_binding_verified = False
while True:
completed = _fixed_subprocess_runner(
ps_command,
check=False,
capture_output=True,
text=True,
shell=False,
cwd=str(Path(command_argv[command_argv.index("-f") + 1]).parent),
env=dict(runtime_environment),
timeout=60,
)
if int(completed.returncode) != 0:
raise ReleasePackageError("Built-in post-apply Compose inventory check failed.")
parsed = _parse_json_value(completed.stdout, label="Compose post-apply inventory")
if isinstance(parsed, dict):
rows = [parsed]
elif isinstance(parsed, list) and all(isinstance(item, dict) for item in parsed):
rows = parsed
else:
raise ReleasePackageError("Compose post-apply inventory has an invalid shape.")
by_service: dict[str, Mapping[str, Any]] = {}
for row in rows:
service = row.get("Service")
if not isinstance(service, str) or service in by_service:
raise ReleasePackageError(
"Compose post-apply inventory repeats or has an invalid service."
)
by_service[service] = row
unexpected = set(by_service).difference(expected_services)
if unexpected:
raise ReleasePackageError("Compose post-apply service inventory exceeds the plan.")
pending_services = set(expected_services).difference(by_service)
current_records: list[dict[str, Any]] = []
current_release_binding_verified = False
for service in expected_services:
row = by_service.get(service)
if row is None:
continue
container_id = row.get("ID")
if not isinstance(container_id, str) or not container_id:
raise ReleasePackageError(f"Post-apply container ID is missing for {service}.")
inspect_command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"inspect",
"--format",
"{{json .}}",
container_id,
]
inspected = _fixed_subprocess_runner(
inspect_command,
check=False,
capture_output=True,
text=True,
shell=False,
env=dict(runtime_environment),
timeout=30,
)
if int(inspected.returncode) != 0:
raise ReleasePackageError(f"Cannot inspect post-apply container {service}.")
detail = _parse_json_text(inspected.stdout, label=f"{service} container inspect")
config = detail.get("Config")
state = detail.get("State")
labels = config.get("Labels") if isinstance(config, Mapping) else None
if (
not isinstance(config, Mapping)
or not isinstance(state, Mapping)
or not isinstance(labels, Mapping)
or labels.get("com.docker.compose.project") != plan["compose_project_name"]
or labels.get("com.docker.compose.service") != service
or detail.get("Image") != service_image_ids[service]
):
raise ReleasePackageError(
f"Post-apply project or image binding failed for {service}."
)
status = state.get("Status")
health_payload = state.get("Health")
health = (
health_payload.get("Status")
if isinstance(health_payload, Mapping)
else None
)
exit_code = state.get("ExitCode")
if service in ONE_SHOT_INSTALL_SERVICES:
if status == "exited":
if exit_code != 0:
raise ReleasePackageError(
f"Post-apply migration/init failed for {service}."
)
elif status in {"created", "running", "restarting"}:
pending_services.add(service)
else:
raise ReleasePackageError(
f"Post-apply migration/init entered an invalid state: {service}."
)
elif status == "running" and health == "healthy":
pass
elif status in {"created", "running", "restarting"} and health in {
None,
"starting",
"unhealthy",
}:
pending_services.add(service)
else:
raise ReleasePackageError(
f"Post-apply service entered an invalid state: {service}."
)
env_items = config.get("Env")
if service == "ai-platform-api":
if (
not isinstance(env_items, list)
or f"AI_PLATFORM_RELEASE_ID={plan['target_release_id']}" not in env_items
):
raise ReleasePackageError("AI runtime release identity is not bound.")
current_release_binding_verified = True
current_records.append(
{
"service": service,
"container_id_sha256": hashlib.sha256(
container_id.encode("utf-8")
).hexdigest(),
"status": status,
"health": health,
"exit_code": exit_code,
"image_id": detail.get("Image"),
}
)
if not pending_services:
records = current_records
release_binding_verified = current_release_binding_verified
break
if time.monotonic() >= deadline:
raise ReleasePackageError("Post-apply services did not become ready before timeout.")
time.sleep(POST_APPLY_POLL_INTERVAL_SECONDS)
if "ai-platform-api" in expected_services and not release_binding_verified:
raise ReleasePackageError("Post-apply release identity check did not run.")
payload: dict[str, Any] = {
"schema_version": POST_APPLY_SCHEMA,
"status": "verified",
"deployment_scope": DEPLOYMENT_SCOPE,
"operation": plan["operation"],
"target_release_id": plan["target_release_id"],
"compose_project_name": plan["compose_project_name"],
"deployed_service_set": expected_services,
"daemon_identity_sha256": daemon_identity["identity_sha256"],
"container_count": len(records),
"services": records,
"release_binding_verified": release_binding_verified,
}
payload["post_check_payload_sha256"] = canonical_sha256(payload)
return payload
def build_plan(
package: ValidatedPackage,
*,
operation: str,
env_file: Path,
repo_root: Path = REPO_ROOT,
inspection_runner: Callable[..., Any] = subprocess.run,
recovery_readiness_receipt: Mapping[str, Any] | None = None,
recovery_readiness_receipt_sha256: str | None = None,
) -> dict[str, Any]:
if operation not in CONFIRMATIONS:
raise ReleasePackageError("Operation must be install, rollback, or forward.")
_assert_operation_supported_for_recovery(package, operation=operation)
if not env_file.is_absolute():
raise ReleasePackageError("--env-file must be an absolute path.")
evidence = _collect_runtime_evidence(
env_file=env_file,
repo_root=repo_root,
inspection_runner=inspection_runner,
)
tls_binding = _tls_material_binding(evidence.preflight_summary)
env_target = evidence.env_file
env_values = evidence.env_values
project_name = package.package["compose_project_name"]
if env_values.get("COMPOSE_PROJECT_NAME") != project_name:
raise ReleasePackageError("Production env COMPOSE_PROJECT_NAME does not match the release package.")
if operation == "rollback":
if package.rollback_release_id is None or package.rollback_images is None:
raise ReleasePackageError("Rollback requires an approved predecessor identity.")
target_release_id = package.rollback_release_id
application_images = package.rollback_images
else:
target_release_id = package.package["release_id"]
application_images = package.application_images
environment_overrides = {
key: application_images[key]["reference"] for key in BACKEND_APPLICATION_IMAGE_KEYS
}
environment_overrides["AI_PLATFORM_RELEASE_IMAGE_DIGEST"] = application_images[
"AI_PLATFORM_API_IMAGE"
]["digest"]
if operation == "install":
environment_overrides.update(
{
key: package.infrastructure_images[key]["reference"]
for key in BACKEND_INSTALL_INFRASTRUCTURE_IMAGE_KEYS
}
)
import_plan = [dict(item) for item in package.docker_import_plan]
offline_image_overrides = (
dict(package.offline_image_overrides)
if operation in {"install", "forward"}
else {}
)
apply_blockers = _real_apply_blockers(
operation,
recovery_strategy=package.recovery_strategy,
package_schema_version=package.package_schema_version,
infrastructure_ready=len(package.docker_import_plan) == 11,
third_party_ready=package.third_party_verification_receipt is not None,
recovery_readiness_ready=recovery_readiness_receipt is not None,
)
source_root = _validated_directory(repo_root, label="repository root")
source_compose_files = (
_artifact_file(source_root, BASE_COMPOSE_RELATIVE, label="base Compose file"),
_artifact_file(
source_root,
PRODUCTION_COMPOSE_RELATIVE,
label="production Compose file",
),
_artifact_file(
source_root,
OFFLINE_COMPOSE_RELATIVE,
label="offline Compose override",
),
)
offline_execution_gate = _run_offline_execution_gate(
env_file=env_target,
repo_root=source_root,
compose_files=source_compose_files,
project_name=project_name,
offline_image_overrides=offline_image_overrides,
inspection_runner=inspection_runner,
)
command = _compose_command(
operation=operation,
project_name=project_name,
env_file=env_target,
repo_root=repo_root,
)
unsigned: dict[str, Any] = {
"schema_version": PLAN_SCHEMA,
"mode": "plan",
"operation": operation,
"deployment_scope": DEPLOYMENT_SCOPE,
"deployed_service_set": list(
RUNTIME_SERVICES if operation in {"rollback", "forward"} else INSTALL_SERVICES
),
"separately_deployed_artifacts": [
{
"image_key": key,
"reference": application_images[key]["reference"],
"digest": application_images[key]["digest"],
"deployment_scope": "separate_web_orchestrator",
"applied_by_this_operation": False,
}
for key in SEPARATELY_DEPLOYED_IMAGE_KEYS
],
"identity_bound_image_keys": list(APPLICATION_IMAGE_KEYS + INFRASTRUCTURE_IMAGE_KEYS),
"release_completion_claimed": False,
"package_schema_version": package.package_schema_version,
"release_id": package.package["release_id"],
"target_release_id": target_release_id,
"recovery_strategy": package.recovery_strategy,
"recovery_contract": _recovery_contract(package),
"recovery_readiness_receipt_sha256": recovery_readiness_receipt_sha256,
"recovery_readiness_status": (
RECOVERY_READINESS_RESULT
if recovery_readiness_receipt is not None
else "missing"
),
"package_manifest_sha256": package.package_sha256,
"release_manifest_sha256": package.release_manifest_sha256,
"checksum_manifest_sha256": package.checksum_manifest_sha256,
"image_identity_sha256": package.image_identity_sha256,
"installable_manifest_sha256": package.installable_manifest_sha256,
"trusted_build_receipt_sha256": package.trusted_build_receipt_sha256,
"infrastructure_source_set_sha256": package.infrastructure_source_set_sha256,
"infrastructure_bundle_sha256": package.infrastructure_bundle_sha256,
"third_party_materials_input_sha256": package.third_party_materials_input_sha256,
"third_party_materials_manifest_sha256": package.third_party_materials_manifest_sha256,
"third_party_verification_receipt_sha256": (
package.third_party_verification_receipt_sha256
),
"third_party_profile": package.third_party_profile,
"distribution_clearance_claimed": False,
"backup_proof_sha256": package.backup_proof_sha256,
"compatibility_proof_sha256": package.compatibility_proof_sha256,
"env_file": str(env_target),
"env_file_sha256": evidence.env_sha256,
"compose_project_name": project_name,
"docker_daemon_identity": evidence.daemon_identity,
"production_preflight": evidence.preflight_summary,
"tls_material_binding": tls_binding,
"tls_material_binding_sha256": canonical_sha256(tls_binding),
"compose_merge_gate": evidence.compose_gate_summary,
"offline_execution_gate": offline_execution_gate,
"docker_import_plan": import_plan,
"offline_image_overrides": offline_image_overrides,
"apply_eligible": not apply_blockers,
"backup_compatibility_evidence_status": (
"runtime_signed_snapshot_restore_evidence_required_blocked"
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else "optional_self_hashed_material_untrusted_external_signatures_missing_blocked"
),
"backup_evidence_status": (
"not_applicable_runtime_snapshot_evidence_required"
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else "missing" if package.backup_proof is None else "unverified_self_hashed"
),
"compatibility_evidence_status": (
"not_applicable_exact_snapshot_restore_required"
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else (
"missing"
if package.compatibility_proof is None
else "unverified_self_hashed"
)
),
"real_apply_feature_gate": {
"status": "blocked",
"blockers": apply_blockers,
},
"offline": True,
"pull_policy": "never",
"build_allowed": False,
"database_downgrade_allowed": False,
"edge_or_dns_change_allowed": False,
"environment_overrides": environment_overrides,
"command_argv": command,
}
unsigned["plan_sha256"] = canonical_sha256(unsigned)
return unsigned
def verification_payload(package: ValidatedPackage) -> dict[str, Any]:
apply_blockers = _real_apply_blockers(
"install",
recovery_strategy=package.recovery_strategy,
package_schema_version=package.package_schema_version,
infrastructure_ready=len(package.docker_import_plan) == 11,
third_party_ready=package.third_party_verification_receipt is not None,
)
payload: dict[str, Any] = {
"schema_version": PLAN_SCHEMA,
"mode": "verify",
"status": "structurally_verified_not_apply_eligible",
"package_structure_valid": True,
"apply_eligible": False,
"real_apply_feature_gate": {
"status": "eligible" if not apply_blockers else "blocked",
"blockers": apply_blockers,
},
"deployment_scope": DEPLOYMENT_SCOPE,
"deployed_service_set": [],
"managed_backend_service_set": list(INSTALL_SERVICES),
"separately_deployed_artifacts": [
{
"image_key": key,
"reference": package.application_images[key]["reference"],
"digest": package.application_images[key]["digest"],
"deployment_scope": "separate_web_orchestrator",
"applied_by_this_operation": False,
}
for key in SEPARATELY_DEPLOYED_IMAGE_KEYS
],
"identity_bound_image_keys": list(APPLICATION_IMAGE_KEYS + INFRASTRUCTURE_IMAGE_KEYS),
"release_completion_claimed": False,
"package_schema_version": package.package_schema_version,
"release_id": package.package["release_id"],
"rollback_target_release_id": package.rollback_release_id,
"recovery_strategy": package.recovery_strategy,
"recovery_contract": _recovery_contract(package),
"package_manifest_sha256": package.package_sha256,
"release_manifest_sha256": package.release_manifest_sha256,
"checksum_manifest_sha256": package.checksum_manifest_sha256,
"image_identity_sha256": package.image_identity_sha256,
"installable_manifest_sha256": package.installable_manifest_sha256,
"trusted_build_receipt_sha256": package.trusted_build_receipt_sha256,
"infrastructure_source_set_sha256": package.infrastructure_source_set_sha256,
"infrastructure_bundle_sha256": package.infrastructure_bundle_sha256,
"third_party_materials_input_sha256": package.third_party_materials_input_sha256,
"third_party_materials_manifest_sha256": package.third_party_materials_manifest_sha256,
"third_party_verification_receipt_sha256": (
package.third_party_verification_receipt_sha256
),
"third_party_profile": package.third_party_profile,
"third_party_materials_status": (
"independently_verified_technical_materials"
if package.third_party_verification_receipt is not None
else "missing_blocked"
),
"distribution_clearance_claimed": False,
"docker_import_plan_status": (
"validated_for_eleven_release_images"
if len(package.docker_import_plan) == 11
else "legacy_four_application_images_only_blocked"
),
"infrastructure_import_plan_status": (
"validated_for_seven_infrastructure_images"
if package.infrastructure_bundle is not None
else "missing_blocked"
),
"backup_compatibility_evidence_status": (
"runtime_signed_snapshot_restore_evidence_required_blocked"
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else "optional_self_hashed_material_untrusted_external_signatures_missing_blocked"
),
"backup_evidence_status": (
"not_applicable_runtime_snapshot_evidence_required"
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else "missing" if package.backup_proof is None else "unverified_self_hashed"
),
"compatibility_evidence_status": (
"not_applicable_exact_snapshot_restore_required"
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE
else (
"missing"
if package.compatibility_proof is None
else "unverified_self_hashed"
)
),
"backup_proof_sha256": package.backup_proof_sha256,
"compatibility_proof_sha256": package.compatibility_proof_sha256,
"offline": True,
"writes_performed": False,
}
payload["plan_sha256"] = canonical_sha256(payload)
return payload
def _validate_state_target(state_dir: Path, *, package_root: Path) -> Path:
lexical = _absolute_lexical(state_dir)
if not state_dir.is_absolute():
raise ReleasePackageError("--state-dir must be an absolute path.")
if lexical == Path(lexical.anchor):
raise ReleasePackageError("--state-dir must not be a filesystem root.")
_reject_symlink_components(lexical, label="state directory")
existing = lexical
while not existing.exists() and existing != existing.parent:
existing = existing.parent
try:
existing_resolved = existing.resolve(strict=True)
except OSError as exc:
raise ReleasePackageError("State directory parent is unavailable.") from exc
if not existing_resolved.is_dir():
raise ReleasePackageError("State directory parent is not a directory.")
try:
lexical.resolve(strict=False).relative_to(package_root)
except ValueError:
pass
else:
raise ReleasePackageError("State directory must be outside the immutable release package.")
if lexical.exists() and not lexical.is_dir():
raise ReleasePackageError("State directory path is not a directory.")
return lexical
def _fsync_directory(path: Path) -> None:
if os.name == "nt":
return
flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)
descriptor = os.open(path, flags)
try:
os.fsync(descriptor)
finally:
os.close(descriptor)
def _prepare_secure_state_root(state_dir: Path, *, require_root_owner: bool) -> None:
existed = state_dir.exists()
state_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
_reject_symlink_components(state_dir, label="state directory")
if not state_dir.is_dir() or state_dir.is_symlink():
raise ReleasePackageError("State root must be a real directory.")
if os.name != "nt":
if not existed:
os.chmod(state_dir, 0o700)
state_stat = state_dir.stat()
expected_uid = 0 if require_root_owner else os.geteuid()
if state_stat.st_uid != expected_uid:
raise ReleasePackageError("State root has the wrong owner.")
if stat.S_IMODE(state_stat.st_mode) != 0o700:
raise ReleasePackageError("State root must have mode 0700.")
_fsync_directory(state_dir.parent)
def _secure_snapshot_copy(
source: Path,
destination: Path,
*,
expected_sha256: str,
expected_size: int | None = None,
label: str,
) -> dict[str, Any]:
source = _validated_file(source, label=label)
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
try:
source_descriptor = os.open(source, flags)
except OSError as exc:
raise ReleasePackageError(f"Cannot open {label} for an immutable snapshot.") from exc
destination_descriptor = -1
try:
before = os.fstat(source_descriptor)
if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
raise ReleasePackageError(f"{label} must be a single-link regular file.")
if expected_size is not None and before.st_size != expected_size:
raise ReleasePackageError(f"{label} size changed before snapshot.")
destination.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
_reject_symlink_components(destination.parent, label="snapshot directory")
destination_descriptor = os.open(
destination,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_BINARY", 0),
0o600,
)
digest = hashlib.sha256()
copied = 0
while True:
chunk = os.read(source_descriptor, 1024 * 1024)
if not chunk:
break
digest.update(chunk)
copied += len(chunk)
view = memoryview(chunk)
while view:
written = os.write(destination_descriptor, view)
view = view[written:]
after = os.fstat(source_descriptor)
if (
before.st_dev,
before.st_ino,
before.st_size,
getattr(before, "st_mtime_ns", None),
) != (
after.st_dev,
after.st_ino,
after.st_size,
getattr(after, "st_mtime_ns", None),
):
raise ReleasePackageError(f"{label} changed while being snapshotted.")
observed_sha256 = digest.hexdigest()
if observed_sha256 != expected_sha256:
raise ReleasePackageError(f"{label} SHA-256 changed before snapshot.")
if expected_size is not None and copied != expected_size:
raise ReleasePackageError(f"{label} size changed while being snapshotted.")
os.fsync(destination_descriptor)
except OSError as exc:
raise ReleasePackageError(f"Cannot create immutable snapshot for {label}.") from exc
finally:
os.close(source_descriptor)
if destination_descriptor >= 0:
os.close(destination_descriptor)
if os.name != "nt":
os.chmod(destination, 0o600)
destination_stat = destination.stat()
if destination_stat.st_nlink != 1:
raise ReleasePackageError(f"Snapshot for {label} is unexpectedly hard linked.")
return {
"path": destination.as_posix(),
"sha256": expected_sha256,
"size_bytes": destination_stat.st_size,
}
def _write_exclusive_bytes(path: Path, payload: bytes, *, label: str) -> dict[str, Any]:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
_reject_symlink_components(path.parent, label=f"{label} directory")
descriptor = -1
try:
descriptor = os.open(
path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_BINARY", 0),
0o600,
)
view = memoryview(payload)
while view:
written = os.write(descriptor, view)
view = view[written:]
os.fsync(descriptor)
except OSError as exc:
raise ReleasePackageError(f"Cannot create immutable {label}.") from exc
finally:
if descriptor >= 0:
os.close(descriptor)
metadata = os.lstat(path)
if not stat.S_ISREG(metadata.st_mode) or metadata.st_nlink != 1:
raise ReleasePackageError(f"Immutable {label} is not a single-link regular file.")
return {
"path": path.as_posix(),
"sha256": hashlib.sha256(payload).hexdigest(),
"size_bytes": len(payload),
}
def _rewrite_execution_env(
source: Path,
destination: Path,
*,
replacements: Mapping[str, str],
) -> dict[str, Any]:
try:
text = source.read_text(encoding="utf-8-sig")
except (OSError, UnicodeError) as exc:
raise ReleasePackageError("Snapshotted production env is not readable UTF-8 text.") from exc
remaining = set(replacements)
rewritten: list[str] = []
for raw_line in text.splitlines(keepends=True):
ending = ""
body = raw_line
for candidate in ("\r\n", "\n", "\r"):
if body.endswith(candidate):
body = body[: -len(candidate)]
ending = candidate
break
normalized = body.strip()
if normalized and not normalized.startswith("#") and "=" in normalized:
key = normalized.split("=", 1)[0]
if key in replacements:
value = replacements[key]
if not value or any(character in value for character in "\r\n\x00"):
raise ReleasePackageError(f"Execution env replacement for {key} is unsafe.")
body = f"{key}={value}"
remaining.discard(key)
rewritten.append(body + ending)
if remaining:
raise ReleasePackageError(
"Production env is missing TLS keys required for execution: "
+ ", ".join(sorted(remaining))
)
payload = "".join(rewritten).encode("utf-8")
record = _write_exclusive_bytes(destination, payload, label="execution env snapshot")
parsed = parse_env_file(destination)
if any(parsed.get(key) != value for key, value in replacements.items()):
raise ReleasePackageError("Execution env TLS paths do not match the immutable snapshot.")
return record
def _verify_immutable_snapshot_file(
path: Path,
*,
expected_sha256: str,
expected_size: int,
label: str,
) -> None:
target = _validated_file(path, label=label)
flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
descriptor = -1
try:
descriptor = os.open(target, flags)
before = os.fstat(descriptor)
if (
not stat.S_ISREG(before.st_mode)
or before.st_nlink != 1
or before.st_size != expected_size
):
raise ReleasePackageError(f"{label} metadata changed before execution.")
digest = hashlib.sha256()
while True:
chunk = os.read(descriptor, 1024 * 1024)
if not chunk:
break
digest.update(chunk)
after = os.fstat(descriptor)
if (
before.st_dev,
before.st_ino,
before.st_size,
getattr(before, "st_mtime_ns", None),
) != (
after.st_dev,
after.st_ino,
after.st_size,
getattr(after, "st_mtime_ns", None),
):
raise ReleasePackageError(f"{label} changed during execution verification.")
if digest.hexdigest() != expected_sha256:
raise ReleasePackageError(f"{label} SHA-256 changed before execution.")
except OSError as exc:
raise ReleasePackageError(f"Cannot verify immutable {label} before execution.") from exc
finally:
if descriptor >= 0:
os.close(descriptor)
def _verify_execution_snapshot(snapshot: SnapshotInputs, plan: Mapping[str, Any]) -> None:
files = snapshot.manifest.get("files")
if not isinstance(files, list):
raise ReleasePackageError("Execution snapshot has no file inventory.")
by_path = {
str(record.get("path")): record
for record in files
if isinstance(record, dict) and isinstance(record.get("path"), str)
}
required = [snapshot.env_file, *snapshot.tls_files.values()]
for path in required:
record = by_path.get(path.as_posix())
if (
not isinstance(record, dict)
or not isinstance(record.get("sha256"), str)
or not SHA256_RE.fullmatch(str(record.get("sha256")))
or not isinstance(record.get("size_bytes"), int)
):
raise ReleasePackageError("Execution snapshot TLS or env inventory is incomplete.")
_verify_immutable_snapshot_file(
path,
expected_sha256=str(record["sha256"]),
expected_size=int(record["size_bytes"]),
label=f"execution snapshot {path.name}",
)
if snapshot.manifest.get("tls_material_binding_sha256") != plan.get(
"tls_material_binding_sha256"
):
raise ReleasePackageError("Execution snapshot TLS binding does not match the release plan.")
def _freeze_snapshot_tree(root: Path) -> None:
if os.name == "nt":
return
files = [path for path in root.rglob("*") if path.is_file()]
directories = [path for path in root.rglob("*") if path.is_dir()]
for path in files:
os.chmod(path, 0o400)
for path in sorted(directories, key=lambda item: len(item.parts), reverse=True):
os.chmod(path, 0o500)
os.chmod(root, 0o500)
def _snapshot_inputs(
*,
state_dir: Path,
idempotency_key: str,
plan: Mapping[str, Any],
package: ValidatedPackage,
repo_root: Path,
) -> SnapshotInputs:
key_digest = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest()
snapshots_root = state_dir / "snapshots"
snapshots_root.mkdir(mode=0o700, exist_ok=True)
_reject_symlink_components(snapshots_root, label="snapshot root")
snapshot_root = snapshots_root / key_digest
try:
snapshot_root.mkdir(mode=0o700)
except FileExistsError as exc:
raise ReleasePackageError(
"A prior snapshot exists for this idempotency key; reconciliation is required."
) from exc
records: list[dict[str, Any]] = []
source_env_destination = snapshot_root / "production.source.env"
records.append(
_secure_snapshot_copy(
Path(str(plan["env_file"])),
source_env_destination,
expected_sha256=str(plan["env_file_sha256"]),
label="production env file",
)
)
binding = _tls_material_binding(plan["production_preflight"])
binding_sha256 = canonical_sha256(binding)
if (
binding != plan.get("tls_material_binding")
or binding_sha256 != plan.get("tls_material_binding_sha256")
):
raise ReleasePackageError("Release plan TLS material binding is inconsistent.")
source_values = parse_env_file(source_env_destination)
source_tls_dir = Path(source_values.get("BACKEND_TLS_CERT_DIR", ""))
source_ca_path = Path(source_values.get("BACKEND_TLS_TRUSTED_CA_FILE", ""))
tls_directory = snapshot_root / "tls"
tls_files = {
"certificate": tls_directory / "tls.crt",
"private_key": tls_directory / "tls.key",
"trusted_ca": tls_directory / "trusted-ca.pem",
}
records.append(
_secure_snapshot_copy(
source_tls_dir / "tls.crt",
tls_files["certificate"],
expected_sha256=binding["certificate_sha256"],
label="TLS certificate",
)
)
records.append(
_secure_snapshot_copy(
source_tls_dir / "tls.key",
tls_files["private_key"],
expected_sha256=binding["private_key_sha256"],
label="TLS private key",
)
)
records.append(
_secure_snapshot_copy(
source_ca_path,
tls_files["trusted_ca"],
expected_sha256=binding["trusted_ca_sha256"],
label="TLS trusted CA",
)
)
env_destination = snapshot_root / "production.env"
execution_env_record = _rewrite_execution_env(
source_env_destination,
env_destination,
replacements={
"BACKEND_TLS_CERT_DIR": tls_directory.as_posix(),
"BACKEND_TLS_TRUSTED_CA_FILE": tls_files["trusted_ca"].as_posix(),
},
)
records.append(execution_env_record)
compose_hashes = {
item["path"]: item["sha256"] for item in plan["compose_merge_gate"]["compose_inputs"]
}
compose_destinations: list[Path] = []
for relative in (
BASE_COMPOSE_RELATIVE,
PRODUCTION_COMPOSE_RELATIVE,
OFFLINE_COMPOSE_RELATIVE,
):
destination = snapshot_root.joinpath(*PurePosixPath(relative).parts)
records.append(
_secure_snapshot_copy(
repo_root.joinpath(*PurePosixPath(relative).parts),
destination,
expected_sha256=compose_hashes[relative],
label=f"Compose input {relative}",
)
)
compose_destinations.append(destination)
archive_destinations: dict[str, Path] = {}
for item in plan["docker_import_plan"]:
role = item["role"]
destination = snapshot_root / "docker-archives" / f"{role}.docker.tar"
records.append(
_secure_snapshot_copy(
package.root.joinpath(
*PurePosixPath(item["archive_package_path"]).parts
),
destination,
expected_sha256=item["archive_sha256"],
expected_size=item["archive_size_bytes"],
label=f"{role} Docker archive",
)
)
archive_destinations[role] = destination
manifest: dict[str, Any] = {
"schema_version": "yuqei.release-input-snapshot.v2",
"plan_sha256": plan["plan_sha256"],
"tls_material_binding_sha256": binding_sha256,
"execution_env_sha256": execution_env_record["sha256"],
"files": sorted(records, key=lambda item: item["path"].encode("utf-8")),
}
manifest["snapshot_payload_sha256"] = canonical_sha256(manifest)
_write_exclusive_canonical_json(snapshot_root / "snapshot-manifest.json", manifest)
for directory in {
snapshot_root,
source_env_destination.parent,
env_destination.parent,
*(path.parent for path in tls_files.values()),
*(path.parent for path in compose_destinations),
*(path.parent for path in archive_destinations.values()),
}:
_fsync_directory(directory)
snapshot = SnapshotInputs(
root=snapshot_root,
source_env_file=source_env_destination,
env_file=env_destination,
compose_files=(
compose_destinations[0],
compose_destinations[1],
compose_destinations[2],
),
docker_archives=archive_destinations,
tls_files=tls_files,
manifest=manifest,
)
_verify_execution_snapshot(snapshot, plan)
_freeze_snapshot_tree(snapshot_root)
return snapshot
@contextmanager
def _exclusive_lock(state_dir: Path) -> Iterator[Path]:
_reject_symlink_components(state_dir, label="state directory")
lock_path = state_dir / ".release-package.lock"
try:
descriptor = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError as exc:
raise ReleasePackageError("Another release package operation holds the exclusive lock.") from exc
except OSError as exc:
raise ReleasePackageError("Unable to acquire the release package lock.") from exc
try:
os.write(descriptor, f"pid={os.getpid()}\n".encode("ascii"))
os.fsync(descriptor)
_fsync_directory(state_dir)
os.close(descriptor)
descriptor = -1
yield lock_path
finally:
if descriptor >= 0:
os.close(descriptor)
try:
lock_path.unlink(missing_ok=True)
except OSError as exc: # pragma: no cover - an unhealthy filesystem should remain visible.
raise ReleasePackageError("Unable to release the release package lock.") from exc
def _receipt_path(state_dir: Path, idempotency_key: str) -> Path:
key_digest = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest()
return state_dir / "receipts" / f"{key_digest}.json"
def _pending_path(state_dir: Path, idempotency_key: str) -> Path:
key_digest = hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest()
return state_dir / "pending" / f"{key_digest}.json"
def _pending_journal(
*,
plan: Mapping[str, Any],
idempotency_key: str,
snapshot: SnapshotInputs,
command: list[str],
recorded_at: datetime,
) -> dict[str, Any]:
if recorded_at.tzinfo is None:
raise ReleasePackageError("Pending journal time must include a timezone.")
payload: dict[str, Any] = {
"schema_version": PENDING_SCHEMA,
"status": "reconcile_required",
"phase": "pending_image_import_and_backend_apply",
"operation": plan["operation"],
"package_schema_version": plan["package_schema_version"],
"deployment_scope": DEPLOYMENT_SCOPE,
"deployed_service_set": plan["deployed_service_set"],
"idempotency_key": idempotency_key,
"recorded_at": recorded_at.astimezone(UTC).isoformat().replace("+00:00", "Z"),
"plan_sha256": plan["plan_sha256"],
"snapshot_payload_sha256": snapshot.manifest["snapshot_payload_sha256"],
"tls_material_binding_sha256": plan["tls_material_binding_sha256"],
"execution_env_sha256": snapshot.manifest["execution_env_sha256"],
"docker_daemon_identity": plan["docker_daemon_identity"],
"installable_manifest_sha256": plan["installable_manifest_sha256"],
"trusted_build_receipt_sha256": plan["trusted_build_receipt_sha256"],
"infrastructure_source_set_sha256": plan["infrastructure_source_set_sha256"],
"infrastructure_bundle_sha256": plan["infrastructure_bundle_sha256"],
"third_party_materials_input_sha256": plan[
"third_party_materials_input_sha256"
],
"third_party_materials_manifest_sha256": plan[
"third_party_materials_manifest_sha256"
],
"third_party_verification_receipt_sha256": plan[
"third_party_verification_receipt_sha256"
],
"recovery_readiness_receipt_sha256": plan.get(
"recovery_readiness_receipt_sha256"
),
"docker_import_plan_sha256": canonical_sha256(plan["docker_import_plan"]),
"command_argv": command,
}
payload["journal_payload_sha256"] = canonical_sha256(payload)
return payload
def _read_pending_journal(path: Path) -> dict[str, Any]:
payload = _read_json(path, label="pending reconciliation journal")
if payload.get("schema_version") != PENDING_SCHEMA:
raise ReleasePackageError("Pending reconciliation journal has an unsupported schema.")
expected_fields = {
"schema_version",
"status",
"phase",
"operation",
"package_schema_version",
"deployment_scope",
"deployed_service_set",
"idempotency_key",
"recorded_at",
"plan_sha256",
"snapshot_payload_sha256",
"tls_material_binding_sha256",
"execution_env_sha256",
"docker_daemon_identity",
"installable_manifest_sha256",
"trusted_build_receipt_sha256",
"infrastructure_source_set_sha256",
"infrastructure_bundle_sha256",
"third_party_materials_input_sha256",
"third_party_materials_manifest_sha256",
"third_party_verification_receipt_sha256",
"recovery_readiness_receipt_sha256",
"docker_import_plan_sha256",
"command_argv",
"journal_payload_sha256",
}
if set(payload) != expected_fields:
raise ReleasePackageError("Pending reconciliation journal has unexpected or missing fields.")
_verify_embedded_hash(
payload,
"journal_payload_sha256",
label="pending reconciliation journal",
)
if payload.get("status") != "reconcile_required":
raise ReleasePackageError("Pending journal has an invalid status.")
for field in (
"plan_sha256",
"snapshot_payload_sha256",
"tls_material_binding_sha256",
"execution_env_sha256",
"installable_manifest_sha256",
"trusted_build_receipt_sha256",
"docker_import_plan_sha256",
"infrastructure_source_set_sha256",
"infrastructure_bundle_sha256",
"third_party_materials_input_sha256",
"third_party_materials_manifest_sha256",
"third_party_verification_receipt_sha256",
):
value = payload.get(field)
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
raise ReleasePackageError(f"Pending reconciliation journal has an invalid {field}.")
readiness_hash = payload.get("recovery_readiness_receipt_sha256")
if readiness_hash is not None and (
not isinstance(readiness_hash, str) or not SHA256_RE.fullmatch(readiness_hash)
):
raise ReleasePackageError(
"Pending reconciliation journal has an invalid recovery readiness receipt hash."
)
if path.read_bytes() != canonical_json_bytes(payload) + b"\n":
raise ReleasePackageError("Pending reconciliation journal is not canonical JSON.")
return payload
def _validate_receipt_payload(payload: Mapping[str, Any], *, label: str) -> None:
expected_fields = {
"schema_version",
"operation",
"status",
"execution_mode",
"package_schema_version",
"deployment_scope",
"deployed_service_set",
"separately_deployed_artifacts",
"identity_bound_image_keys",
"release_completion_claimed",
"release_id",
"target_release_id",
"compose_project_name",
"idempotency_key",
"recorded_at",
"plan_sha256",
"package_manifest_sha256",
"release_manifest_sha256",
"checksum_manifest_sha256",
"image_identity_sha256",
"installable_manifest_sha256",
"trusted_build_receipt_sha256",
"infrastructure_source_set_sha256",
"infrastructure_bundle_sha256",
"third_party_materials_input_sha256",
"third_party_materials_manifest_sha256",
"third_party_verification_receipt_sha256",
"third_party_profile",
"distribution_clearance_claimed",
"recovery_readiness_receipt_sha256",
"backup_proof_sha256",
"compatibility_proof_sha256",
"docker_daemon_identity",
"production_preflight_summary_sha256",
"compose_merge_gate_summary_sha256",
"offline_execution_gate_summary_sha256",
"snapshot_payload_sha256",
"tls_material_binding_sha256",
"execution_env_sha256",
"docker_image_import_result",
"post_apply_check",
"offline",
"pull_policy",
"build_allowed",
"database_downgrade_performed",
"edge_or_dns_change_performed",
"environment_overrides",
"command_argv",
"result",
"receipt_payload_sha256",
}
if set(payload) != expected_fields:
raise ReleasePackageError(f"{label} has unexpected or missing fields.")
if payload.get("schema_version") != RECEIPT_SCHEMA:
raise ReleasePackageError(f"{label} has an unsupported schema.")
_verify_embedded_hash(payload, "receipt_payload_sha256", label=label)
operation = payload.get("operation")
if operation not in CONFIRMATIONS:
raise ReleasePackageError(f"{label} has an invalid operation.")
if payload.get("status") != "backend_verified":
raise ReleasePackageError(f"{label} has an invalid status.")
if payload.get("execution_mode") not in {
"test_injected_runner",
"builtin_fixed_subprocess_runner",
}:
raise ReleasePackageError(f"{label} has an invalid execution mode.")
if payload.get("package_schema_version") != PACKAGE_SCHEMA_V2:
raise ReleasePackageError(f"{label} is not from an apply-eligible package schema.")
if payload.get("deployment_scope") != DEPLOYMENT_SCOPE:
raise ReleasePackageError(f"{label} does not declare the backend-only deployment scope.")
expected_services = RUNTIME_SERVICES if operation in {"rollback", "forward"} else INSTALL_SERVICES
if payload.get("deployed_service_set") != list(expected_services):
raise ReleasePackageError(f"{label} has an invalid backend deployed_service_set.")
if payload.get("release_completion_claimed") is not False:
raise ReleasePackageError(f"{label} must not claim completion of the full release.")
if payload.get("identity_bound_image_keys") != list(
APPLICATION_IMAGE_KEYS + INFRASTRUCTURE_IMAGE_KEYS
):
raise ReleasePackageError(f"{label} does not bind the complete production image inventory.")
separate = payload.get("separately_deployed_artifacts")
if not isinstance(separate, list) or len(separate) != len(SEPARATELY_DEPLOYED_IMAGE_KEYS):
raise ReleasePackageError(f"{label} has an invalid separately deployed artifact inventory.")
for expected_key, artifact in zip(SEPARATELY_DEPLOYED_IMAGE_KEYS, separate, strict=True):
if (
not isinstance(artifact, dict)
or set(artifact)
!= {
"image_key",
"reference",
"digest",
"deployment_scope",
"applied_by_this_operation",
}
or artifact.get("image_key") != expected_key
or artifact.get("deployment_scope") != "separate_web_orchestrator"
or artifact.get("applied_by_this_operation") is not False
or not IMAGE_REFERENCE_RE.fullmatch(str(artifact.get("reference", "")))
or not IMAGE_DIGEST_RE.fullmatch(str(artifact.get("digest", "")))
or str(artifact.get("reference", "")).rsplit("@", 1)[-1]
!= artifact.get("digest")
):
raise ReleasePackageError(f"{label} has an invalid separately deployed artifact record.")
_validate_identifier(payload.get("release_id"), RELEASE_ID_RE, label=f"{label} release_id")
_validate_identifier(
payload.get("target_release_id"),
RELEASE_ID_RE,
label=f"{label} target_release_id",
)
_validate_identifier(
payload.get("compose_project_name"),
PROJECT_RE,
label=f"{label} compose_project_name",
)
_validate_identifier(
payload.get("idempotency_key"),
IDEMPOTENCY_RE,
label=f"{label} idempotency_key",
)
recorded_at = payload.get("recorded_at")
if not isinstance(recorded_at, str) or not recorded_at.endswith("Z"):
raise ReleasePackageError(f"{label} has an invalid recorded_at timestamp.")
try:
parsed_time = datetime.fromisoformat(recorded_at[:-1] + "+00:00")
except ValueError as exc:
raise ReleasePackageError(f"{label} has an invalid recorded_at timestamp.") from exc
if parsed_time.tzinfo is None or parsed_time.utcoffset() != UTC.utcoffset(parsed_time):
raise ReleasePackageError(f"{label} recorded_at must be UTC.")
for hash_field in (
"plan_sha256",
"package_manifest_sha256",
"release_manifest_sha256",
"checksum_manifest_sha256",
"image_identity_sha256",
"installable_manifest_sha256",
"trusted_build_receipt_sha256",
"infrastructure_source_set_sha256",
"infrastructure_bundle_sha256",
"third_party_materials_input_sha256",
"third_party_materials_manifest_sha256",
"third_party_verification_receipt_sha256",
"production_preflight_summary_sha256",
"compose_merge_gate_summary_sha256",
"offline_execution_gate_summary_sha256",
"snapshot_payload_sha256",
"tls_material_binding_sha256",
"execution_env_sha256",
):
value = payload.get(hash_field)
if not isinstance(value, str) or not SHA256_RE.fullmatch(value):
raise ReleasePackageError(f"{label} has an invalid {hash_field}.")
for optional_hash_field in (
"backup_proof_sha256",
"compatibility_proof_sha256",
"recovery_readiness_receipt_sha256",
):
value = payload.get(optional_hash_field)
if value is not None and (
not isinstance(value, str) or not SHA256_RE.fullmatch(value)
):
raise ReleasePackageError(f"{label} has an invalid {optional_hash_field}.")
if (
payload.get("third_party_profile") not in {"internal-hosted", "redistributable"}
or payload.get("distribution_clearance_claimed") is not False
):
raise ReleasePackageError(
f"{label} makes an invalid third-party materials or clearance claim."
)
daemon = payload.get("docker_daemon_identity")
daemon_fields = {
"docker_cli",
"docker_host",
"daemon_id",
"daemon_name",
"docker_root_dir",
"server_version",
"identity_sha256",
}
if not isinstance(daemon, dict) or set(daemon) != daemon_fields:
raise ReleasePackageError(f"{label} has an invalid Docker daemon identity.")
_verify_embedded_hash(daemon, "identity_sha256", label=f"{label} Docker daemon identity")
if daemon.get("docker_cli") != str(DOCKER_CLI) or daemon.get("docker_host") != LOCAL_DOCKER_HOST:
raise ReleasePackageError(f"{label} is not bound to the fixed local Docker target.")
if any(
not isinstance(daemon.get(key), str) or not str(daemon.get(key)).strip()
for key in ("daemon_id", "daemon_name", "docker_root_dir", "server_version")
):
raise ReleasePackageError(f"{label} has an incomplete Docker daemon identity.")
import_result = payload.get("docker_image_import_result")
if not isinstance(import_result, dict) or set(import_result) != {
"schema_version",
"status",
"images",
"summary_sha256",
}:
raise ReleasePackageError(f"{label} has an invalid Docker image import result.")
_verify_embedded_hash(
import_result,
"summary_sha256",
label=f"{label} Docker image import result",
)
imported_images = import_result.get("images")
if (
import_result.get("schema_version") != RELEASE_IMPORT_RESULT_SCHEMA
or import_result.get("status") != "verified"
or not isinstance(imported_images, list)
or len(imported_images) != len(RELEASE_IMPORT_ROLE_ENV_SERVICES)
):
raise ReleasePackageError(f"{label} has an invalid Docker image import result.")
import_fields = {
"sequence",
"role",
"archive_sha256",
"loaded_image_id",
"offline_compose_override",
"load_stdout_sha256",
"load_stderr_sha256",
"tag_stdout_sha256",
"tag_stderr_sha256",
}
for sequence, ((role, env_key, _services), record) in enumerate(
zip(RELEASE_IMPORT_ROLE_ENV_SERVICES, imported_images, strict=True),
start=1,
):
if (
not isinstance(record, dict)
or set(record) != import_fields
or record.get("sequence") != sequence
or record.get("role") != role
or not IMAGE_DIGEST_RE.fullmatch(str(record.get("loaded_image_id", "")))
or record.get("offline_compose_override") != record.get("loaded_image_id")
or any(
not isinstance(record.get(field), str)
or not SHA256_RE.fullmatch(str(record.get(field)))
for field in (
"archive_sha256",
"load_stdout_sha256",
"load_stderr_sha256",
"tag_stdout_sha256",
"tag_stderr_sha256",
)
)
):
raise ReleasePackageError(f"{label} has an invalid imported image record.")
overrides = payload.get("environment_overrides")
expected_override_keys = {
*(key for _role, key, _services in RELEASE_IMPORT_ROLE_ENV_SERVICES),
"AI_PLATFORM_RELEASE_IMAGE_DIGEST",
}
if not isinstance(overrides, dict) or set(overrides) != expected_override_keys:
raise ReleasePackageError(f"{label} has an invalid environment override set.")
for (_role, env_key, _services), record in zip(
RELEASE_IMPORT_ROLE_ENV_SERVICES, imported_images, strict=True
):
if overrides.get(env_key) != record["loaded_image_id"]:
raise ReleasePackageError(f"{label} does not bind {env_key} to its loaded image ID.")
if not IMAGE_DIGEST_RE.fullmatch(str(overrides.get("AI_PLATFORM_RELEASE_IMAGE_DIGEST", ""))):
raise ReleasePackageError(f"{label} has an invalid AI release image digest.")
command = payload.get("command_argv")
if not isinstance(command, list) or any(not isinstance(item, str) for item in command):
raise ReleasePackageError(f"{label} has an invalid command argv.")
_assert_safe_compose_command(command, operation=operation)
post_check = payload.get("post_apply_check")
expected_post_fields = {
"schema_version",
"status",
"deployment_scope",
"operation",
"target_release_id",
"compose_project_name",
"deployed_service_set",
"daemon_identity_sha256",
"container_count",
"services",
"release_binding_verified",
"post_check_payload_sha256",
}
if not isinstance(post_check, dict) or set(post_check) != expected_post_fields:
raise ReleasePackageError(f"{label} has an invalid post-apply check.")
_verify_embedded_hash(post_check, "post_check_payload_sha256", label=f"{label} post-apply check")
if (
post_check.get("schema_version") != POST_APPLY_SCHEMA
or post_check.get("status") != "verified"
or post_check.get("deployment_scope") != DEPLOYMENT_SCOPE
or post_check.get("operation") != operation
or post_check.get("target_release_id") != payload.get("target_release_id")
or post_check.get("compose_project_name") != payload.get("compose_project_name")
or post_check.get("deployed_service_set") != payload.get("deployed_service_set")
or post_check.get("daemon_identity_sha256") != daemon.get("identity_sha256")
or post_check.get("container_count") != len(payload.get("deployed_service_set", []))
or not isinstance(post_check.get("services"), list)
or len(post_check.get("services")) != len(payload.get("deployed_service_set", []))
or post_check.get("release_binding_verified") is not True
):
raise ReleasePackageError(f"{label} has an invalid post-apply check binding.")
if (
payload.get("offline") is not True
or payload.get("pull_policy") != "never"
or payload.get("build_allowed") is not False
or payload.get("database_downgrade_performed") is not False
or payload.get("edge_or_dns_change_performed") is not False
):
raise ReleasePackageError(f"{label} violates offline or destructive-action boundaries.")
result = payload.get("result")
if (
not isinstance(result, dict)
or set(result) != {"returncode", "stdout_sha256", "stderr_sha256"}
or result.get("returncode") != 0
or any(
not isinstance(result.get(field), str)
or not SHA256_RE.fullmatch(str(result.get(field)))
for field in ("stdout_sha256", "stderr_sha256")
)
):
raise ReleasePackageError(f"{label} has an invalid successful execution result.")
def _read_existing_receipt(path: Path) -> dict[str, Any]:
if path.is_symlink():
raise ReleasePackageError("Idempotency receipt must not be a symbolic link.")
payload = _read_json(path, label="idempotency receipt")
_validate_receipt_payload(payload, label="idempotency receipt")
if path.read_bytes() != canonical_json_bytes(payload) + b"\n":
raise ReleasePackageError("Existing idempotency receipt is not canonical JSON.")
return payload
def _write_exclusive_canonical_json(path: Path, payload: Mapping[str, Any]) -> None:
data = canonical_json_bytes(payload) + b"\n"
try:
descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
except FileExistsError as exc:
raise ReleasePackageError(f"Refusing to overwrite existing receipt: {path}") from exc
try:
with os.fdopen(descriptor, "wb") as handle:
descriptor = -1
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
_fsync_directory(path.parent)
finally:
if descriptor >= 0:
os.close(descriptor)
def _output_bytes(value: object) -> bytes:
if value is None:
return b""
if isinstance(value, bytes):
return value
return str(value).encode("utf-8", "replace")
def _run_release_image_imports(
*,
plan: Mapping[str, Any],
snapshot: SnapshotInputs,
runner: Callable[..., Any],
) -> tuple[bool, dict[str, Any]]:
results: list[dict[str, Any]] = []
runtime_environment = _fixed_runtime_environment()
for item in plan["docker_import_plan"]:
role = item["role"]
archive = snapshot.docker_archives[role]
if file_sha256(archive) != item["archive_sha256"]:
return False, {"stage": "archive_recheck", "role": role}
load_command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"image",
"load",
"--input",
str(archive),
]
try:
loaded = runner(
load_command,
check=False,
capture_output=True,
shell=False,
env=runtime_environment,
timeout=1800,
)
except (OSError, subprocess.TimeoutExpired) as exc:
return False, {"stage": "docker_load", "role": role, "error": type(exc).__name__}
loaded_stdout = _output_bytes(getattr(loaded, "stdout", b""))
loaded_stderr = _output_bytes(getattr(loaded, "stderr", b""))
if int(loaded.returncode) != 0:
return False, {
"stage": "docker_load",
"role": role,
"returncode": int(loaded.returncode),
"stdout_sha256": hashlib.sha256(loaded_stdout).hexdigest(),
"stderr_sha256": hashlib.sha256(loaded_stderr).hexdigest(),
}
tag_stdout = b""
tag_stderr = b""
if item["tag_command_argv"]:
tag_command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"image",
"tag",
item["expected_loaded_image_id"],
item["expected_repo_tag"],
]
try:
tagged = runner(
tag_command,
check=False,
capture_output=True,
shell=False,
env=runtime_environment,
timeout=60,
)
except (OSError, subprocess.TimeoutExpired) as exc:
return False, {
"stage": "docker_tag",
"role": role,
"error": type(exc).__name__,
}
tag_stdout = _output_bytes(getattr(tagged, "stdout", b""))
tag_stderr = _output_bytes(getattr(tagged, "stderr", b""))
if int(tagged.returncode) != 0:
return False, {
"stage": "docker_tag",
"role": role,
"returncode": int(tagged.returncode),
"stdout_sha256": hashlib.sha256(tag_stdout).hexdigest(),
"stderr_sha256": hashlib.sha256(tag_stderr).hexdigest(),
}
inspect_command = [
str(DOCKER_CLI),
"--host",
LOCAL_DOCKER_HOST,
"image",
"inspect",
"--format",
"{{.Id}}",
item["expected_repo_tag"],
]
try:
inspected = runner(
inspect_command,
check=False,
capture_output=True,
shell=False,
env=runtime_environment,
timeout=60,
)
except (OSError, subprocess.TimeoutExpired) as exc:
return False, {"stage": "docker_inspect", "role": role, "error": type(exc).__name__}
observed_id = _output_bytes(getattr(inspected, "stdout", b"")).decode(
"utf-8",
errors="replace",
).strip()
if int(inspected.returncode) != 0 or observed_id != item["expected_loaded_image_id"]:
return False, {
"stage": "docker_inspect",
"role": role,
"returncode": int(inspected.returncode),
"expected_loaded_image_id": item["expected_loaded_image_id"],
"observed_image_id_sha256": hashlib.sha256(
observed_id.encode("utf-8")
).hexdigest(),
}
results.append(
{
"sequence": item["sequence"],
"role": role,
"archive_sha256": item["archive_sha256"],
"loaded_image_id": observed_id,
"offline_compose_override": item["offline_compose_override"],
"load_stdout_sha256": hashlib.sha256(loaded_stdout).hexdigest(),
"load_stderr_sha256": hashlib.sha256(loaded_stderr).hexdigest(),
"tag_stdout_sha256": hashlib.sha256(tag_stdout).hexdigest(),
"tag_stderr_sha256": hashlib.sha256(tag_stderr).hexdigest(),
}
)
summary: dict[str, Any] = {
"schema_version": RELEASE_IMPORT_RESULT_SCHEMA,
"status": "verified",
"images": results,
}
summary["summary_sha256"] = canonical_sha256(summary)
return True, summary
def _validate_post_apply_result(
value: object,
*,
plan: Mapping[str, Any],
) -> dict[str, Any]:
if not isinstance(value, dict):
raise ReleasePackageError("Post-apply checker must return an object.")
_verify_embedded_hash(value, "post_check_payload_sha256", label="post-apply check")
expected = {
"schema_version": POST_APPLY_SCHEMA,
"status": "verified",
"deployment_scope": DEPLOYMENT_SCOPE,
"operation": plan["operation"],
"target_release_id": plan["target_release_id"],
"compose_project_name": plan["compose_project_name"],
"deployed_service_set": plan["deployed_service_set"],
"daemon_identity_sha256": plan["docker_daemon_identity"]["identity_sha256"],
"container_count": len(plan["deployed_service_set"]),
"release_binding_verified": True,
}
for key, expected_value in expected.items():
if value.get(key) != expected_value:
raise ReleasePackageError(f"Post-apply check has invalid {key}.")
services = value.get("services")
if not isinstance(services, list) or len(services) != len(plan["deployed_service_set"]):
raise ReleasePackageError("Post-apply check has an invalid service inventory.")
return dict(value)
def _receipt(
*,
plan: Mapping[str, Any],
snapshot: SnapshotInputs,
idempotency_key: str,
command_argv: list[str],
runtime_overrides: Mapping[str, str],
image_import_result: Mapping[str, Any],
post_apply_check: Mapping[str, Any],
snapshot_offline_gate: Mapping[str, Any],
returncode: int,
stdout: bytes,
stderr: bytes,
recorded_at: datetime,
execution_mode: str,
) -> dict[str, Any]:
if recorded_at.tzinfo is None:
raise ReleasePackageError("Receipt time must include a timezone.")
payload: dict[str, Any] = {
"schema_version": RECEIPT_SCHEMA,
"operation": plan["operation"],
"status": "backend_verified",
"execution_mode": execution_mode,
"package_schema_version": plan["package_schema_version"],
"deployment_scope": plan["deployment_scope"],
"deployed_service_set": plan["deployed_service_set"],
"separately_deployed_artifacts": plan["separately_deployed_artifacts"],
"identity_bound_image_keys": plan["identity_bound_image_keys"],
"release_completion_claimed": False,
"release_id": plan["release_id"],
"target_release_id": plan["target_release_id"],
"compose_project_name": plan["compose_project_name"],
"idempotency_key": idempotency_key,
"recorded_at": recorded_at.astimezone(UTC).isoformat().replace("+00:00", "Z"),
"plan_sha256": plan["plan_sha256"],
"package_manifest_sha256": plan["package_manifest_sha256"],
"release_manifest_sha256": plan["release_manifest_sha256"],
"checksum_manifest_sha256": plan["checksum_manifest_sha256"],
"image_identity_sha256": plan["image_identity_sha256"],
"installable_manifest_sha256": plan["installable_manifest_sha256"],
"trusted_build_receipt_sha256": plan["trusted_build_receipt_sha256"],
"infrastructure_source_set_sha256": plan["infrastructure_source_set_sha256"],
"infrastructure_bundle_sha256": plan["infrastructure_bundle_sha256"],
"third_party_materials_input_sha256": plan[
"third_party_materials_input_sha256"
],
"third_party_materials_manifest_sha256": plan[
"third_party_materials_manifest_sha256"
],
"third_party_verification_receipt_sha256": plan[
"third_party_verification_receipt_sha256"
],
"third_party_profile": plan["third_party_profile"],
"distribution_clearance_claimed": False,
"recovery_readiness_receipt_sha256": plan.get(
"recovery_readiness_receipt_sha256"
),
"backup_proof_sha256": plan["backup_proof_sha256"],
"compatibility_proof_sha256": plan["compatibility_proof_sha256"],
"docker_daemon_identity": plan["docker_daemon_identity"],
"production_preflight_summary_sha256": plan["production_preflight"][
"summary_sha256"
],
"compose_merge_gate_summary_sha256": plan["compose_merge_gate"]["summary_sha256"],
"offline_execution_gate_summary_sha256": snapshot_offline_gate["summary_sha256"],
"snapshot_payload_sha256": snapshot.manifest["snapshot_payload_sha256"],
"tls_material_binding_sha256": plan["tls_material_binding_sha256"],
"execution_env_sha256": snapshot.manifest["execution_env_sha256"],
"docker_image_import_result": dict(image_import_result),
"post_apply_check": dict(post_apply_check),
"offline": True,
"pull_policy": "never",
"build_allowed": False,
"database_downgrade_performed": False,
"edge_or_dns_change_performed": False,
"environment_overrides": dict(runtime_overrides),
"command_argv": command_argv,
"result": {
"returncode": returncode,
"stdout_sha256": hashlib.sha256(stdout).hexdigest(),
"stderr_sha256": hashlib.sha256(stderr).hexdigest(),
},
}
payload["receipt_payload_sha256"] = canonical_sha256(payload)
return payload
def _require_apply_authorization(
*,
operation: str,
confirmation: str,
idempotency_key: str | None,
) -> str:
expected = confirmation_for(operation)
if confirmation != expected:
raise ReleasePackageError(f"Apply confirmation must exactly equal: {expected}")
if idempotency_key is None or not IDEMPOTENCY_RE.fullmatch(idempotency_key):
raise ReleasePackageError("--idempotency-key is required and must use the safe identifier syntax.")
return idempotency_key
def _validate_first_release_install_readiness(
*,
package: ValidatedPackage,
expected_package_sha256: str,
evidence_file: Path | None,
keyring_file: Path | None,
trusted_root_public_key_file: Path | None,
receipt_file: Path | None,
expected_receipt_sha256: str | None,
now: datetime,
) -> tuple[dict[str, Any], str, dict[str, str]]:
inputs = {
"evidence": evidence_file,
"keyring": keyring_file,
"trusted_root_public_key": trusted_root_public_key_file,
"receipt": receipt_file,
}
if (
any(path is None for path in inputs.values())
or not isinstance(expected_receipt_sha256, str)
or not SHA256_RE.fullmatch(expected_receipt_sha256)
):
raise ReleasePackageError(
"First-release install apply requires the raw evidence, keyring, pinned root key, "
"readiness receipt, and externally approved receipt SHA-256."
)
validated_paths = {
name: _validated_file(path, label=f"first-release {name}")
for name, path in inputs.items()
if path is not None
}
actual_receipt_sha = file_sha256(validated_paths["receipt"])
if actual_receipt_sha != expected_receipt_sha256:
raise ReleasePackageError(
"First-release readiness receipt does not match its approved SHA-256."
)
module = _load_snapshot_restore_planner_module()
validator = getattr(module, "validate_install_readiness", None)
if not callable(validator):
raise ReleasePackageError(
"The frozen snapshot planner has no validated install-readiness API; apply remains closed."
)
try:
validated = validator(
package_file=package.root / PACKAGE_FILE,
expected_package_sha256=expected_package_sha256,
evidence_file=validated_paths["evidence"],
keyring_file=validated_paths["keyring"],
trusted_root_public_key_file=validated_paths["trusted_root_public_key"],
receipt_file=validated_paths["receipt"],
expected_receipt_sha256=expected_receipt_sha256,
now=now,
)
except Exception as exc:
raise ReleasePackageError(
f"First-release install readiness validation failed: {exc}"
) from exc
included = _read_json(validated_paths["receipt"], label="first-release readiness receipt")
if not isinstance(validated, Mapping) or dict(validated) != included:
raise ReleasePackageError(
"First-release readiness receipt differs from cryptographic recomputation."
)
release_binding = included.get("release_binding")
if (
included.get("result") != RECOVERY_READINESS_RESULT
or included.get("purpose") != "release_package_install_apply_gate"
or included.get("install_apply_gate") is not True
or included.get("snapshot_restore_apply_allowed") is not False
or not isinstance(release_binding, Mapping)
or release_binding.get("release_id") != package.package["release_id"]
or release_binding.get("package_sha256") != package.package_sha256
or release_binding.get("source_commit")
!= package.image_identity.get("source_commit")
or release_binding.get("image_identity_sha256") != package.image_identity_sha256
):
raise ReleasePackageError(
"First-release readiness receipt is not an install-only PASS for this package."
)
input_hashes = {name: file_sha256(path) for name, path in validated_paths.items()}
return dict(included), actual_receipt_sha, input_hashes
def _validate_readiness_runtime_binding(
*,
readiness: Mapping[str, Any],
env_values: Mapping[str, str],
env_sha256: str,
daemon_identity: Mapping[str, str],
tls_binding: Mapping[str, str],
project_name: str,
) -> None:
bindings = readiness.get("release_environment_and_tls_bindings")
target = readiness.get("target")
expected_bindings = {
"production_env_sha256": env_sha256,
"tls_certificate_sha256": tls_binding.get("certificate_sha256"),
"tls_private_key_sha256": tls_binding.get("private_key_sha256"),
"tls_trusted_ca_sha256": tls_binding.get("trusted_ca_sha256"),
}
planner = _load_snapshot_restore_planner_module()
host_identity = getattr(planner, "runtime_host_identity_sha256", None)
if not callable(host_identity):
raise ReleasePackageError(
"The frozen snapshot planner has no runtime host identity algorithm."
)
expected_target = {
"environment_id": env_values.get("YUQEI_RELEASE_ENVIRONMENT_ID"),
"environment_class": env_values.get("YUQEI_RELEASE_ENVIRONMENT_CLASS"),
"host_id": env_values.get("YUQEI_RELEASE_HOST_ID"),
"host_identity_sha256": host_identity(
env_values.get("YUQEI_RELEASE_HOST_ID"),
daemon_identity,
),
"docker_daemon_identity_sha256": daemon_identity.get("identity_sha256"),
"compose_project_name": project_name,
}
if (
not isinstance(bindings, Mapping)
or dict(bindings) != expected_bindings
or not isinstance(target, Mapping)
or dict(target) != expected_target
):
raise ReleasePackageError(
"First-release readiness receipt does not match the exact runtime "
"environment/host/daemon/TLS/project."
)
def execute(
*,
package_dir: Path,
expected_package_sha256: str,
operation: str = "verify",
env_file: Path | None = None,
apply: bool = False,
confirmation: str = "",
idempotency_key: str | None = None,
state_dir: Path = Path("/var/lib/yuqei/release-package"),
repo_root: Path = REPO_ROOT,
runner: Callable[..., Any] | None = None,
inspection_runner: Callable[..., Any] = subprocess.run,
post_apply_checker: Callable[..., Any] | None = None,
first_release_evidence_file: Path | None = None,
first_release_keyring_file: Path | None = None,
first_release_trusted_root_public_key_file: Path | None = None,
first_release_readiness_receipt_file: Path | None = None,
expected_first_release_readiness_receipt_sha256: str | None = None,
now: datetime | None = None,
) -> ExecutionOutcome:
current_time = now or datetime.now(UTC)
package = validate_release_package(
package_dir,
expected_package_sha256=expected_package_sha256,
repo_root=repo_root,
)
if operation == "verify":
if apply:
raise ReleasePackageError("The verify operation is read-only and does not accept --apply.")
return ExecutionOutcome(verification_payload(package))
if operation not in CONFIRMATIONS:
raise ReleasePackageError("Operation must be verify, install, rollback, or forward.")
_assert_operation_supported_for_recovery(package, operation=operation)
if env_file is None:
raise ReleasePackageError("--env-file is required for install, rollback, and forward plans.")
safe_key: str | None = None
readiness: dict[str, Any] | None = None
readiness_sha256: str | None = None
readiness_input_hashes: dict[str, str] = {}
execution_mode = "test_injected_runner"
if apply:
safe_key = _require_apply_authorization(
operation=operation,
confirmation=confirmation,
idempotency_key=idempotency_key,
)
if package.package_schema_version != PACKAGE_SCHEMA_V2:
raise ReleasePackageError(
"Release package v1 is permanently read-only; real or injected apply requires v2."
)
if package.recovery_strategy == RECOVERY_STRATEGY_FIRST_RELEASE:
readiness, readiness_sha256, readiness_input_hashes = (
_validate_first_release_install_readiness(
package=package,
expected_package_sha256=expected_package_sha256,
evidence_file=first_release_evidence_file,
keyring_file=first_release_keyring_file,
trusted_root_public_key_file=(
first_release_trusted_root_public_key_file
),
receipt_file=first_release_readiness_receipt_file,
expected_receipt_sha256=(
expected_first_release_readiness_receipt_sha256
),
now=current_time,
)
)
if runner is None:
raise ReleasePackageError(
"Apply requires the built-in fixed subprocess runner or an explicitly injected test runner."
)
if post_apply_checker is None:
raise ReleasePackageError(
"Injected apply tests require the fixed post-apply checker interface."
)
builtin_mode = (
runner is _fixed_subprocess_runner
and inspection_runner is _fixed_subprocess_runner
and post_apply_checker is _builtin_post_apply_checker
)
execution_mode = (
"builtin_fixed_subprocess_runner"
if builtin_mode
else "test_injected_runner"
)
if builtin_mode:
blockers = _real_apply_blockers(
operation,
recovery_strategy=package.recovery_strategy,
package_schema_version=package.package_schema_version,
infrastructure_ready=len(package.docker_import_plan) == 11,
third_party_ready=package.third_party_verification_receipt is not None,
recovery_readiness_ready=readiness is not None,
)
if blockers:
raise ReleasePackageError(
"Real apply is fail-closed until all release blockers are resolved: "
+ ", ".join(blockers)
)
if operation == "rollback":
raise ReleasePackageError(
"Rollback apply is fail-closed because no approved rollback Docker import "
"plan exists."
)
plan = build_plan(
package,
operation=operation,
env_file=env_file,
repo_root=repo_root,
inspection_runner=inspection_runner,
recovery_readiness_receipt=readiness,
recovery_readiness_receipt_sha256=readiness_sha256,
)
if readiness is not None:
_validate_readiness_runtime_binding(
readiness=readiness,
env_values=parse_env_file(Path(plan["env_file"])),
env_sha256=plan["env_file_sha256"],
daemon_identity=plan["docker_daemon_identity"],
tls_binding=plan["tls_material_binding"],
project_name=plan["compose_project_name"],
)
if not apply:
return ExecutionOutcome(plan)
assert safe_key is not None # Authorization above is required for every write path.
for name, expected_hash in readiness_input_hashes.items():
source_path = {
"evidence": first_release_evidence_file,
"keyring": first_release_keyring_file,
"trusted_root_public_key": first_release_trusted_root_public_key_file,
"receipt": first_release_readiness_receipt_file,
}[name]
if source_path is None or file_sha256(_validated_file(source_path, label=name)) != expected_hash:
raise ReleasePackageError(
"First-release readiness input changed after validation and before state write."
)
state = _validate_state_target(state_dir, package_root=package.root)
_prepare_secure_state_root(state, require_root_owner=False)
with _exclusive_lock(state):
locked_package = validate_release_package(
package_dir,
expected_package_sha256=expected_package_sha256,
repo_root=repo_root,
)
locked_plan = build_plan(
locked_package,
operation=operation,
env_file=env_file,
repo_root=repo_root,
inspection_runner=inspection_runner,
recovery_readiness_receipt=readiness,
recovery_readiness_receipt_sha256=readiness_sha256,
)
if locked_plan["plan_sha256"] != plan["plan_sha256"]:
raise ReleasePackageError("Release package, env, or execution plan changed while acquiring the lock.")
receipt_path = _receipt_path(state, safe_key)
pending_path = _pending_path(state, safe_key)
receipts_dir = receipt_path.parent
pending_dir = pending_path.parent
if receipts_dir.exists() and (receipts_dir.is_symlink() or not receipts_dir.is_dir()):
raise ReleasePackageError("Receipt directory is unsafe.")
if pending_dir.exists() and (pending_dir.is_symlink() or not pending_dir.is_dir()):
raise ReleasePackageError("Pending journal directory is unsafe.")
receipts_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
pending_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
_reject_symlink_components(receipts_dir, label="receipt directory")
_reject_symlink_components(pending_dir, label="pending journal directory")
if pending_path.exists() or pending_path.is_symlink():
pending = _read_pending_journal(pending_path)
if pending.get("idempotency_key") != safe_key:
raise ReleasePackageError("Pending reconciliation journal is bound to another key.")
return ExecutionOutcome(pending, exit_code=1, replayed=True)
if receipt_path.exists() or receipt_path.is_symlink():
existing = _read_existing_receipt(receipt_path)
if existing.get("idempotency_key") != safe_key:
raise ReleasePackageError("Idempotency receipt is bound to another key.")
if existing.get("execution_mode") != execution_mode:
raise ReleasePackageError(
"Idempotency receipt execution mode does not match the current apply mode."
)
if existing.get("plan_sha256") != plan["plan_sha256"]:
raise ReleasePackageError(
"Idempotency key is already bound to a different release operation."
)
return ExecutionOutcome(existing, exit_code=0, replayed=True)
snapshot = _snapshot_inputs(
state_dir=state,
idempotency_key=safe_key,
plan=plan,
package=locked_package,
repo_root=_validated_directory(repo_root, label="repository root"),
)
source_snapshot_evidence = _collect_runtime_evidence(
env_file=snapshot.source_env_file,
repo_root=snapshot.root,
inspection_runner=inspection_runner,
)
if source_snapshot_evidence.env_sha256 != plan["env_file_sha256"]:
raise ReleasePackageError("Source production env snapshot SHA-256 does not match the plan.")
if source_snapshot_evidence.preflight_summary != plan["production_preflight"]:
raise ReleasePackageError("Source production preflight summary changed.")
if source_snapshot_evidence.daemon_identity != plan["docker_daemon_identity"]:
raise ReleasePackageError("Docker daemon identity changed before apply.")
execution_snapshot_evidence = _collect_runtime_evidence(
env_file=snapshot.env_file,
repo_root=snapshot.root,
inspection_runner=inspection_runner,
)
if execution_snapshot_evidence.env_sha256 != snapshot.manifest["execution_env_sha256"]:
raise ReleasePackageError("Execution env snapshot SHA-256 is inconsistent.")
if execution_snapshot_evidence.preflight_summary != plan["production_preflight"]:
raise ReleasePackageError("Execution TLS snapshot does not match the production preflight.")
if execution_snapshot_evidence.daemon_identity != plan["docker_daemon_identity"]:
raise ReleasePackageError("Docker daemon identity changed while preparing execution.")
_verify_execution_snapshot(snapshot, plan)
snapshot_offline_gate = _run_offline_execution_gate(
env_file=snapshot.env_file,
repo_root=snapshot.root,
compose_files=snapshot.compose_files,
project_name=plan["compose_project_name"],
offline_image_overrides=plan["offline_image_overrides"],
inspection_runner=inspection_runner,
)
runtime_overrides = dict(plan["environment_overrides"])
runtime_overrides.update(plan["offline_image_overrides"])
runtime_environment = _fixed_runtime_environment(runtime_overrides)
command = _compose_command(
operation=operation,
project_name=plan["compose_project_name"],
env_file=snapshot.env_file,
repo_root=snapshot.root,
compose_files=snapshot.compose_files,
)
_assert_safe_compose_command(command, operation=operation)
journal = _pending_journal(
plan=plan,
idempotency_key=safe_key,
snapshot=snapshot,
command=command,
recorded_at=current_time,
)
_write_exclusive_canonical_json(pending_path, journal)
imported, import_result = _run_release_image_imports(
plan=plan,
snapshot=snapshot,
runner=runner,
)
if not imported:
return ExecutionOutcome(journal, exit_code=1)
_verify_execution_snapshot(snapshot, plan)
try:
completed = runner(
command,
check=False,
capture_output=True,
shell=False,
env=runtime_environment,
timeout=1800,
)
returncode = int(completed.returncode)
stdout = _output_bytes(getattr(completed, "stdout", b""))
stderr = _output_bytes(getattr(completed, "stderr", b""))
except (OSError, subprocess.TimeoutExpired) as exc:
returncode = 127
stdout = b""
stderr = type(exc).__name__.encode("ascii", "replace")
if returncode != 0:
return ExecutionOutcome(journal, exit_code=1)
post_daemon_identity = _docker_daemon_identity(
runner=inspection_runner,
repo_root=snapshot.root,
)
if post_daemon_identity != plan["docker_daemon_identity"]:
return ExecutionOutcome(journal, exit_code=1)
try:
post_apply = _validate_post_apply_result(
post_apply_checker(
plan=dict(plan),
command_argv=list(command),
runtime_environment=dict(runtime_environment),
daemon_identity=dict(post_daemon_identity),
),
plan=plan,
)
except Exception:
return ExecutionOutcome(journal, exit_code=1)
receipt = _receipt(
plan=plan,
snapshot=snapshot,
idempotency_key=safe_key,
command_argv=command,
runtime_overrides=runtime_overrides,
image_import_result=import_result,
post_apply_check=post_apply,
snapshot_offline_gate=snapshot_offline_gate,
returncode=returncode,
stdout=stdout,
stderr=stderr,
recorded_at=current_time,
execution_mode=execution_mode,
)
_write_exclusive_canonical_json(receipt_path, receipt)
try:
pending_path.unlink()
_fsync_directory(pending_dir)
except OSError as exc:
raise ReleasePackageError(
"Backend was verified but pending reconciliation journal could not be cleared."
) from exc
return ExecutionOutcome(receipt, exit_code=0)
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--package-dir", type=Path, required=True)
parser.add_argument(
"--expected-package-sha256",
required=True,
help="Externally approved raw SHA-256 of release-package.json.",
)
parser.add_argument(
"--operation",
choices=("verify", "install", "rollback", "forward"),
default="verify",
help="Defaults to read-only package verification; non-verify operations are plans without --apply.",
)
parser.add_argument("--env-file", type=Path, help="Absolute protected production env file.")
parser.add_argument("--state-dir", type=Path, default=Path("/var/lib/yuqei/release-package"))
parser.add_argument(
"--apply",
action="store_true",
help=(
"Request execution after authorization and all feature gates; "
"the current production gate is closed."
),
)
parser.add_argument(
"--confirmation",
default="",
help="Exact operation confirmation; see the dry-run plan or --help.",
)
parser.add_argument("--idempotency-key", default=None)
parser.add_argument("--first-release-evidence-file", type=Path)
parser.add_argument("--first-release-keyring-file", type=Path)
parser.add_argument("--first-release-trusted-root-public-key-file", type=Path)
parser.add_argument("--first-release-readiness-receipt-file", type=Path)
parser.add_argument("--expected-first-release-readiness-receipt-sha256")
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
try:
outcome = execute(
package_dir=args.package_dir,
expected_package_sha256=args.expected_package_sha256,
operation=args.operation,
env_file=args.env_file,
apply=args.apply,
confirmation=args.confirmation,
idempotency_key=args.idempotency_key,
state_dir=args.state_dir,
runner=_fixed_subprocess_runner if args.apply else None,
inspection_runner=_fixed_subprocess_runner,
post_apply_checker=_builtin_post_apply_checker if args.apply else None,
first_release_evidence_file=args.first_release_evidence_file,
first_release_keyring_file=args.first_release_keyring_file,
first_release_trusted_root_public_key_file=(
args.first_release_trusted_root_public_key_file
),
first_release_readiness_receipt_file=(
args.first_release_readiness_receipt_file
),
expected_first_release_readiness_receipt_sha256=(
args.expected_first_release_readiness_receipt_sha256
),
)
except (ReleasePackageError, OSError) as exc:
print(
canonical_json_bytes(
{"schema_version": RECEIPT_SCHEMA, "status": "rejected", "error": str(exc)}
).decode("utf-8"),
file=sys.stderr,
)
return 2
print(canonical_json_bytes(outcome.payload).decode("utf-8"))
return outcome.exit_code
if __name__ == "__main__":
raise SystemExit(main())
Reference in New Issue View Git Blame Copy Permalink