yuqei/ai-contract-platform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
5ce16ced1097a0c7cf071a4a71f2ee19deb97109
ai-contract-platform/yuqei-infra/scripts/run-linux-release-build.py
T

1947 lines
77 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""Build and scan the four RC application images on one pinned Linux/amd64 builder.
``--plan`` and ``--dry-run`` perform all source provenance checks but never
invoke Docker, Syft, Trivy, or the network. A successful real run publishes
its output directory atomically and writes a canonical, self-hashed receipt.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import platform
import re
import shutil
import subprocess
import sys
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Mapping, Sequence
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
# The repository must remain byte-for-byte clean while candidate identity is
# checked. Do not let importing the adjacent release helper create a tracked
# or untracked ``__pycache__`` side effect in a clean clone.
sys.dont_write_bytecode = True
import linux_release_common as release_common # noqa: E402
from linux_release_common import ( # noqa: E402
BUILD_RECEIPT_SCHEMA,
BUILDER_POLICY,
COMMAND_TIMEOUTS_SECONDS,
COMPOSE_GATE_POLICY,
CONTROLLED_SUBPROCESS_ENV,
LOCAL_DOCKER_HOST,
PINNED_EXECUTABLES,
REQUIRED_TOOL_VERSIONS,
TARGET,
TRIVY_DB_REPOSITORY_TRANSPORT,
ReleaseValidationError,
bind_spdx_image_root,
canonical_json_bytes,
controlled_subprocess_env,
file_record,
format_rfc3339,
inspect_docker_archive,
inspect_oci_archive,
load_json_bytes,
load_json_file,
load_policy,
load_source_manifest,
parse_rfc3339,
sha256_bytes,
sha256_file,
validate_spdx,
validate_compose_gate_report,
validate_candidate_checkout,
validate_buildx_config_root,
validate_manifest_commit_tree,
validate_pinned_executables,
validate_tag_binding,
validate_trivy_db_metadata,
validate_trivy_license_scan,
validate_trivy_scan,
verify_source_archive,
synthetic_compose_gate_env_bytes,
with_payload_digest,
)
REPO_ROOT = SCRIPT_DIR.parents[1]
DEFAULT_POLICY_PATH = REPO_ROOT / "yuqei-infra/release/linux-release-security-policy.v1.json"
PLAN_SCHEMA = "yuqei.linux-release-build-plan.v1"
VERSION_RE = re.compile(r"(?:^|\s)v?([0-9]+\.[0-9]+\.[0-9]+)(?:\s|$)")
BUILDER_NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}")
RUNNER_SOURCE_RELATIVE = "yuqei-infra/scripts/run-linux-release-build.py"
COMMON_SOURCE_RELATIVE = "yuqei-infra/scripts/linux_release_common.py"
POLICY_SOURCE_RELATIVE = "yuqei-infra/release/linux-release-security-policy.v1.json"
COMPOSE_CONFIG_MAX_BYTES = 8 * 1024 * 1024
COMPOSE_ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
COMPOSE_PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
COMPOSE_VOLUME_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
COMPOSE_IMAGE_REFERENCE_RE = re.compile(
r"^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$"
)
TRUSTED_APPLICATION_SERVICE_IMAGE_ENV: Mapping[str, str] = {
"contract-api-migrate": "CONTRACT_API_IMAGE",
"contract-api": "CONTRACT_API_IMAGE",
"contract-review-worker": "CONTRACT_API_IMAGE",
"contract-playbook-governance-worker": "CONTRACT_API_IMAGE",
"ai-platform-api-migrate": "AI_PLATFORM_API_IMAGE",
"ai-platform-prompt-assets-import": "AI_PLATFORM_API_IMAGE",
"ai-platform-api": "AI_PLATFORM_API_IMAGE",
"worker": "AI_WORKER_IMAGE",
"contract-web": "CONTRACT_WEB_IMAGE",
}
TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV: Mapping[str, str] = {
"postgres": "POSTGRES_IMAGE",
"postgres-init-databases": "POSTGRES_CLIENT_IMAGE",
"redis": "REDIS_IMAGE",
"ai-redis": "REDIS_IMAGE",
"minio": "MINIO_IMAGE",
"edge": "NGINX_IMAGE",
"contract-files-backup": "BACKUP_UTILITY_IMAGE",
"minio-backup": "MINIO_MC_IMAGE",
}
TRUSTED_PRODUCTION_SERVICES = (
"postgres",
"postgres-init-databases",
"redis",
"ai-redis",
"minio",
"contract-api-migrate",
"contract-api",
"contract-review-worker",
"contract-playbook-governance-worker",
"ai-platform-api-migrate",
"ai-platform-prompt-assets-import",
"ai-platform-api",
"worker",
"contract-web",
"edge",
"contract-files-backup",
"minio-backup",
)
TRUSTED_PROFILE_ONLY_SERVICES: Mapping[str, str] = {
"contract-web": "backend-hosted-web",
"contract-files-backup": "ops",
"minio-backup": "ops",
}
TRUSTED_PROFILE_CASES = (
("default", ()),
("backend-hosted-web", ("backend-hosted-web",)),
("ops", ("ops",)),
("backend-hosted-web+ops", ("backend-hosted-web", "ops")),
)
def _archive_suffix(path: Path) -> str:
lowered = path.name.casefold()
if lowered.endswith(".tar.gz"):
return ".tar.gz"
if lowered.endswith(".tgz"):
return ".tgz"
if lowered.endswith(".tar"):
return ".tar"
return ".tar"
def _run_command(
arguments: Sequence[str],
*,
cwd: Path | None = None,
allowed_returncodes: set[int] | None = None,
env: Mapping[str, str] | None = None,
timeout_seconds: int = COMMAND_TIMEOUTS_SECONDS["tool_version"],
) -> subprocess.CompletedProcess[str]:
allowed = {0} if allowed_returncodes is None else allowed_returncodes
try:
completed = subprocess.run(
list(arguments),
cwd=cwd,
check=False,
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
env=controlled_subprocess_env() if env is None else dict(env),
timeout=timeout_seconds,
)
except subprocess.TimeoutExpired as error:
raise ReleaseValidationError(
f"release command timed out after {timeout_seconds} seconds: {' '.join(arguments)}"
) from error
except OSError as error:
raise ReleaseValidationError(
f"cannot execute required release command {arguments[0]!r}: {error}"
) from error
if completed.returncode not in allowed:
detail = (completed.stderr or completed.stdout).strip()[-4000:]
raise ReleaseValidationError(
f"command failed closed ({completed.returncode}): {' '.join(arguments)}\n{detail}"
)
return completed
def _json_version(output: str, *, label: str) -> dict[str, Any]:
parsed = load_json_bytes(output.encode("utf-8"), label=label)
if not isinstance(parsed, dict):
raise ReleaseValidationError(f"{label} version output must be a JSON object")
return parsed
def _simple_version(output: str, *, label: str) -> str:
matches = VERSION_RE.findall(output)
if not matches:
raise ReleaseValidationError(f"cannot parse {label} version output: {output!r}")
return matches[0]
def parse_tool_versions(outputs: dict[str, str]) -> dict[str, str]:
"""Parse and enforce every version in the controlled builder contract."""
docker = _json_version(outputs["docker"], label="docker version")
try:
docker_cli = str(docker["Client"]["Version"])
docker_engine = str(docker["Server"]["Version"])
except (KeyError, TypeError) as error:
raise ReleaseValidationError("docker version output lacks Client/Server.Version") from error
syft_payload = _json_version(outputs["syft"], label="Syft version")
trivy_payload = _json_version(outputs["trivy"], label="Trivy version")
syft_version = syft_payload.get("version", syft_payload.get("Version"))
trivy_version = trivy_payload.get("Version", trivy_payload.get("version"))
actual = {
"docker_engine": docker_engine,
"docker_cli": docker_cli,
"buildx": _simple_version(outputs["buildx"], label="Buildx"),
"compose": _simple_version(outputs["compose"], label="Compose"),
"syft": str(syft_version),
"trivy": str(trivy_version),
}
drift = {
tool: {"required": required, "actual": actual.get(tool)}
for tool, required in REQUIRED_TOOL_VERSIONS.items()
if actual.get(tool) != required
}
if drift:
raise ReleaseValidationError(f"builder toolchain version drift: {json.dumps(drift, sort_keys=True)}")
return actual
def validate_toolchain(
runner: Callable[..., subprocess.CompletedProcess[str]] = _run_command,
) -> dict[str, str]:
outputs = {
"docker": runner(
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"version",
"--format",
"{{json .}}",
],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["tool_version"],
).stdout,
"buildx": runner(
[PINNED_EXECUTABLES["docker"], "--host", LOCAL_DOCKER_HOST, "buildx", "version"],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["tool_version"],
).stdout,
"compose": runner(
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"compose",
"version",
"--short",
],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["tool_version"],
).stdout,
"syft": runner(
[PINNED_EXECUTABLES["syft"], "version", "-o", "json"],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["tool_version"],
).stdout,
"trivy": runner(
[PINNED_EXECUTABLES["trivy"], "version", "--format", "json"],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["tool_version"],
).stdout,
}
return parse_tool_versions(outputs)
def parse_builder_inspect(output: str, *, builder_name: str) -> dict[str, Any]:
if not BUILDER_NAME_RE.fullmatch(builder_name):
raise ReleaseValidationError("Buildx builder name is missing or unsafe")
name_match = re.search(r"(?m)^Name:\s+(\S+)\s*$", output)
driver_match = re.search(r"(?m)^Driver:\s+(\S+)\s*$", output)
if not name_match or name_match.group(1) != builder_name or not driver_match:
raise ReleaseValidationError("Buildx inspect output does not identify the explicit builder")
driver = driver_match.group(1)
if driver not in BUILDER_POLICY["allowed_drivers"]:
raise ReleaseValidationError(
f"Buildx builder driver {driver!r} cannot satisfy the OCI/Docker exporter contract"
)
if "Nodes:" not in output:
raise ReleaseValidationError("Buildx inspect output has no node inventory")
nodes = output.split("Nodes:", 1)[1]
node_names = re.findall(r"(?m)^Name:\s+(\S+)\s*$", nodes)
statuses = re.findall(r"(?m)^Status:\s+(\S+)\s*$", nodes)
buildkit_versions = re.findall(r"(?m)^BuildKit version:\s+v?(\S+)\s*$", nodes)
platform_lines = re.findall(r"(?m)^Platforms:\s+(.+?)\s*$", nodes)
maximum_nodes = BUILDER_POLICY["maximum_nodes"]
if not (
len(node_names) == len(statuses) == len(buildkit_versions) == len(platform_lines) == maximum_nodes
):
raise ReleaseValidationError(
"Buildx builder must expose exactly one fully described node for deterministic serial builds"
)
if statuses != ["running"]:
raise ReleaseValidationError(f"Buildx builder node is not running: {statuses}")
platforms = [
item.strip().rstrip("*")
for item in platform_lines[0].split(",")
if item.strip()
]
if BUILDER_POLICY["required_platform"] not in platforms:
raise ReleaseValidationError("Buildx builder node does not advertise linux/amd64")
if buildkit_versions[0] != BUILDER_POLICY["required_buildkit_version"]:
raise ReleaseValidationError(
"BuildKit version drift: "
f"{buildkit_versions[0]!r} != {BUILDER_POLICY['required_buildkit_version']!r}"
)
return {
"name": builder_name,
"driver": driver,
"node": node_names[0],
"node_status": statuses[0],
"buildkit_version": buildkit_versions[0],
"platforms": platforms,
"exporters": BUILDER_POLICY["required_exporters"],
"inspect_sha256": sha256_bytes(output.encode("utf-8")),
}
def parse_builder_container_inspect(
output: str,
*,
node_name: str,
) -> dict[str, Any]:
payload = load_json_bytes(output.encode("utf-8"), label="BuildKit driver container inspect")
if not isinstance(payload, list) or len(payload) != 1 or not isinstance(payload[0], dict):
raise ReleaseValidationError("BuildKit driver container inspect must describe exactly one container")
container = payload[0]
expected_name = f"/buildx_buildkit_{node_name}"
if container.get("Name") != expected_name:
raise ReleaseValidationError(
f"BuildKit driver container name drift: {container.get('Name')!r} != {expected_name!r}"
)
config = container.get("Config")
state = container.get("State")
if not isinstance(config, dict) or not isinstance(state, dict) or state.get("Running") is not True:
raise ReleaseValidationError("BuildKit driver container is not running or inspect is incomplete")
if config.get("Image") != BUILDER_POLICY["required_driver_image"]:
raise ReleaseValidationError("BuildKit driver image reference/digest drift")
if container.get("Image") != BUILDER_POLICY["required_local_driver_image_id"]:
raise ReleaseValidationError("BuildKit local driver image ID drift")
return {
"container_name": expected_name.removeprefix("/"),
"driver_image": config["Image"],
"driver_image_digest": BUILDER_POLICY["required_driver_image_digest"],
"local_driver_image_id": container["Image"],
"image_transport": "domestic_registry_mirror",
"container_inspect_sha256": sha256_bytes(output.encode("utf-8")),
}
def validate_buildx_builder(
builder_name: str,
runner: Callable[..., subprocess.CompletedProcess[str]] = _run_command,
) -> dict[str, Any]:
if not BUILDER_NAME_RE.fullmatch(builder_name):
raise ReleaseValidationError("--builder must be an explicit safe Buildx builder name")
config_root = validate_buildx_config_root(builder_name)
completed = runner(
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"inspect",
builder_name,
"--bootstrap",
],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["builder_inspect"],
)
summary = parse_builder_inspect(completed.stdout, builder_name=builder_name)
container_name = f"buildx_buildkit_{summary['node']}"
container = runner(
[PINNED_EXECUTABLES["docker"], "--host", LOCAL_DOCKER_HOST, "inspect", container_name],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["container_inspect"],
)
summary.update(
parse_builder_container_inspect(container.stdout, node_name=summary["node"])
)
summary["config_root"] = config_root
return summary
def parse_imagetools_inspect(
output: str,
*,
transport_reference: str,
content_digest: str,
) -> dict[str, str]:
digest_match = re.search(r"(?m)^Digest:\s+(sha256:[0-9a-f]{64})\s*$", output)
media_match = re.search(r"(?m)^MediaType:\s+(\S+)\s*$", output)
if not digest_match or digest_match.group(1) != content_digest:
raise ReleaseValidationError(
f"base-image mirror digest drift for {transport_reference}"
)
if not media_match or media_match.group(1) != "application/vnd.oci.image.manifest.v1+json":
raise ReleaseValidationError(
f"base-image mirror did not return one OCI image manifest for {transport_reference}"
)
return {
"transport_reference": transport_reference,
"content_digest": content_digest,
"media_type": media_match.group(1),
"transport": "domestic_registry_mirror",
"inspect_sha256": sha256_bytes(output.encode("utf-8")),
}
def validate_base_image_transports(
policy: dict[str, Any],
runner: Callable[..., subprocess.CompletedProcess[str]] = _run_command,
) -> list[dict[str, str]]:
unique: dict[str, dict[str, Any]] = {}
for image in policy["images"]:
base = image["base_image"]
unique[base["transport_reference"]] = base
results = []
for reference in sorted(unique):
completed = runner(
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"imagetools",
"inspect",
reference,
],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["base_image_inspect"],
)
results.append(
parse_imagetools_inspect(
completed.stdout,
transport_reference=reference,
content_digest=unique[reference]["content_digest"],
)
)
return results
def inspect_trivy_db_repository_transport(
policy: dict[str, Any],
runner: Callable[..., subprocess.CompletedProcess[str]] = _run_command,
) -> dict[str, Any]:
transport = policy["vulnerability_scan"]["database"]["repository_transport"]
reference = transport["reference"]
completed = runner(
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"imagetools",
"inspect",
reference,
],
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["trivy_db_inspect"],
)
digest_match = re.search(r"(?m)^Digest:\s+(sha256:[0-9a-f]{64})\s*$", completed.stdout)
media_match = re.search(r"(?m)^MediaType:\s+(\S+)\s*$", completed.stdout)
if not digest_match:
raise ReleaseValidationError("Trivy DB repository inspection has no valid resolved digest")
if not media_match or media_match.group(1) not in transport["allowed_media_types"]:
raise ReleaseValidationError("Trivy DB repository inspection has an unsupported media type")
return {
"reference": reference,
"transport": transport["transport"],
"schema_version": transport["schema_version"],
"resolved_digest": digest_match.group(1),
"media_type": media_match.group(1),
"inspect_sha256": sha256_bytes(completed.stdout.encode("utf-8")),
}
def _source_file_records(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
return {item["path"]: item for item in manifest["files"]["items"]}
def _git_blob_oid(data: bytes, hex_length: int) -> str:
algorithm = "sha1" if hex_length == 40 else "sha256" if hex_length == 64 else ""
if not algorithm:
raise ReleaseValidationError("candidate Git blob object id has an unsupported length")
digest = hashlib.new(algorithm)
digest.update(f"blob {len(data)}\0".encode("ascii"))
digest.update(data)
return digest.hexdigest()
def _bind_running_release_control(
*,
manifest: dict[str, Any],
relative_path: str,
running_path: Path,
) -> dict[str, Any]:
record = _source_file_records(manifest).get(relative_path)
if record is None:
raise ReleaseValidationError(
f"source manifest does not bind running release control: {relative_path}"
)
lexical = Path(running_path).absolute()
if lexical.is_symlink() or not lexical.is_file():
raise ReleaseValidationError(
f"running release control is missing or a symlink: {relative_path}"
)
data = lexical.read_bytes()
raw_sha256 = sha256_bytes(data)
blob_oid = _git_blob_oid(data, len(str(record["git_blob_oid"])))
if (
len(data) != record["size"]
or raw_sha256 != record["sha256"]
or blob_oid != record["git_blob_oid"]
):
raise ReleaseValidationError(
f"running release control differs from candidate manifest/Git blob: {relative_path}"
)
return {
"path": relative_path,
"size_bytes": len(data),
"sha256": raw_sha256,
"git_blob_oid": blob_oid,
}
def validate_running_release_controls(
*,
manifest: dict[str, Any],
policy_path: Path,
) -> dict[str, dict[str, Any]]:
controls = {
"runner": (RUNNER_SOURCE_RELATIVE, Path(__file__)),
"common": (COMMON_SOURCE_RELATIVE, Path(str(release_common.__file__))),
"policy": (POLICY_SOURCE_RELATIVE, policy_path),
}
return {
name: _bind_running_release_control(
manifest=manifest,
relative_path=relative,
running_path=path,
)
for name, (relative, path) in controls.items()
}
def _required_compose_gate_source_records(
manifest: dict[str, Any],
) -> dict[str, dict[str, Any]]:
records = _source_file_records(manifest)
required = [COMPOSE_GATE_POLICY["validator"], *COMPOSE_GATE_POLICY["compose_inputs"]]
missing = [path for path in required if path not in records]
if missing:
raise ReleaseValidationError(
f"source manifest lacks production Compose gate inputs: {missing}"
)
return {path: records[path] for path in required}
def _strict_synthetic_compose_env(data: bytes) -> dict[str, str]:
try:
text = data.decode("utf-8")
except UnicodeError as error:
raise ReleaseValidationError(
"synthetic Compose gate environment is not UTF-8"
) from error
values: dict[str, str] = {}
for line_number, raw_line in enumerate(text.splitlines(), start=1):
line = raw_line.strip()
if not line or line.startswith("#"):
continue
if line.startswith("export ") or "=" not in line:
raise ReleaseValidationError(
f"synthetic Compose gate environment has invalid syntax at line {line_number}"
)
raw_key, raw_value = line.split("=", 1)
if raw_key != raw_key.strip():
raise ReleaseValidationError(
f"synthetic Compose gate environment has invalid key spacing at line {line_number}"
)
if not COMPOSE_ENV_KEY_RE.fullmatch(raw_key) or raw_key in values:
raise ReleaseValidationError(
f"synthetic Compose gate environment has invalid or duplicate key at line {line_number}"
)
values[raw_key] = raw_value.strip()
project_name = values.get("COMPOSE_PROJECT_NAME")
if not isinstance(project_name, str) or not COMPOSE_PROJECT_RE.fullmatch(project_name):
raise ReleaseValidationError(
"synthetic Compose gate environment lacks a safe COMPOSE_PROJECT_NAME"
)
storage_volume = values.get("MINIO_VOLUME_NAME", "")
if (
not COMPOSE_VOLUME_NAME_RE.fullmatch(storage_volume)
or "rustfs" not in storage_volume.lower()
or "minio" in storage_volume.lower()
):
raise ReleaseValidationError(
"synthetic Compose gate environment lacks a dedicated RustFS volume name"
)
for key in ("MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"):
if not values.get(key, ""):
raise ReleaseValidationError(
f"synthetic Compose gate environment lacks {key}"
)
image_env_keys = set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values()) | set(
TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.values()
)
for env_key in sorted(image_env_keys):
image = values.get(env_key)
if not isinstance(image, str) or not COMPOSE_IMAGE_REFERENCE_RE.fullmatch(image):
raise ReleaseValidationError(
f"synthetic Compose gate environment lacks immutable {env_key}"
)
return values
def _reject_compose_json_constant(value: str) -> None:
raise ReleaseValidationError(
f"Docker Compose JSON contains forbidden numeric constant {value}"
)
def _compose_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
result: dict[str, Any] = {}
for key, value in pairs:
if key in result:
raise ReleaseValidationError(
f"Docker Compose JSON contains duplicate key {key!r}"
)
result[key] = value
return result
def _parse_compose_model(stdout: Any) -> dict[str, Any]:
if isinstance(stdout, bytes):
try:
raw = stdout.decode("utf-8")
except UnicodeError as error:
raise ReleaseValidationError("Docker Compose emitted non-UTF-8 JSON") from error
elif isinstance(stdout, str):
raw = stdout
else:
raise ReleaseValidationError("Docker Compose did not emit textual JSON")
if len(raw.encode("utf-8")) > COMPOSE_CONFIG_MAX_BYTES:
raise ReleaseValidationError("Docker Compose JSON exceeds the maximum allowed size")
try:
payload = json.loads(
raw,
object_pairs_hook=_compose_json_object,
parse_constant=_reject_compose_json_constant,
)
except ReleaseValidationError:
raise
except (TypeError, ValueError, json.JSONDecodeError) as error:
raise ReleaseValidationError("Docker Compose did not emit valid JSON") from error
if not isinstance(payload, dict):
raise ReleaseValidationError("Docker Compose JSON must be an object")
return payload
def _required_compose_services(profiles: Sequence[str]) -> set[str]:
active = set(profiles)
return {
service
for service in TRUSTED_PRODUCTION_SERVICES
if service not in TRUSTED_PROFILE_ONLY_SERVICES
or TRUSTED_PROFILE_ONLY_SERVICES[service] in active
}
def _validate_trusted_object_storage_contract(
services: Mapping[str, Any],
*,
volumes: object,
env_values: Mapping[str, str],
mode: str,
profile_name: str,
) -> None:
label = f"{mode}/{profile_name}"
storage = services.get("minio")
if not isinstance(storage, dict):
raise ReleaseValidationError(
f"{label}: object-storage service is missing or invalid"
)
mounts = storage.get("volumes")
valid_mount = (
isinstance(mounts, list)
and len(mounts) == 1
and isinstance(mounts[0], dict)
and mounts[0].get("type") == "volume"
and mounts[0].get("source") == "rustfs-data"
and mounts[0].get("target") == "/data"
and mounts[0].get("read_only") in (None, False)
)
volume = volumes.get("rustfs-data") if isinstance(volumes, dict) else None
valid_volume = (
isinstance(volume, dict)
and volume.get("name") == env_values["MINIO_VOLUME_NAME"]
)
if not valid_mount or not valid_volume:
raise ReleaseValidationError(
f"{label}: object storage must use only the dedicated rustfs-data volume"
)
command = storage.get("command")
command_text = "\n".join(command) if isinstance(command, list) else str(command or "")
if (
storage.get("entrypoint") != ["/bin/sh", "-ec"]
or "set -eu" not in command_text
or "/data/.minio.sys" not in command_text
or ".yuqei-rustfs-volume-v1" not in command_text
or "exec /entrypoint.sh rustfs" not in command_text
):
raise ReleaseValidationError(
f"{label}: object-storage volume guard or fixed RustFS startup is missing"
)
environment = storage.get("environment")
required_environment = {
"RUSTFS_VOLUMES": "/data",
"RUSTFS_ADDRESS": "0.0.0.0:9000",
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
}
if not isinstance(environment, dict) or any(
str(environment.get(key, "")) != value
for key, value in required_environment.items()
):
raise ReleaseValidationError(f"{label}: RustFS runtime environment is invalid")
healthcheck = storage.get("healthcheck")
health_test = healthcheck.get("test") if isinstance(healthcheck, dict) else None
if not isinstance(health_test, list) or not any(
isinstance(token, str) and "/health/ready" in token for token in health_test
):
raise ReleaseValidationError(f"{label}: RustFS readiness healthcheck is missing")
if (
storage.get("user") != "10001:10001"
or storage.get("read_only") is not True
or not isinstance(storage.get("tmpfs"), list)
or not any(str(item).startswith("/tmp:") for item in storage["tmpfs"])
or not any(str(item).startswith("/logs:") for item in storage["tmpfs"])
or "ALL" not in storage.get("cap_drop", [])
or "no-new-privileges:true" not in storage.get("security_opt", [])
or storage.get("privileged") not in (None, False)
):
raise ReleaseValidationError(f"{label}: RustFS runtime hardening is incomplete")
def _validate_trusted_compose_model(
payload: Mapping[str, Any],
*,
env_values: Mapping[str, str],
mode: str,
profile_name: str,
profiles: Sequence[str],
) -> dict[str, Any]:
services = payload.get("services")
if not isinstance(services, dict):
raise ReleaseValidationError(
f"{mode}/{profile_name}: merged Compose model has no services object"
)
actual_names = set(services)
required_names = _required_compose_services(profiles)
if actual_names != required_names:
raise ReleaseValidationError(
f"{mode}/{profile_name}: production service inventory drift: "
f"missing={sorted(required_names - actual_names)}, "
f"unexpected={sorted(actual_names - required_names)}"
)
for service_name in sorted(actual_names):
if not isinstance(services[service_name], dict):
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} is not an object"
)
_validate_trusted_object_storage_contract(
services,
volumes=payload.get("volumes"),
env_values=env_values,
mode=mode,
profile_name=profile_name,
)
validated_application_services: list[str] = []
for service_name, env_key in TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.items():
if service_name not in services:
continue
service = services[service_name]
if service.get("image") != env_values[env_key]:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} image does not exactly match {env_key}"
)
if service.get("build") is not None:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} retains a build definition"
)
expected_pull = "never" if mode == "offline" else "always"
if service.get("pull_policy") != expected_pull:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} pull_policy must be {expected_pull}"
)
validated_application_services.append(service_name)
for service_name, env_key in TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.items():
if service_name not in services:
continue
service = services[service_name]
if service.get("image") != env_values[env_key]:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} image does not exactly match {env_key}"
)
if service.get("build") is not None:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} retains a build definition"
)
if mode == "offline":
for service_name in sorted(actual_names):
service = services[service_name]
if service.get("build") is not None:
raise ReleaseValidationError(
f"offline/{profile_name}: service {service_name} retains a build definition"
)
if service.get("pull_policy") != "never":
raise ReleaseValidationError(
f"offline/{profile_name}: service {service_name} pull_policy must be never"
)
model_bytes = json.dumps(
payload,
ensure_ascii=False,
allow_nan=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return {
"mode": mode,
"profile_case": profile_name,
"profiles": list(profiles),
"service_count": len(actual_names),
"validated_application_services": validated_application_services,
"config_sha256": sha256_bytes(model_bytes),
}
def _trusted_compose_command(
*,
project_name: str,
env_path: Path,
compose_paths: Sequence[Path],
profiles: Sequence[str],
) -> list[str]:
command = [
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"compose",
"--project-name",
project_name,
"--env-file",
str(env_path),
]
for compose_path in compose_paths:
command.extend(("-f", str(compose_path)))
for profile in profiles:
command.extend(("--profile", profile))
command.extend(("config", "--format", "json"))
return command
def run_production_compose_gate(
*,
source_root: Path,
staging: Path,
manifest: dict[str, Any],
policy: dict[str, Any],
runner: Callable[..., subprocess.CompletedProcess[str]] = _run_command,
) -> dict[str, Any]:
"""Run a self-bound, read-only eight-case gate with deterministic fake values.
The candidate validator remains an immutable reviewed input and is recorded
for audit continuity, but it is deliberately never executed or trusted to
attest its own result. The running release builder contains the gate logic
and is itself bound to the candidate manifest before this function runs.
"""
source_records = _required_compose_gate_source_records(manifest)
validator_relative = COMPOSE_GATE_POLICY["validator"]
source_root_resolved = source_root.resolve(strict=True)
compose_paths: dict[str, Path] = {}
for relative, record in source_records.items():
candidate = source_root.joinpath(*relative.split("/"))
if candidate.is_symlink() or not candidate.is_file():
raise ReleaseValidationError(
f"candidate Compose gate input is missing or unsafe: {relative}"
)
resolved = candidate.resolve(strict=True)
try:
resolved.relative_to(source_root_resolved)
except ValueError as error:
raise ReleaseValidationError(
f"candidate Compose gate input escapes the extracted source: {relative}"
) from error
if resolved.stat().st_size != record["size"] or sha256_file(resolved) != record["sha256"]:
raise ReleaseValidationError(
f"candidate Compose gate input digest/size drift: {relative}"
)
compose_paths[relative] = resolved
env_relative = "validation/synthetic-compose-gate.env"
report_relative = "validation/production-compose-gate.json"
env_path = staging / env_relative
report_path = staging / report_relative
env_bytes = synthetic_compose_gate_env_bytes()
env_values = _strict_synthetic_compose_env(env_bytes)
_write_bytes_exclusive(env_path, env_bytes)
base_path, production_path, offline_path = (
compose_paths[path] for path in COMPOSE_GATE_POLICY["compose_inputs"]
)
cases: list[dict[str, Any]] = []
for profile_name, profiles in TRUSTED_PROFILE_CASES:
for mode, selected_paths in (
("online", (base_path, production_path)),
("offline", (base_path, production_path, offline_path)),
):
command = _trusted_compose_command(
project_name=env_values["COMPOSE_PROJECT_NAME"],
env_path=env_path,
compose_paths=selected_paths,
profiles=profiles,
)
completed = runner(
command,
cwd=source_root_resolved,
env=controlled_subprocess_env(),
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["compose_gate"],
)
cases.append(
_validate_trusted_compose_model(
_parse_compose_model(completed.stdout),
env_values=env_values,
mode=mode,
profile_name=profile_name,
profiles=profiles,
)
)
report_payload = {
"schema_version": COMPOSE_GATE_POLICY["schema_version"],
"status": "passed",
"env_file_sha256": sha256_bytes(env_bytes),
"compose_inputs": [
{"path": path, "sha256": source_records[path]["sha256"]}
for path in COMPOSE_GATE_POLICY["compose_inputs"]
],
"application_service_count": len(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV),
"production_service_count": len(TRUSTED_PRODUCTION_SERVICES),
"cases": cases,
}
summary = validate_compose_gate_report(
report_payload,
policy,
expected_env_sha256=sha256_bytes(env_bytes),
source_file_sha256={
path: record["sha256"] for path, record in source_records.items()
},
)
_write_bytes_exclusive(report_path, canonical_json_bytes(report_payload))
validator_record = source_records[validator_relative]
return {
"synthetic_environment": True,
"production_secrets_used": False,
"execution_engine": "self_bound_release_runner_builtin_v1",
"validator": {
"source_path": validator_relative,
"sha256": validator_record["sha256"],
"size_bytes": validator_record["size"],
},
"environment": file_record(env_path, env_relative),
"report": file_record(report_path, report_relative, **summary),
}
def validate_release_inputs(
*,
repo: Path,
source_archive: Path,
source_manifest: Path,
commit: str,
tag: str,
policy_path: Path,
) -> dict[str, Any]:
policy = load_policy(policy_path)
manifest = load_source_manifest(source_manifest, commit=commit)
checkout = validate_candidate_checkout(repo, commit=commit)
commit_tree = validate_manifest_commit_tree(repo, commit=commit, manifest=manifest)
archive = verify_source_archive(source_archive, manifest)
release_controls = validate_running_release_controls(
manifest=manifest,
policy_path=policy_path,
)
tag_binding = validate_tag_binding(
repo,
tag=tag,
commit=commit,
expected_tree=manifest["candidate"]["tree"],
)
return {
"policy": policy,
"policy_sha256": sha256_file(policy_path),
"source_manifest": manifest,
"source_manifest_sha256": sha256_file(source_manifest),
"archive": archive,
"checkout": checkout,
"release_controls": release_controls,
"commit_tree": commit_tree,
"tag_binding": tag_binding,
}
def _image_build_command(
*,
spec: Mapping[str, Any],
commit: str,
tag: str,
builder_name: str,
source_date_epoch: int,
dockerfile: str | Path,
oci_output: str | Path,
docker_output: str | Path,
) -> list[str]:
"""Return the single-platform build command shared by plan and execution.
``BUILDKIT_MULTI_PLATFORM=1`` is intentionally absent. Even with one
explicit target platform, that BuildKit option changes the result into a
manifest list, which the required Docker archive exporter rejects.
"""
role = spec["role"]
image_ref = f"{spec['name']}:{commit}"
return [
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"build",
"--builder",
builder_name,
"--platform",
TARGET["platform"],
"--file",
str(dockerfile),
"--tag",
image_ref,
"--label",
f"org.opencontainers.image.revision={commit}",
"--label",
f"org.opencontainers.image.version={tag}",
"--label",
f"org.opencontainers.image.title={spec['name']}",
"--label",
f"io.yuqei.release.role={role}",
"--build-arg",
f"SOURCE_DATE_EPOCH={source_date_epoch}",
"--build-arg",
f"{spec['base_image']['build_arg']}={spec['base_image']['transport_reference']}",
"--pull",
"--no-cache",
"--provenance=false",
"--sbom=false",
"--output",
f"type=oci,dest={oci_output}",
"--output",
f"type=docker,dest={docker_output}",
".",
]
def create_build_plan(
*,
repo: Path,
source_archive: Path,
source_manifest: Path,
commit: str,
tag: str,
builder_name: str,
output_dir: Path,
policy_path: Path = DEFAULT_POLICY_PATH,
) -> dict[str, Any]:
validated = validate_release_inputs(
repo=repo,
source_archive=source_archive,
source_manifest=source_manifest,
commit=commit,
tag=tag,
policy_path=policy_path,
)
source_date_epoch = int(
parse_rfc3339(
validated["source_manifest"]["candidate"]["commit_timestamp"],
label="candidate.commit_timestamp",
).timestamp()
)
if not BUILDER_NAME_RE.fullmatch(builder_name):
raise ReleaseValidationError("--builder must be an explicit safe Buildx builder name")
source_file_records = {
item["path"]: item for item in validated["source_manifest"]["files"]["items"]
}
required_gate_sources = [
COMPOSE_GATE_POLICY["validator"],
*COMPOSE_GATE_POLICY["compose_inputs"],
]
missing_gate_sources = [path for path in required_gate_sources if path not in source_file_records]
if missing_gate_sources:
raise ReleaseValidationError(
f"source manifest lacks production Compose gate inputs: {missing_gate_sources}"
)
synthetic_env_sha256 = sha256_bytes(synthetic_compose_gate_env_bytes())
compose_gate_commands: list[list[str]] = []
for _profile_name, profiles in TRUSTED_PROFILE_CASES:
for mode in ("online", "offline"):
compose_inputs = COMPOSE_GATE_POLICY["compose_inputs"][: 3 if mode == "offline" else 2]
command = [
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"compose",
"--project-name",
"yuqei-rc04-structural-gate",
"--env-file",
"<output-dir>/validation/synthetic-compose-gate.env",
]
for compose_input in compose_inputs:
command.extend(("-f", f"<source-root>/{compose_input}"))
for profile in profiles:
command.extend(("--profile", profile))
command.extend(("config", "--format", "json"))
compose_gate_commands.append(command)
images = []
for sequence, spec in enumerate(validated["policy"]["images"], 1):
role = spec["role"]
image_ref = f"{spec['name']}:{commit}"
oci_path = f"images/{role}.oci.tar"
docker_path = f"images-docker/{role}.docker.tar"
sbom_path = f"sbom/{role}.spdx.json"
scan_path = f"scans/{role}.trivy.json"
license_scan_path = f"licenses/{role}.trivy-license.json"
images.append(
{
"sequence": sequence,
"role": role,
"image_ref": image_ref,
"environment_key": spec["environment_key"],
"dockerfile": spec["dockerfile"],
"base_image": spec["base_image"],
"outputs": {
"oci": oci_path,
"docker_archive": docker_path,
"sbom": sbom_path,
"scan": scan_path,
"license_scan": license_scan_path,
},
"commands": [
_image_build_command(
spec=spec,
commit=commit,
tag=tag,
builder_name=builder_name,
source_date_epoch=source_date_epoch,
dockerfile=spec["dockerfile"],
oci_output=f"<output-dir>/{oci_path}",
docker_output=f"<output-dir>/{docker_path}",
),
[
PINNED_EXECUTABLES["syft"],
"scan",
f"oci-archive:<output-dir>/{oci_path}",
"--output",
f"spdx-json=<output-dir>/{sbom_path}",
],
[
PINNED_EXECUTABLES["trivy"],
"image",
"--cache-dir",
"<temporary>/trivy-cache",
"--db-repository",
TRIVY_DB_REPOSITORY_TRANSPORT["reference"],
"--input",
f"<output-dir>/{docker_path}",
"--scanners",
"vuln",
"--severity",
"HIGH,CRITICAL",
"--format",
"json",
"--output",
f"<output-dir>/{scan_path}",
"--exit-code",
"1",
"--ignore-unfixed=false",
"--skip-db-update",
"--skip-java-db-update",
],
[
PINNED_EXECUTABLES["trivy"],
"image",
"--cache-dir",
"<temporary>/trivy-cache",
"--input",
f"<output-dir>/{docker_path}",
"--scanners",
"license",
"--severity",
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
"--license-full",
"--offline-scan",
"--format",
"json",
"--output",
f"<output-dir>/{license_scan_path}",
"--skip-db-update",
"--skip-java-db-update",
],
],
}
)
return {
"schema_version": PLAN_SCHEMA,
"execution": False,
"network_or_builder_tools_invoked": False,
"candidate": {
"commit": commit,
"tag": tag,
"tag_object_oid": validated["tag_binding"]["tag_object_oid"],
"tree": validated["source_manifest"]["candidate"]["tree"],
"commit_tree": validated["commit_tree"],
"source_manifest_sha256": validated["source_manifest_sha256"],
"source_manifest_payload_sha256": validated["source_manifest"]["manifest_payload_sha256"],
"source_archive_sha256": validated["archive"]["archive_sha256"],
"source_archive_root_prefix": validated["archive"]["root_prefix"],
},
"policy": {
"path": str(policy_path),
"sha256": validated["policy_sha256"],
"target": TARGET,
"tool_versions": REQUIRED_TOOL_VERSIONS,
"pinned_executables": PINNED_EXECUTABLES,
"docker_host": LOCAL_DOCKER_HOST,
"controlled_environment": CONTROLLED_SUBPROCESS_ENV,
"command_timeouts_seconds": COMMAND_TIMEOUTS_SECONDS,
"trivy_db_repository_transport": TRIVY_DB_REPOSITORY_TRANSPORT,
},
"candidate_checkout": validated["checkout"],
"release_controls": validated["release_controls"],
"output_dir": str(output_dir),
"builder": {
"name": builder_name,
"validation_at_execution": BUILDER_POLICY,
"implicit_default_allowed": False,
},
"compose_gate": {
"execution_at_build": True,
"execution_engine": "self_bound_release_runner_builtin_v1",
"synthetic_environment": True,
"production_secrets_used": False,
"environment_sha256": synthetic_env_sha256,
"validator": {
"path": COMPOSE_GATE_POLICY["validator"],
"sha256": source_file_records[COMPOSE_GATE_POLICY["validator"]]["sha256"],
},
"compose_inputs": [
{
"path": path,
"sha256": source_file_records[path]["sha256"],
}
for path in COMPOSE_GATE_POLICY["compose_inputs"]
],
"case_count": COMPOSE_GATE_POLICY["case_count"],
},
"source_date_epoch": source_date_epoch,
"pre_build_commands": [
["validate", "pinned toolchain versions"],
*compose_gate_commands,
*[
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"imagetools",
"inspect",
reference,
]
for reference in sorted(
{image["base_image"]["transport_reference"] for image in validated["policy"]["images"]}
)
],
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"imagetools",
"inspect",
TRIVY_DB_REPOSITORY_TRANSPORT["reference"],
],
[
PINNED_EXECUTABLES["trivy"],
"image",
"--cache-dir",
"<temporary>/trivy-cache",
"--db-repository",
TRIVY_DB_REPOSITORY_TRANSPORT["reference"],
"--download-db-only",
],
[
PINNED_EXECUTABLES["docker"],
"--host",
LOCAL_DOCKER_HOST,
"buildx",
"imagetools",
"inspect",
TRIVY_DB_REPOSITORY_TRANSPORT["reference"],
],
[
PINNED_EXECUTABLES["trivy"],
"version",
"--cache-dir",
"<temporary>/trivy-cache",
"--format",
"json",
],
],
"parallelism": 1,
"images": images,
"post_build": ["validate every artifact", "write canonical self-hashed build receipt", "publish atomically"],
}
def _copy_input(source: Path, destination: Path) -> None:
if source.is_symlink() or not source.is_file():
raise ReleaseValidationError(f"release input must be a regular non-symlink file: {source}")
destination.parent.mkdir(parents=True, exist_ok=True)
if destination.exists() or destination.is_symlink():
raise ReleaseValidationError(f"refusing to overwrite staged input: {destination}")
shutil.copyfile(source, destination, follow_symlinks=False)
def _write_bytes_exclusive(path: Path, data: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("xb") as handle:
handle.write(data)
def _assert_real_builder_platform() -> None:
machine = platform.machine().casefold()
if not sys.platform.startswith("linux") or machine not in {"x86_64", "amd64"}:
raise ReleaseValidationError(
f"real release builds require Linux x86_64/amd64, found {sys.platform}/{platform.machine()}"
)
def _safe_cleanup_staging(staging: Path, parent: Path) -> None:
try:
resolved = staging.resolve(strict=False)
resolved.relative_to(parent.resolve(strict=True))
except (OSError, ValueError):
return
if staging.name.startswith(".linux-release-partial-") and staging.exists() and not staging.is_symlink():
shutil.rmtree(staging)
def execute_build(
*,
repo: Path,
source_archive: Path,
source_manifest: Path,
commit: str,
tag: str,
builder_name: str,
output_dir: Path,
policy_path: Path = DEFAULT_POLICY_PATH,
runner: Callable[..., subprocess.CompletedProcess[str]] = _run_command,
) -> Path:
_assert_real_builder_platform()
executable_evidence = validate_pinned_executables()
validated = validate_release_inputs(
repo=repo,
source_archive=source_archive,
source_manifest=source_manifest,
commit=commit,
tag=tag,
policy_path=policy_path,
)
tools = validate_toolchain(runner)
builder = validate_buildx_builder(builder_name, runner)
base_image_transports = validate_base_image_transports(validated["policy"], runner)
if output_dir.exists() or output_dir.is_symlink():
raise ReleaseValidationError("output directory must not already exist")
output_parent = output_dir.parent
output_parent.mkdir(parents=True, exist_ok=True)
if output_parent.is_symlink() or not output_parent.is_dir():
raise ReleaseValidationError("output parent must be a real directory")
staging = Path(tempfile.mkdtemp(prefix=".linux-release-partial-", dir=output_parent))
started = datetime.now(timezone.utc)
policy = validated["policy"]
manifest = validated["source_manifest"]
source_date_epoch = int(
parse_rfc3339(manifest["candidate"]["commit_timestamp"], label="candidate.commit_timestamp").timestamp()
)
try:
archive_relative = f"source/source-archive{_archive_suffix(source_archive)}"
manifest_relative = "source/source-manifest.json"
policy_relative = "policy/linux-release-security-policy.v1.json"
staged_archive = staging / archive_relative
staged_manifest = staging / manifest_relative
staged_policy = staging / policy_relative
_copy_input(source_archive, staged_archive)
_copy_input(source_manifest, staged_manifest)
_copy_input(policy_path, staged_policy)
# Re-parse the staged bytes so a source-side replacement between the
# initial validation and copy cannot influence the build unnoticed.
policy = load_policy(staged_policy)
manifest = load_source_manifest(staged_manifest, commit=commit)
if sha256_file(staged_policy) != validated["policy_sha256"]:
raise ReleaseValidationError("staged release policy differs from the validated input")
if sha256_file(staged_manifest) != validated["source_manifest_sha256"]:
raise ReleaseValidationError("staged source manifest differs from the validated input")
staged_archive_summary = verify_source_archive(staged_archive, manifest)
if staged_archive_summary != validated["archive"]:
raise ReleaseValidationError("staged source archive differs from the validated input")
image_receipts: list[dict[str, Any]] = []
with tempfile.TemporaryDirectory(prefix="yuqei-release-source-", dir=output_parent) as temp_source:
source_root = Path(temp_source) / "root"
trivy_cache = Path(temp_source) / "trivy-cache"
syft_cache = Path(temp_source) / "syft-cache"
trivy_cache.mkdir(mode=0o700)
syft_cache.mkdir(mode=0o700)
trivy_env = controlled_subprocess_env(
{"TRIVY_CACHE_DIR": str(trivy_cache)}
)
syft_env = controlled_subprocess_env(
{
"SYFT_CHECK_FOR_APP_UPDATE": "false",
"XDG_CACHE_HOME": str(syft_cache),
}
)
verify_source_archive(staged_archive, manifest, extract_to=source_root)
compose_gate_receipt = run_production_compose_gate(
source_root=source_root,
staging=staging,
manifest=manifest,
policy=policy,
runner=runner,
)
db_transport_before = inspect_trivy_db_repository_transport(policy, runner)
runner(
[
PINNED_EXECUTABLES["trivy"],
"image",
"--cache-dir",
str(trivy_cache),
"--db-repository",
TRIVY_DB_REPOSITORY_TRANSPORT["reference"],
"--download-db-only",
],
env=trivy_env,
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["trivy_db_download"],
)
db_transport_after = inspect_trivy_db_repository_transport(policy, runner)
if (
db_transport_after["resolved_digest"]
!= db_transport_before["resolved_digest"]
):
raise ReleaseValidationError(
"Trivy DB repository tag changed while the database was downloaded"
)
db_transport = {
"reference": db_transport_after["reference"],
"transport": db_transport_after["transport"],
"schema_version": db_transport_after["schema_version"],
"resolved_digest": db_transport_after["resolved_digest"],
"media_type": db_transport_after["media_type"],
"stable_during_download": True,
"before_download_inspect_sha256": db_transport_before["inspect_sha256"],
"after_download_inspect_sha256": db_transport_after["inspect_sha256"],
}
db_completed = runner(
[
PINNED_EXECUTABLES["trivy"],
"version",
"--cache-dir",
str(trivy_cache),
"--format",
"json",
],
env=trivy_env,
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["tool_version"],
)
db_payload = load_json_bytes(
db_completed.stdout.encode("utf-8"), label="Trivy DB metadata"
)
db_validated_at = datetime.now(timezone.utc)
db_summary = validate_trivy_db_metadata(
db_payload, policy, at=db_validated_at
)
db_relative = "security/trivy-db-metadata.json"
db_path = staging / db_relative
_write_bytes_exclusive(db_path, canonical_json_bytes(db_payload))
for sequence, spec in enumerate(policy["images"], 1):
role = spec["role"]
dockerfile = source_root.joinpath(*spec["dockerfile"].split("/"))
if not dockerfile.is_file() or dockerfile.is_symlink():
raise ReleaseValidationError(f"validated source lacks Dockerfile for {role}")
dockerfile_text = dockerfile.read_text(encoding="utf-8")
expected_arg = (
f"ARG {spec['base_image']['build_arg']}="
f"{spec['base_image']['source_reference']}"
)
expected_from = f"FROM ${{{spec['base_image']['build_arg']}}}"
if expected_arg not in dockerfile_text.splitlines() or expected_from not in dockerfile_text.splitlines():
raise ReleaseValidationError(
f"{role} Dockerfile base image does not match the digest-bound transport override policy"
)
image_ref = f"{spec['name']}:{commit}"
oci_relative = f"images/{role}.oci.tar"
docker_relative = f"images-docker/{role}.docker.tar"
sbom_relative = f"sbom/{role}.spdx.json"
scan_relative = f"scans/{role}.trivy.json"
license_scan_relative = f"licenses/{role}.trivy-license.json"
oci_path = staging / oci_relative
docker_path = staging / docker_relative
sbom_path = staging / sbom_relative
scan_path = staging / scan_relative
license_scan_path = staging / license_scan_relative
oci_path.parent.mkdir(parents=True, exist_ok=True)
docker_path.parent.mkdir(parents=True, exist_ok=True)
sbom_path.parent.mkdir(parents=True, exist_ok=True)
scan_path.parent.mkdir(parents=True, exist_ok=True)
license_scan_path.parent.mkdir(parents=True, exist_ok=True)
runner(
_image_build_command(
spec=spec,
commit=commit,
tag=tag,
builder_name=builder_name,
source_date_epoch=source_date_epoch,
dockerfile=dockerfile,
oci_output=oci_path,
docker_output=docker_path,
),
cwd=source_root,
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["image_build"],
)
oci_summary = inspect_oci_archive(oci_path)
if (
oci_summary["revision"] != commit
or oci_summary["version"] != tag
or oci_summary["release_role"] != role
or oci_summary["title"] != spec["name"]
):
raise ReleaseValidationError(f"{role} OCI labels are not bound to the candidate")
docker_summary = inspect_docker_archive(
docker_path, expected_repo_tag=image_ref
)
if (
docker_summary["config_digest"] != oci_summary["config_digest"]
or docker_summary["rootfs_diff_ids"] != oci_summary["rootfs_diff_ids"]
or docker_summary["rootfs_identity_sha256"]
!= oci_summary["rootfs_identity_sha256"]
):
raise ReleaseValidationError(
f"{role} OCI and Docker archives do not share one complete rootfs identity"
)
if (
docker_summary["revision"] != commit
or docker_summary["version"] != tag
or docker_summary["release_role"] != role
or docker_summary["title"] != spec["name"]
):
raise ReleaseValidationError(
f"{role} Docker archive labels are not bound to the candidate"
)
runner(
[
PINNED_EXECUTABLES["syft"],
"scan",
f"oci-archive:{oci_path}",
"--output",
f"spdx-json={sbom_path}",
],
env=syft_env,
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["sbom_generation"],
)
sbom_payload = load_json_file(sbom_path, label=f"{role} SPDX SBOM")
sbom_payload = bind_spdx_image_root(
sbom_payload,
role=role,
manifest_digest=oci_summary["manifest_digest"],
)
canonical_sbom_path = sbom_path.with_name(sbom_path.name + ".canonical")
_write_bytes_exclusive(
canonical_sbom_path, canonical_json_bytes(sbom_payload)
)
os.replace(canonical_sbom_path, sbom_path)
sbom_payload = load_json_file(
sbom_path, label=f"{role} canonical SPDX SBOM"
)
sbom_summary = validate_spdx(
sbom_payload,
policy,
expected_role=role,
expected_manifest_digest=oci_summary["manifest_digest"],
)
# A database that was fresh when the build began may expire while
# four images are built. Re-evaluate freshness immediately before
# every scan; --skip-db-update then holds all scans to those bytes.
db_summary = validate_trivy_db_metadata(
db_payload, policy, at=datetime.now(timezone.utc)
)
scan_completed = runner(
[
PINNED_EXECUTABLES["trivy"],
"image",
"--cache-dir",
str(trivy_cache),
"--db-repository",
TRIVY_DB_REPOSITORY_TRANSPORT["reference"],
"--input",
str(docker_path),
"--scanners",
"vuln",
"--severity",
"HIGH,CRITICAL",
"--format",
"json",
"--output",
str(scan_path),
"--exit-code",
"1",
"--ignore-unfixed=false",
"--skip-db-update",
"--skip-java-db-update",
],
allowed_returncodes={0, 1},
env=trivy_env,
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["vulnerability_scan"],
)
scan_payload = load_json_file(scan_path, label=f"{role} Trivy scan")
scan_evaluated_at = datetime.now(timezone.utc)
scan_summary = validate_trivy_scan(
scan_payload,
policy,
expected_config_digest=oci_summary["config_digest"],
expected_artifact_basename=docker_path.name,
role=role,
at=scan_evaluated_at,
)
expected_scan_returncode = (
1
if any(scan_summary["counts"]["total"].values())
else 0
)
if scan_completed.returncode != expected_scan_returncode:
raise ReleaseValidationError(
f"{role} Trivy return code {scan_completed.returncode} disagrees with "
f"the complete parsed finding count (expected {expected_scan_returncode})"
)
runner(
[
PINNED_EXECUTABLES["trivy"],
"image",
"--cache-dir",
str(trivy_cache),
"--input",
str(docker_path),
"--scanners",
"license",
"--severity",
"UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL",
"--license-full",
"--offline-scan",
"--format",
"json",
"--output",
str(license_scan_path),
"--skip-db-update",
"--skip-java-db-update",
],
env=trivy_env,
timeout_seconds=COMMAND_TIMEOUTS_SECONDS["vulnerability_scan"],
)
license_scan_payload = load_json_file(
license_scan_path, label=f"{role} Trivy license scan"
)
license_scan_summary = validate_trivy_license_scan(
license_scan_payload,
expected_config_digest=oci_summary["config_digest"],
expected_artifact_basename=docker_path.name,
)
image_receipts.append(
{
"sequence": sequence,
"role": role,
"image_name": spec["name"],
"environment_key": spec["environment_key"],
"image_ref": image_ref,
"dockerfile": spec["dockerfile"],
"base_image": spec["base_image"],
"platform": TARGET["platform"],
"oci": file_record(oci_path, oci_relative, **oci_summary),
"docker_archive": file_record(
docker_path, docker_relative, **docker_summary
),
"sbom": file_record(sbom_path, sbom_relative, **sbom_summary),
"scan": file_record(
scan_path,
scan_relative,
scanner="trivy",
scanner_version=tools["trivy"],
**scan_summary,
),
"license_scan": file_record(
license_scan_path,
license_scan_relative,
scanner="trivy",
scanner_version=tools["trivy"],
**license_scan_summary,
),
}
)
db_summary = validate_trivy_db_metadata(
db_payload, policy, at=datetime.now(timezone.utc)
)
final_checkout = validate_candidate_checkout(repo, commit=commit)
if final_checkout != validated["checkout"]:
raise ReleaseValidationError("candidate checkout changed during the release build")
final_release_controls = validate_running_release_controls(
manifest=manifest,
policy_path=staged_policy,
)
if final_release_controls != validated["release_controls"]:
raise ReleaseValidationError("running release controls changed during the release build")
final_tag_binding = validate_tag_binding(
repo,
tag=tag,
commit=commit,
expected_tree=manifest["candidate"]["tree"],
)
if final_tag_binding != validated["tag_binding"]:
raise ReleaseValidationError("annotated tag object changed during the release build")
final_commit_tree = validate_manifest_commit_tree(
repo, commit=commit, manifest=manifest
)
if final_commit_tree != validated["commit_tree"]:
raise ReleaseValidationError("candidate Git tree changed during the release build")
verify_source_archive(staged_archive, manifest)
completed_at = datetime.now(timezone.utc)
vulnerability_counts = {
bucket: {
severity: sum(
image["scan"]["counts"][bucket][severity]
for image in image_receipts
)
for severity in policy["vulnerability_scan"]["thresholds"]
}
for bucket in ("total", "exempted", "blocking")
}
applied_waiver_ids = sorted(
{
waiver_id
for image in image_receipts
for waiver_id in image["scan"]["waiver_ids"]
}
)
license_inventory = {
"scanner": "trivy",
"scanner_version": tools["trivy"],
"report_count": len(image_receipts),
"result_count": sum(
image["license_scan"]["result_count"] for image in image_receipts
),
"license_count": sum(
image["license_scan"]["license_count"] for image in image_receipts
),
"unresolved_count": sum(
image["license_scan"]["unresolved_count"] for image in image_receipts
),
"passed": True,
}
receipt_without_digest: dict[str, Any] = {
"schema_version": BUILD_RECEIPT_SCHEMA,
"candidate": {
"commit": commit,
"tag": tag,
"tag_object_oid": final_tag_binding["tag_object_oid"],
"tree": manifest["candidate"]["tree"],
"commit_tree": final_commit_tree,
"commit_timestamp": manifest["candidate"]["commit_timestamp"],
"source_manifest_payload_sha256": manifest["manifest_payload_sha256"],
"source_manifest": file_record(staged_manifest, manifest_relative),
"source_archive": file_record(
staged_archive,
archive_relative,
root_prefix=validated["archive"]["root_prefix"],
file_count=validated["archive"]["file_count"],
),
},
"policy": file_record(
staged_policy,
policy_relative,
schema_version=policy["schema_version"],
target=TARGET,
),
"builder": {
"os": "linux",
"architecture": "amd64",
"target_platform": TARGET["platform"],
"tools": tools,
"executables": executable_evidence,
"controlled_environment": CONTROLLED_SUBPROCESS_ENV,
"docker_host": LOCAL_DOCKER_HOST,
"command_timeouts_seconds": COMMAND_TIMEOUTS_SECONDS,
"buildx_builder": builder,
"base_image_transports": base_image_transports,
"candidate_checkout": final_checkout,
"release_controls": final_release_controls,
"skopeo_used": False,
},
"build": {
"started_at": format_rfc3339(started),
"completed_at": format_rfc3339(completed_at),
"source_date_epoch": source_date_epoch,
"parallelism": 1,
"image_order": [image["role"] for image in image_receipts],
"compose_gate": compose_gate_receipt,
},
"security": {
"passed": True,
"fail_closed": True,
"thresholds": policy["vulnerability_scan"]["thresholds"],
"exemptions": policy["vulnerability_scan"]["exemptions"],
"vulnerability_counts": vulnerability_counts,
"waiver_ids": applied_waiver_ids,
"database": file_record(
db_path,
db_relative,
repository_transport=db_transport,
**db_summary,
),
"license_inventory": license_inventory,
},
"images": image_receipts,
"claims": [
"The candidate Compose model passed eight synthetic, secret-free online/offline profile cases before image construction.",
"All four application images were built serially for linux/amd64.",
"Every image has OCI and Docker-loadable archives from one solve, an SPDX-2.3 SBOM, a complete Trivy vulnerability report with zero blocking HIGH/CRITICAL findings after only the policy's exact time-limited waivers, and a non-empty resolved Trivy license inventory.",
"This receipt does not claim deployment, migration, health, UAT, or public-release approval.",
],
}
receipt = with_payload_digest(receipt_without_digest, "receipt_payload_sha256")
receipt_path = staging / "build-receipt.json"
_write_bytes_exclusive(receipt_path, canonical_json_bytes(receipt))
os.replace(staging, output_dir)
return output_dir / "build-receipt.json"
except BaseException:
_safe_cleanup_staging(staging, output_parent)
raise
def _write_plan(path: Path, plan: dict[str, Any]) -> None:
if path.exists() or path.is_symlink():
raise ReleaseValidationError("plan output already exists")
path.parent.mkdir(parents=True, exist_ok=True)
_write_bytes_exclusive(path, canonical_json_bytes(plan))
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", type=Path, default=Path.cwd(), help="Git repository containing the exact tag")
parser.add_argument("--source-archive", type=Path, required=True)
parser.add_argument("--source-manifest", type=Path, required=True)
parser.add_argument("--commit", required=True, help="full lowercase 40-character commit object id")
parser.add_argument("--tag", required=True, help="immutable tag that must resolve to --commit")
parser.add_argument(
"--builder",
required=True,
help="explicit single-node docker-container Buildx builder; the implicit default is forbidden",
)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--policy", type=Path, default=DEFAULT_POLICY_PATH)
parser.add_argument("--plan", "--dry-run", action="store_true", dest="plan")
parser.add_argument("--plan-output", type=Path)
args = parser.parse_args(argv)
try:
if args.plan:
plan = create_build_plan(
repo=args.repo.resolve(),
source_archive=args.source_archive.resolve(),
source_manifest=args.source_manifest.resolve(),
commit=args.commit,
tag=args.tag,
builder_name=args.builder,
output_dir=args.output_dir.resolve(),
policy_path=args.policy.resolve(),
)
if args.plan_output:
_write_plan(args.plan_output.resolve(), plan)
print(canonical_json_bytes(plan).decode("utf-8"), end="")
return 0
receipt = execute_build(
repo=args.repo.resolve(),
source_archive=args.source_archive.resolve(),
source_manifest=args.source_manifest.resolve(),
commit=args.commit,
tag=args.tag,
builder_name=args.builder,
output_dir=args.output_dir.resolve(),
policy_path=args.policy.resolve(),
)
print(json.dumps({"status": "passed", "receipt": str(receipt)}, sort_keys=True))
return 0
except ReleaseValidationError as error:
print(f"linux_release_build_error: {error}", file=sys.stderr)
return 2
if __name__ == "__main__":
raise SystemExit(main())
Reference in New Issue View Git Blame Copy Permalink