Merge branch 'codex/build05-object-storage-p0-fix2' into codex/build05-rc06-receipt-integration
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# Production backup and restore verification
|
||||
|
||||
Application backup APIs are not infrastructure backups. Production evidence requires PostgreSQL custom dumps, contract-file archives, MinIO object exports, checksums, and a disposable database restore rehearsal.
|
||||
Application backup APIs are not infrastructure backups. Production evidence requires PostgreSQL custom dumps, contract-file archives, S3 object exports, checksums, and a disposable database restore rehearsal.
|
||||
|
||||
## Backup
|
||||
|
||||
@@ -21,16 +21,58 @@ python3 yuqei-infra/backup/run-production-backup.py \
|
||||
--prune-keep 14
|
||||
```
|
||||
|
||||
An online deployment whose containers retain the immutable `repo@sha256:...`
|
||||
references needs no additional identity input. An offline BUILD-05 install runs
|
||||
the same candidate by local config digest instead. By default the apply path
|
||||
looks under `/var/lib/yuqei/release-package`: it accepts exactly one fully valid
|
||||
successful receipt matching the current release, project, frozen candidate
|
||||
environment, and frozen three-layer Compose input. Zero or multiple matching
|
||||
receipts, drift, links, non-private permissions, or a different owner fail
|
||||
closed. An explicit receipt is an operator override, not a required nightly
|
||||
manual step:
|
||||
|
||||
```bash
|
||||
python3 yuqei-infra/backup/run-production-backup.py \
|
||||
--env-file /etc/yuqei/production.env \
|
||||
--runtime-receipt /var/lib/yuqei/release-package/receipts/APPROVED_RECEIPT.json \
|
||||
--apply --quiesce-writers --prune-keep 14
|
||||
|
||||
printf 'YUQEI_BACKUP_RUNTIME_RECEIPT=/var/lib/yuqei/release-package/receipts/APPROVED_RECEIPT.json\n' \
|
||||
| sudo tee /etc/yuqei/backup-runtime.env
|
||||
```
|
||||
|
||||
The receipt must validate against its frozen execution environment and must
|
||||
bind the current release/project, all candidate manifest references, imported
|
||||
config digests, and the deployed RustFS container. A raw config digest without
|
||||
that candidate binding is rejected. The release-package apply and the timer
|
||||
must run as the same dedicated `yuqei-ops` account so its `0700` state and
|
||||
`0600`/`0400` receipts and snapshots remain both readable and private. If a
|
||||
different protected state root is required, set
|
||||
`YUQEI_BACKUP_RUNTIME_STATE_DIR` in `/etc/yuqei/backup-runtime.env`; do not copy
|
||||
or loosen the receipt files.
|
||||
|
||||
`BACKUP_HOST_DIR` must normally be a separate NAS/NFS/OSS-mounted filesystem. `--allow-local-target` exists only for rehearsals and cannot be used as production disaster-recovery evidence.
|
||||
|
||||
Each completed directory contains:
|
||||
|
||||
- three PostgreSQL custom-format dumps;
|
||||
- `contract-files.tar.gz`;
|
||||
- a MinIO bucket mirror;
|
||||
- `manifest.json` with release ID, image digests, sizes, and SHA-256 values.
|
||||
- an object-storage bucket mirror (the `minio/` directory name is retained only for package compatibility);
|
||||
- a v2 `manifest.json` with the RustFS engine/client versions and immutable image identities, release ID, sizes, and SHA-256 values.
|
||||
|
||||
Incomplete runs remain as `.backup-*.partial` with a `FAILED` marker and are never treated as completed backups.
|
||||
The backup client is the pinned RustFS `rc` image and direct client runs use
|
||||
`--pull=never`. Credentials are omitted from command arguments, container
|
||||
environment, and host temporary files; the runner streams a TOML document over
|
||||
standard input into a private tmpfs-backed client home. The backup container
|
||||
runs as the invoking host UID/GID with a read-only root filesystem, no Linux
|
||||
capabilities, and `no-new-privileges`.
|
||||
|
||||
The v2 manifest, every artifact, the declared image identity, and the running RustFS/client image IDs are verified while the run still has a `.backup-*.partial` name. Files and directories are flushed before one same-filesystem atomic rename publishes `backup-*`. Caught failures receive a `FAILED` marker; an uncatchable host/power loss can leave an unmarked partial directory, but partial directories are never treated or pruned as completed backups.
|
||||
|
||||
When writers are quiesced, every writer is restarted independently in a bounded
|
||||
`finally` path and its running state is checked. A failed restart or SIGTERM/
|
||||
SIGINT quarantines any completed-name directory back to `.partial`, writes
|
||||
`failure.json`, and prevents `PRODUCTION_BACKUP_PASS`.
|
||||
|
||||
## Restore check
|
||||
|
||||
@@ -51,7 +93,27 @@ python3 yuqei-infra/backup/verify-production-backup.py \
|
||||
--apply
|
||||
```
|
||||
|
||||
The script creates only generated `yuqei_restore_check_*` databases and drops them in `finally`. It never restores into the configured production database names. Contract-file and MinIO artifacts are integrity-checked but are not written into production volumes by this command.
|
||||
For RC07, provide `--environment-id` and `--consistency-set-id` when they differ
|
||||
from the protected environment value and backup ID. A successful three-database
|
||||
restore of an artifacts-v2 backup also emits
|
||||
`consistent-backup-manifest.json` in the frozen
|
||||
`yuqei-production-backup.v3` shape consumed by the first-release recovery
|
||||
planner. The v3 document carries an external path/SHA-256 record for the raw
|
||||
artifacts-v2 manifest, the application and RustFS/rc runtime identities, and the
|
||||
restored artifact inventory. The planner reopens that original manifest and
|
||||
cross-checks it against the current release package's online references and
|
||||
offline config image IDs. Legacy `yuqei-production-backup.v2` documents remain
|
||||
readable for non-writing plans but cannot mint a new install-readiness receipt.
|
||||
The restore-check summary is versioned and self-hashed and binds both the raw
|
||||
artifacts-v2 manifest and that adapter. It deliberately sets
|
||||
`independent_restore_test_claimed=false`; host snapshot restore and TLS binding
|
||||
still require the separate isolated-UAT evidence expected by RC07.
|
||||
|
||||
The script creates only generated `yuqei_restore_check_*` databases and drops them in `finally`. It never restores into the configured production database names. Contract-file and object-storage artifacts are integrity-checked but are not written into production volumes by this command. The verifier remains read-compatible with legacy v1 MinIO manifests but new backups write only v2 manifests.
|
||||
|
||||
## MinIO-to-RustFS volume boundary
|
||||
|
||||
RustFS uses the new `rustfs-data` Compose volume and the production example names it `yuqei-v2-production-rustfs-v1`. Never point that volume at the historical MinIO volume. The RustFS container checks the volume on every start: it rejects `.minio.sys`, linked/invalid ownership markers, and an unowned non-empty volume; after the private marker is established, legitimate RustFS data is allowed across restarts. Migration must be a logical S3 export/import with the old MinIO volume retained unchanged for rollback; direct block/file-level reuse is prohibited.
|
||||
|
||||
## Scheduling
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -22,12 +22,29 @@ PRODUCTION_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.pro
|
||||
DEFAULT_ARTIFACT_ROOT = REPO_ROOT / "project-docs" / "qa-artifacts" / "v2-production-restore-check"
|
||||
SAFE_DATABASE_PATTERN = re.compile(r"^yuqei_restore_check_(contract|ai_platform|operations)_[0-9]{14}$")
|
||||
S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$")
|
||||
BACKUP_RUN_PATTERN = re.compile(r"^backup-[0-9]{8}T[0-9]{6}Z$")
|
||||
DOCKER_PROJECT_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
|
||||
IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
|
||||
IMMUTABLE_IMAGE_PATTERN = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$")
|
||||
IMAGE_ID_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$")
|
||||
V2_BACKUP_SCHEMA = "yuqei-production-backup-artifacts.v2"
|
||||
PLANNER_BACKUP_MANIFEST_SCHEMA_V2 = "yuqei-production-backup.v2"
|
||||
PLANNER_BACKUP_MANIFEST_SCHEMA = "yuqei-production-backup.v3"
|
||||
RESTORE_CHECK_SUMMARY_SCHEMA = "yuqei.production-restore-check-summary.v1"
|
||||
OBJECT_STORAGE_ENGINE_VERSION = "1.0.0"
|
||||
OBJECT_STORAGE_CLIENT_VERSION = "0.1.36"
|
||||
REQUIRED_BACKUP_ARTIFACTS = {
|
||||
"postgres-contract.dump",
|
||||
"postgres-ai_platform.dump",
|
||||
"postgres-operations.dump",
|
||||
"contract-files.tar.gz",
|
||||
}
|
||||
RECOVERY_IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/+-]{0,255}$")
|
||||
APPLICATION_RUNTIME_IMAGE_KEYS = {
|
||||
"contract_api": "CONTRACT_API_IMAGE",
|
||||
"ai_platform_api": "AI_PLATFORM_API_IMAGE",
|
||||
"ai_worker": "AI_WORKER_IMAGE",
|
||||
}
|
||||
|
||||
|
||||
def read_env(path: Path) -> dict[str, str]:
|
||||
@@ -44,10 +61,81 @@ def read_env(path: Path) -> dict[str, str]:
|
||||
return values
|
||||
|
||||
|
||||
def _validate_v2_manifest_envelope(
|
||||
backup_dir: Path,
|
||||
manifest: dict[str, object],
|
||||
) -> None:
|
||||
expected_fields = {
|
||||
"schema_version",
|
||||
"backup_id",
|
||||
"created_at",
|
||||
"release_id",
|
||||
"compose_project_name",
|
||||
"consistency_mode",
|
||||
"database_names",
|
||||
"images",
|
||||
"object_storage_mirror",
|
||||
"artifacts",
|
||||
}
|
||||
if set(manifest) != expected_fields:
|
||||
raise RuntimeError("V2 backup manifest fields are invalid.")
|
||||
if (
|
||||
not BACKUP_RUN_PATTERN.fullmatch(backup_dir.name)
|
||||
or manifest.get("backup_id") != backup_dir.name
|
||||
):
|
||||
raise RuntimeError("V2 backup ID does not match its directory.")
|
||||
created_at = manifest.get("created_at")
|
||||
try:
|
||||
parsed_at = datetime.fromisoformat(str(created_at).replace("Z", "+00:00"))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise RuntimeError("V2 backup creation timestamp is invalid.") from exc
|
||||
if not isinstance(created_at, str) or parsed_at.tzinfo is None:
|
||||
raise RuntimeError("V2 backup creation timestamp must include a timezone.")
|
||||
if not isinstance(manifest.get("release_id"), str) or not manifest["release_id"].strip():
|
||||
raise RuntimeError("V2 backup release identity is missing.")
|
||||
project_name = manifest.get("compose_project_name")
|
||||
if not isinstance(project_name, str) or not DOCKER_PROJECT_PATTERN.fullmatch(project_name):
|
||||
raise RuntimeError("V2 backup Compose project identity is invalid.")
|
||||
if manifest.get("consistency_mode") not in {"online", "quiesced"}:
|
||||
raise RuntimeError("V2 backup consistency mode is invalid.")
|
||||
databases = manifest.get("database_names")
|
||||
if (
|
||||
not isinstance(databases, dict)
|
||||
or set(databases) != {"contract", "ai_platform", "operations"}
|
||||
or any(
|
||||
not isinstance(value, str) or not IDENTIFIER_PATTERN.fullmatch(value)
|
||||
for value in databases.values()
|
||||
)
|
||||
):
|
||||
raise RuntimeError("V2 backup database inventory is invalid.")
|
||||
images = manifest.get("images")
|
||||
if (
|
||||
not isinstance(images, dict)
|
||||
or set(images) != {"contract_api", "ai_platform_api", "ai_worker"}
|
||||
or any(
|
||||
not isinstance(value, str)
|
||||
or not (
|
||||
IMMUTABLE_IMAGE_PATTERN.fullmatch(value)
|
||||
or IMAGE_ID_PATTERN.fullmatch(value)
|
||||
)
|
||||
for value in images.values()
|
||||
)
|
||||
):
|
||||
raise RuntimeError("V2 backup application image inventory is invalid.")
|
||||
|
||||
|
||||
def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str, object]]]:
|
||||
manifest = json.loads((backup_dir / "manifest.json").read_text(encoding="utf-8"))
|
||||
if manifest.get("schema_version") != "yuqei-production-backup.v1":
|
||||
if not isinstance(manifest, dict):
|
||||
raise RuntimeError("Backup manifest must be a JSON object.")
|
||||
schema_version = manifest.get("schema_version")
|
||||
if schema_version not in {
|
||||
"yuqei-production-backup.v1",
|
||||
V2_BACKUP_SCHEMA,
|
||||
}:
|
||||
raise RuntimeError("Unsupported backup manifest schema.")
|
||||
if schema_version == V2_BACKUP_SCHEMA:
|
||||
_validate_v2_manifest_envelope(backup_dir, manifest)
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list) or not artifacts:
|
||||
raise RuntimeError("Backup manifest has no artifacts.")
|
||||
@@ -58,7 +146,24 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
|
||||
for item in artifacts:
|
||||
if not isinstance(item, dict):
|
||||
raise RuntimeError("Invalid artifact entry.")
|
||||
relative_path = str(item.get("path", ""))
|
||||
if set(item) != {"path", "size_bytes", "sha256"}:
|
||||
raise RuntimeError("Invalid artifact fields.")
|
||||
relative_path = item.get("path")
|
||||
size_bytes = item.get("size_bytes")
|
||||
expected_sha256 = item.get("sha256")
|
||||
if (
|
||||
not isinstance(relative_path, str)
|
||||
or not relative_path
|
||||
or not isinstance(size_bytes, int)
|
||||
or isinstance(size_bytes, bool)
|
||||
or size_bytes < 0
|
||||
or not isinstance(expected_sha256, str)
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", expected_sha256)
|
||||
):
|
||||
raise RuntimeError("Invalid artifact entry.")
|
||||
relative = PurePosixPath(relative_path)
|
||||
if relative.is_absolute() or ".." in relative.parts or "\\" in relative_path:
|
||||
raise RuntimeError("Backup artifact path is unsafe.")
|
||||
if relative_path in artifact_paths:
|
||||
raise RuntimeError(f"Duplicate backup artifact path: {relative_path}")
|
||||
artifact_paths.add(relative_path)
|
||||
@@ -68,7 +173,7 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
|
||||
raise RuntimeError(f"Missing backup artifact: {relative_path}")
|
||||
actual_size = path.stat().st_size
|
||||
actual_sha256 = sha256(path)
|
||||
if actual_size != int(item.get("size_bytes", -1)) or actual_sha256 != item.get("sha256"):
|
||||
if actual_size != size_bytes or actual_sha256 != expected_sha256:
|
||||
raise RuntimeError(f"Backup artifact integrity mismatch: {relative_path}")
|
||||
verified.append({"path": relative_path, "size_bytes": actual_size, "sha256_match": True})
|
||||
if relative_path.startswith("minio/"):
|
||||
@@ -79,7 +184,10 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
|
||||
missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths)
|
||||
if missing:
|
||||
raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}")
|
||||
_verify_minio_mirror(backup_dir, manifest, minio_records)
|
||||
if schema_version == "yuqei-production-backup.v1":
|
||||
_verify_minio_mirror(backup_dir, manifest, minio_records)
|
||||
else:
|
||||
_verify_object_storage_mirror(backup_dir, manifest, minio_records)
|
||||
|
||||
contract_archive = backup_dir / "contract-files.tar.gz"
|
||||
with tarfile.open(contract_archive, mode="r:gz") as archive:
|
||||
@@ -104,6 +212,266 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
|
||||
return manifest, verified
|
||||
|
||||
|
||||
def canonical_sha256(payload: dict[str, object]) -> str:
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
def build_planner_backup_manifest(
|
||||
*,
|
||||
backup_dir: Path,
|
||||
source_manifest: dict[str, object],
|
||||
restore_checks: list[dict[str, str]],
|
||||
environment_id: str,
|
||||
consistency_set_id: str,
|
||||
) -> dict[str, object]:
|
||||
"""Adapt a verified artifacts.v2 backup for the RC07 first-release planner."""
|
||||
|
||||
if source_manifest.get("schema_version") != V2_BACKUP_SCHEMA:
|
||||
raise RuntimeError("Only artifacts.v2 backups can produce a recovery-planner manifest.")
|
||||
validated_manifest, _verified = validate_backup(backup_dir)
|
||||
if validated_manifest != source_manifest:
|
||||
raise RuntimeError("Recovery adapter input differs from the verified backup manifest.")
|
||||
if source_manifest.get("consistency_mode") != "quiesced":
|
||||
raise RuntimeError("Recovery-planner manifest requires a quiesced backup.")
|
||||
if not RECOVERY_IDENTIFIER_PATTERN.fullmatch(environment_id):
|
||||
raise RuntimeError("Recovery environment identity is invalid.")
|
||||
if not RECOVERY_IDENTIFIER_PATTERN.fullmatch(consistency_set_id):
|
||||
raise RuntimeError("Recovery consistency-set identity is invalid.")
|
||||
if len(restore_checks) != 3 or any(not isinstance(item, dict) for item in restore_checks):
|
||||
raise RuntimeError("Exactly three database restore checks are required for recovery planning.")
|
||||
by_label = {
|
||||
item.get("label"): item
|
||||
for item in restore_checks
|
||||
if isinstance(item, dict) and isinstance(item.get("label"), str)
|
||||
}
|
||||
if set(by_label) != {"contract", "ai_platform", "operations"}:
|
||||
raise RuntimeError("All three database restore checks are required for recovery planning.")
|
||||
source_database_names = source_manifest.get("database_names")
|
||||
if (
|
||||
not isinstance(source_database_names, dict)
|
||||
or set(source_database_names) != {"contract", "ai_platform", "operations"}
|
||||
):
|
||||
raise RuntimeError("Source backup database inventory is invalid.")
|
||||
expected_validations = {
|
||||
"contract": "contract_api_alembic_version",
|
||||
"ai_platform": "ai_platform_alembic_version",
|
||||
"operations": "public_schema_fingerprint",
|
||||
}
|
||||
for label, item in by_label.items():
|
||||
revision = item.get("schema_revision")
|
||||
database = item.get("database")
|
||||
restore_database = item.get("restore_database")
|
||||
if (
|
||||
item.get("validation") != expected_validations[label]
|
||||
or item.get("validation_value") != revision
|
||||
or not isinstance(database, str)
|
||||
or database != source_database_names[label]
|
||||
or not IDENTIFIER_PATTERN.fullmatch(database)
|
||||
or not isinstance(restore_database, str)
|
||||
or not SAFE_DATABASE_PATTERN.fullmatch(restore_database)
|
||||
or not isinstance(revision, str)
|
||||
or not RECOVERY_IDENTIFIER_PATTERN.fullmatch(revision)
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"{label} restore check does not bind the source database and schema revision."
|
||||
)
|
||||
if not re.fullmatch(r"[0-9a-f]{32}", by_label["operations"]["schema_revision"]):
|
||||
raise RuntimeError("Operations restore check has no valid schema fingerprint.")
|
||||
|
||||
artifacts = source_manifest.get("artifacts")
|
||||
if not isinstance(artifacts, list):
|
||||
raise RuntimeError("Backup artifact inventory is invalid.")
|
||||
by_path = {
|
||||
item.get("path"): item
|
||||
for item in artifacts
|
||||
if isinstance(item, dict) and isinstance(item.get("path"), str)
|
||||
}
|
||||
|
||||
def artifact(relative: str) -> dict[str, object]:
|
||||
item = by_path.get(relative)
|
||||
if (
|
||||
not isinstance(item, dict)
|
||||
or not isinstance(item.get("sha256"), str)
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", item["sha256"])
|
||||
or not isinstance(item.get("size_bytes"), int)
|
||||
or isinstance(item.get("size_bytes"), bool)
|
||||
or item["size_bytes"] <= 0
|
||||
):
|
||||
raise RuntimeError(f"Recovery backup artifact is invalid: {relative}")
|
||||
return item
|
||||
|
||||
database_paths = {
|
||||
"contract": "postgres-contract.dump",
|
||||
"ai": "postgres-ai_platform.dump",
|
||||
"operations": "postgres-operations.dump",
|
||||
}
|
||||
restore_labels = {
|
||||
"contract": "contract",
|
||||
"ai": "ai_platform",
|
||||
"operations": "operations",
|
||||
}
|
||||
databases = {}
|
||||
for planner_name, relative in database_paths.items():
|
||||
item = artifact(relative)
|
||||
databases[planner_name] = {
|
||||
"artifact_sha256": item["sha256"],
|
||||
"size_bytes": item["size_bytes"],
|
||||
"schema_revision": by_label[restore_labels[planner_name]]["schema_revision"],
|
||||
}
|
||||
|
||||
contract_archive = artifact("contract-files.tar.gz")
|
||||
file_count = 0
|
||||
with tarfile.open(backup_dir / "contract-files.tar.gz", mode="r:gz") as archive:
|
||||
file_count = sum(1 for member in archive.getmembers() if member.isfile())
|
||||
object_storage = source_manifest.get("object_storage_mirror")
|
||||
if not isinstance(object_storage, dict):
|
||||
raise RuntimeError("Object-storage mirror metadata is missing.")
|
||||
content_index = object_storage.get("content_index_sha256")
|
||||
object_count = object_storage.get("object_count")
|
||||
total_size = object_storage.get("total_size_bytes")
|
||||
if (
|
||||
not isinstance(content_index, str)
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", content_index)
|
||||
or not isinstance(object_count, int)
|
||||
or isinstance(object_count, bool)
|
||||
or object_count < 0
|
||||
or not isinstance(total_size, int)
|
||||
or isinstance(total_size, bool)
|
||||
or total_size < 0
|
||||
):
|
||||
raise RuntimeError("Object-storage recovery metadata is invalid.")
|
||||
|
||||
source_images = source_manifest.get("images")
|
||||
if not isinstance(source_images, dict):
|
||||
raise RuntimeError("Backup application runtime image inventory is missing.")
|
||||
application_runtime_images = {
|
||||
environment_key: source_images[source_key]
|
||||
for source_key, environment_key in APPLICATION_RUNTIME_IMAGE_KEYS.items()
|
||||
}
|
||||
engine_candidate = object_storage.get("engine_image")
|
||||
engine_runtime = object_storage.get("engine_runtime_image")
|
||||
engine_resolved = object_storage.get("engine_resolved_image_id")
|
||||
client_candidate = object_storage.get("client_image")
|
||||
client_resolved = object_storage.get("client_resolved_image_id")
|
||||
if not all(
|
||||
isinstance(value, str)
|
||||
for value in (
|
||||
engine_candidate,
|
||||
engine_runtime,
|
||||
engine_resolved,
|
||||
client_candidate,
|
||||
client_resolved,
|
||||
)
|
||||
):
|
||||
raise RuntimeError("Object-storage runtime image identity is incomplete.")
|
||||
offline_runtime = bool(IMAGE_ID_PATTERN.fullmatch(str(engine_runtime)))
|
||||
application_offline = {
|
||||
bool(IMAGE_ID_PATTERN.fullmatch(str(value)))
|
||||
for value in application_runtime_images.values()
|
||||
}
|
||||
if application_offline != {offline_runtime}:
|
||||
raise RuntimeError(
|
||||
"Application and object-storage runtime image modes are inconsistent."
|
||||
)
|
||||
client_runtime = client_resolved if offline_runtime else client_candidate
|
||||
source_backup_manifest_path = (backup_dir / "manifest.json").resolve()
|
||||
source_backup_manifest_sha256 = sha256(source_backup_manifest_path)
|
||||
|
||||
unsigned: dict[str, object] = {
|
||||
"schema_version": PLANNER_BACKUP_MANIFEST_SCHEMA,
|
||||
"backup_id": source_manifest["backup_id"],
|
||||
"release_id": source_manifest["release_id"],
|
||||
"environment_id": environment_id,
|
||||
"consistency_set_id": consistency_set_id,
|
||||
"consistency_mode": "quiesced",
|
||||
"databases": databases,
|
||||
"contract_files": {
|
||||
"archive_sha256": contract_archive["sha256"],
|
||||
"size_bytes": contract_archive["size_bytes"],
|
||||
"file_count": file_count,
|
||||
},
|
||||
# The field name is retained by the frozen planner for compatibility;
|
||||
# the artifacts.v2 source is strictly validated as RustFS above.
|
||||
"minio": {
|
||||
"content_index_sha256": content_index,
|
||||
"object_count": object_count,
|
||||
"total_size_bytes": total_size,
|
||||
},
|
||||
"source_backup_schema": V2_BACKUP_SCHEMA,
|
||||
"source_backup_manifest_sha256": source_backup_manifest_sha256,
|
||||
"source_backup_manifest": {
|
||||
"path": str(source_backup_manifest_path),
|
||||
"sha256": source_backup_manifest_sha256,
|
||||
},
|
||||
"application_runtime_images": application_runtime_images,
|
||||
"object_storage_runtime_images": {
|
||||
"MINIO_IMAGE": {
|
||||
"candidate_reference": engine_candidate,
|
||||
"runtime_reference": engine_runtime,
|
||||
"resolved_image_id": engine_resolved,
|
||||
},
|
||||
"MINIO_MC_IMAGE": {
|
||||
"candidate_reference": client_candidate,
|
||||
"runtime_reference": client_runtime,
|
||||
"resolved_image_id": client_resolved,
|
||||
},
|
||||
},
|
||||
"completed_at": source_manifest["created_at"],
|
||||
"status": "complete",
|
||||
}
|
||||
unsigned["manifest_payload_sha256"] = canonical_sha256(unsigned)
|
||||
return unsigned
|
||||
|
||||
|
||||
def build_restore_check_summary(
|
||||
*,
|
||||
checked_at: datetime,
|
||||
source_manifest: dict[str, object],
|
||||
verified: list[dict[str, object]],
|
||||
restored: list[dict[str, str]],
|
||||
source_manifest_sha256: str,
|
||||
planner_manifest_record: dict[str, object] | None,
|
||||
) -> dict[str, object]:
|
||||
if checked_at.tzinfo is None:
|
||||
raise RuntimeError("Restore-check summary timestamp must include a timezone.")
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", source_manifest_sha256):
|
||||
raise RuntimeError("Restore-check source manifest SHA-256 is invalid.")
|
||||
if planner_manifest_record is not None:
|
||||
if (
|
||||
set(planner_manifest_record) != {"schema_version", "path", "sha256", "size_bytes"}
|
||||
or planner_manifest_record.get("schema_version") != PLANNER_BACKUP_MANIFEST_SCHEMA
|
||||
or not isinstance(planner_manifest_record.get("path"), str)
|
||||
or not Path(str(planner_manifest_record["path"])).is_absolute()
|
||||
or not isinstance(planner_manifest_record.get("sha256"), str)
|
||||
or not re.fullmatch(r"[0-9a-f]{64}", str(planner_manifest_record["sha256"]))
|
||||
or not isinstance(planner_manifest_record.get("size_bytes"), int)
|
||||
or isinstance(planner_manifest_record.get("size_bytes"), bool)
|
||||
or int(planner_manifest_record["size_bytes"]) <= 0
|
||||
):
|
||||
raise RuntimeError("Planner backup manifest record is invalid.")
|
||||
unsigned: dict[str, object] = {
|
||||
"schema_version": RESTORE_CHECK_SUMMARY_SCHEMA,
|
||||
"status": "pass",
|
||||
"checked_at": checked_at.isoformat(),
|
||||
"backup_id": source_manifest.get("backup_id"),
|
||||
"source_backup_schema": source_manifest.get("schema_version"),
|
||||
"source_backup_manifest_sha256": source_manifest_sha256,
|
||||
"integrity_artifacts_verified": len(verified),
|
||||
"database_restore_checks": restored,
|
||||
"database_restore_check_executed": bool(restored),
|
||||
"planner_backup_manifest": planner_manifest_record,
|
||||
"independent_restore_test_claimed": False,
|
||||
}
|
||||
unsigned["summary_payload_sha256"] = canonical_sha256(unsigned)
|
||||
return unsigned
|
||||
|
||||
|
||||
def _minio_content_index(records: list[dict[str, object]]) -> str:
|
||||
canonical = json.dumps(
|
||||
sorted(records, key=lambda item: str(item["path"]).encode("utf-8")),
|
||||
@@ -166,6 +534,80 @@ def _verify_minio_mirror(
|
||||
raise RuntimeError("MinIO mirror content index mismatch.")
|
||||
|
||||
|
||||
def _verify_object_storage_mirror(
|
||||
backup_dir: Path,
|
||||
manifest: dict[str, object],
|
||||
records: list[dict[str, object]],
|
||||
) -> None:
|
||||
mirror_root = backup_dir / "minio"
|
||||
if mirror_root.is_symlink() or not mirror_root.is_dir():
|
||||
raise RuntimeError("Object-storage mirror directory is missing or unsafe.")
|
||||
actual_files: set[str] = set()
|
||||
for path in mirror_root.rglob("*"):
|
||||
relative = path.relative_to(backup_dir).as_posix()
|
||||
if path.is_symlink():
|
||||
raise RuntimeError(f"Object-storage mirror contains a symbolic link: {relative}")
|
||||
if path.is_file():
|
||||
actual_files.add(relative)
|
||||
elif not path.is_dir():
|
||||
raise RuntimeError(f"Object-storage mirror contains a special entry: {relative}")
|
||||
if actual_files != {str(item["path"]) for item in records}:
|
||||
raise RuntimeError("Object-storage mirror content differs from the backup manifest.")
|
||||
|
||||
metadata = manifest.get("object_storage_mirror")
|
||||
if not isinstance(metadata, dict):
|
||||
raise RuntimeError("Object-storage mirror metadata is invalid.")
|
||||
expected = {
|
||||
"engine",
|
||||
"engine_version",
|
||||
"engine_image",
|
||||
"client",
|
||||
"client_version",
|
||||
"client_image",
|
||||
"engine_runtime_image",
|
||||
"engine_resolved_image_id",
|
||||
"client_resolved_image_id",
|
||||
"bucket",
|
||||
"path",
|
||||
"object_count",
|
||||
"total_size_bytes",
|
||||
"content_index_sha256",
|
||||
}
|
||||
if set(metadata) != expected or metadata.get("path") != "minio":
|
||||
raise RuntimeError("Object-storage mirror metadata fields are invalid.")
|
||||
if metadata.get("engine") != "rustfs" or metadata.get("client") != "rustfs-rc":
|
||||
raise RuntimeError("Object-storage engine or backup client is unsupported.")
|
||||
if (
|
||||
metadata.get("engine_version") != OBJECT_STORAGE_ENGINE_VERSION
|
||||
or metadata.get("client_version") != OBJECT_STORAGE_CLIENT_VERSION
|
||||
):
|
||||
raise RuntimeError("Object-storage version metadata is invalid.")
|
||||
for key in ("engine_image", "client_image"):
|
||||
value = metadata.get(key)
|
||||
if not isinstance(value, str) or not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", value):
|
||||
raise RuntimeError("Object-storage image identity is not immutable.")
|
||||
if metadata.get("engine_runtime_image") not in {
|
||||
metadata.get("engine_image"),
|
||||
metadata.get("engine_resolved_image_id"),
|
||||
}:
|
||||
raise RuntimeError("Object-storage running image does not match the candidate binding.")
|
||||
for key in ("engine_resolved_image_id", "client_resolved_image_id"):
|
||||
value = metadata.get(key)
|
||||
if not isinstance(value, str) or not IMAGE_ID_PATTERN.fullmatch(value):
|
||||
raise RuntimeError("Object-storage resolved image identity is invalid.")
|
||||
if (
|
||||
not isinstance(metadata.get("bucket"), str)
|
||||
or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"])
|
||||
):
|
||||
raise RuntimeError("Object-storage bucket metadata is invalid.")
|
||||
if metadata.get("object_count") != len(records):
|
||||
raise RuntimeError("Object-storage mirror object count mismatch.")
|
||||
if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records):
|
||||
raise RuntimeError("Object-storage mirror total size mismatch.")
|
||||
if metadata.get("content_index_sha256") != _minio_content_index(records):
|
||||
raise RuntimeError("Object-storage mirror content index mismatch.")
|
||||
|
||||
|
||||
def restore_database_checks(
|
||||
*,
|
||||
backup_dir: Path,
|
||||
@@ -177,8 +619,16 @@ def restore_database_checks(
|
||||
suffix = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
|
||||
restored: list[dict[str, str]] = []
|
||||
targets: list[str] = []
|
||||
source_databases = {
|
||||
"contract": values["CONTRACT_POSTGRES_DB"],
|
||||
"ai_platform": values["AI_PLATFORM_POSTGRES_DB"],
|
||||
"operations": values["OPS_POSTGRES_DB"],
|
||||
}
|
||||
try:
|
||||
for label in ("contract", "ai_platform", "operations"):
|
||||
source_database = source_databases[label]
|
||||
if not IDENTIFIER_PATTERN.fullmatch(source_database):
|
||||
raise RuntimeError("Source backup database name is unsafe.")
|
||||
dump_path = backup_dir / f"postgres-{label}.dump"
|
||||
target_database = f"yuqei_restore_check_{label}_{suffix}"
|
||||
if not SAFE_DATABASE_PATTERN.fullmatch(target_database):
|
||||
@@ -228,9 +678,11 @@ def restore_database_checks(
|
||||
restored.append(
|
||||
{
|
||||
"label": label,
|
||||
"database": target_database,
|
||||
"database": source_database,
|
||||
"restore_database": target_database,
|
||||
"validation": validation_name,
|
||||
"validation_value": version.strip(),
|
||||
"schema_revision": version.strip(),
|
||||
}
|
||||
)
|
||||
return restored
|
||||
@@ -258,8 +710,8 @@ def restore_validation_query(label: str) -> tuple[str, str]:
|
||||
"SELECT version_num FROM ai_platform_alembic_version LIMIT 1",
|
||||
),
|
||||
"operations": (
|
||||
"public_table_count",
|
||||
"SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'",
|
||||
"public_schema_fingerprint",
|
||||
"SELECT md5(COALESCE(string_agg(table_name || ':' || column_name || ':' || data_type || ':' || is_nullable, ',' ORDER BY table_name, ordinal_position), 'empty')) FROM information_schema.columns WHERE table_schema = 'public'",
|
||||
),
|
||||
}
|
||||
try:
|
||||
@@ -340,6 +792,14 @@ def _parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--restore-databases", action="store_true", help="Restore dumps into disposable check databases. Requires --apply.")
|
||||
parser.add_argument("--apply", action="store_true", help="Allow creation and removal of disposable restore-check databases.")
|
||||
parser.add_argument("--artifact-dir", type=Path)
|
||||
parser.add_argument(
|
||||
"--environment-id",
|
||||
help="RC07 source environment identity; defaults to YUQEI_RELEASE_ENVIRONMENT_ID from --env-file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--consistency-set-id",
|
||||
help="RC07 consistency-set identity; defaults to the immutable backup ID.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -349,16 +809,18 @@ def main() -> int:
|
||||
try:
|
||||
manifest, verified = validate_backup(backup_dir)
|
||||
restored: list[dict[str, str]] = []
|
||||
restore_values: dict[str, str] = {}
|
||||
if args.restore_databases:
|
||||
if not args.apply or args.env_file is None:
|
||||
raise RuntimeError("--restore-databases requires --apply and --env-file.")
|
||||
if shutil.which("docker") is None:
|
||||
raise RuntimeError("docker is not available.")
|
||||
env_file = args.env_file.resolve()
|
||||
restore_values = read_env(env_file)
|
||||
restored = restore_database_checks(
|
||||
backup_dir=backup_dir,
|
||||
env_file=env_file,
|
||||
values=read_env(env_file),
|
||||
values=restore_values,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"PRODUCTION_RESTORE_CHECK_FAIL error={type(exc).__name__}: {exc}", file=sys.stderr)
|
||||
@@ -367,14 +829,43 @@ def main() -> int:
|
||||
checked_at = datetime.now(timezone.utc).astimezone()
|
||||
artifact_dir = (args.artifact_dir or DEFAULT_ARTIFACT_ROOT / checked_at.strftime("%Y-%m-%dT%H-%M-%S")).resolve()
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
summary = {
|
||||
"status": "pass",
|
||||
"checked_at": checked_at.isoformat(),
|
||||
"backup_id": manifest.get("backup_id"),
|
||||
"integrity_artifacts_verified": len(verified),
|
||||
"database_restore_checks": restored,
|
||||
"database_restore_check_executed": bool(restored),
|
||||
}
|
||||
planner_manifest_record: dict[str, object] | None = None
|
||||
try:
|
||||
if restored and manifest.get("schema_version") == V2_BACKUP_SCHEMA:
|
||||
environment_id = args.environment_id or restore_values.get(
|
||||
"YUQEI_RELEASE_ENVIRONMENT_ID",
|
||||
"",
|
||||
)
|
||||
consistency_set_id = args.consistency_set_id or str(manifest.get("backup_id", ""))
|
||||
planner_manifest = build_planner_backup_manifest(
|
||||
backup_dir=backup_dir,
|
||||
source_manifest=manifest,
|
||||
restore_checks=restored,
|
||||
environment_id=environment_id,
|
||||
consistency_set_id=consistency_set_id,
|
||||
)
|
||||
planner_path = artifact_dir / "consistent-backup-manifest.json"
|
||||
planner_path.write_text(
|
||||
json.dumps(planner_manifest, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
planner_manifest_record = {
|
||||
"schema_version": PLANNER_BACKUP_MANIFEST_SCHEMA,
|
||||
"path": str(planner_path),
|
||||
"sha256": sha256(planner_path),
|
||||
"size_bytes": planner_path.stat().st_size,
|
||||
}
|
||||
summary = build_restore_check_summary(
|
||||
checked_at=checked_at,
|
||||
source_manifest=manifest,
|
||||
verified=verified,
|
||||
restored=restored,
|
||||
source_manifest_sha256=sha256(backup_dir / "manifest.json"),
|
||||
planner_manifest_record=planner_manifest_record,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"PRODUCTION_RESTORE_CHECK_FAIL error={type(exc).__name__}: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
summary_path = artifact_dir / "production-restore-check-summary.json"
|
||||
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"PRODUCTION_RESTORE_CHECK_PASS artifacts={len(verified)} restored_databases={len(restored)} artifact={summary_path}")
|
||||
|
||||
@@ -8,6 +8,7 @@ Type=oneshot
|
||||
User=yuqei-ops
|
||||
Group=yuqei-ops
|
||||
WorkingDirectory=/opt/apps/ai-contract-platform-v2-production
|
||||
EnvironmentFile=-/etc/yuqei/backup-runtime.env
|
||||
ExecStart=/usr/bin/python3 yuqei-infra/backup/run-production-backup.py --env-file /etc/yuqei/production.env --apply --quiesce-writers --prune-keep 14
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
|
||||
@@ -14,8 +14,8 @@ MINIO_CONSOLE_HOST_PORT=9001
|
||||
CONTRACT_API_HOST_PORT=8100
|
||||
AI_PLATFORM_API_HOST_PORT=8101
|
||||
CONTRACT_WEB_HOST_PORT=3000
|
||||
MINIO_IMAGE=minio/minio:RELEASE.2023-06-23T20-26-00Z
|
||||
MINIO_MC_IMAGE=minio/mc:latest
|
||||
MINIO_IMAGE=rustfs/rustfs:1.0.0
|
||||
MINIO_MC_IMAGE=rustfs/rc:0.1.36
|
||||
MINIO_ROOT_USER=yuqei-minio
|
||||
MINIO_ROOT_PASSWORD=yuqei-minio-local
|
||||
AI_PLATFORM_OBJECT_STORAGE_BUCKET=ai-platform-uploads
|
||||
|
||||
@@ -13,8 +13,8 @@ AI_WORKER_IMAGE=registry.example.com/yuqei/ai-worker@sha256:CHANGE_ME_64_HEX_DIG
|
||||
POSTGRES_IMAGE=pgvector/pgvector@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
POSTGRES_CLIENT_IMAGE=postgres@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
REDIS_IMAGE=redis@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
MINIO_IMAGE=minio/minio@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
MINIO_MC_IMAGE=minio/mc@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
MINIO_IMAGE=rustfs/rustfs@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
MINIO_MC_IMAGE=registry.example.invalid/yuqei/rustfs-rc@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
NGINX_IMAGE=nginx@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
BACKUP_UTILITY_IMAGE=alpine@sha256:CHANGE_ME_64_HEX_DIGEST
|
||||
|
||||
@@ -41,7 +41,7 @@ AI_PLATFORM_API_BIND_ADDRESS=127.0.0.1
|
||||
CONTRACT_WEB_BIND_ADDRESS=127.0.0.1
|
||||
|
||||
POSTGRES_VOLUME_NAME=yuqei-v2-production-postgres
|
||||
MINIO_VOLUME_NAME=yuqei-v2-production-minio
|
||||
MINIO_VOLUME_NAME=yuqei-v2-production-rustfs-v1
|
||||
CONTRACT_FILES_VOLUME_NAME=yuqei-v2-production-contract-files
|
||||
BACKUP_HOST_DIR=/mnt/yuqei-v2-backups
|
||||
BACKUP_REQUIRE_SEPARATE_MOUNT=true
|
||||
@@ -86,6 +86,10 @@ AI_PLATFORM_API_HOST_PORT=8101
|
||||
CONTRACT_WEB_HOST_PORT=3000
|
||||
MINIO_ROOT_USER=CHANGE_ME
|
||||
MINIO_ROOT_PASSWORD=CHANGE_ME
|
||||
OBJECT_STORAGE_ENGINE=rustfs
|
||||
OBJECT_STORAGE_ENGINE_VERSION=1.0.0
|
||||
OBJECT_STORAGE_BACKUP_CLIENT=rustfs-rc
|
||||
OBJECT_STORAGE_BACKUP_CLIENT_VERSION=0.1.36
|
||||
AI_PLATFORM_OBJECT_STORAGE_BUCKET=ai-platform-uploads
|
||||
|
||||
CONTRACT_API_ENVIRONMENT=production
|
||||
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
minio:
|
||||
image: ${MINIO_IMAGE:?MINIO_IMAGE must reference an immutable production image}
|
||||
volumes:
|
||||
- minio-data:/data
|
||||
- rustfs-data:/data
|
||||
|
||||
contract-api-migrate:
|
||||
image: ${CONTRACT_API_IMAGE:?CONTRACT_API_IMAGE must reference an immutable production image}
|
||||
@@ -209,10 +209,7 @@ services:
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command: ["exit 0"]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required}
|
||||
AI_PLATFORM_OBJECT_STORAGE_BUCKET: ${AI_PLATFORM_OBJECT_STORAGE_BUCKET:-ai-platform-uploads}
|
||||
MC_CONFIG_DIR: /tmp/.mc
|
||||
HOME: /tmp/home
|
||||
volumes:
|
||||
- ${BACKUP_HOST_DIR:?BACKUP_HOST_DIR is required}:/backup
|
||||
read_only: true
|
||||
@@ -227,8 +224,10 @@ services:
|
||||
volumes:
|
||||
postgres-data:
|
||||
name: ${POSTGRES_VOLUME_NAME:-yuqei-v2-production-postgres}
|
||||
minio-data:
|
||||
name: ${MINIO_VOLUME_NAME:-yuqei-v2-production-minio}
|
||||
rustfs-data:
|
||||
# This default is intentionally different from the historical MinIO
|
||||
# volume. The preflight service also fails closed on .minio.sys.
|
||||
name: ${MINIO_VOLUME_NAME:-yuqei-v2-production-rustfs-v1}
|
||||
contract-files:
|
||||
name: ${CONTRACT_FILES_VOLUME_NAME:-yuqei-v2-production-contract-files}
|
||||
|
||||
|
||||
@@ -1,4 +1,32 @@
|
||||
services:
|
||||
contract-api-migrate:
|
||||
image: ${CONTRACT_API_IMAGE:?CONTRACT_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
contract-review-worker:
|
||||
image: ${CONTRACT_API_IMAGE:?CONTRACT_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
ai-platform-api-migrate:
|
||||
image: ${AI_PLATFORM_API_IMAGE:?AI_PLATFORM_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
ai-platform-prompt-assets-import:
|
||||
image: ${AI_PLATFORM_API_IMAGE:?AI_PLATFORM_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
ai-platform-api:
|
||||
image: ${AI_PLATFORM_API_IMAGE:?AI_PLATFORM_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
worker:
|
||||
image: ${AI_WORKER_IMAGE:?AI_WORKER_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
contract-web:
|
||||
image: ${CONTRACT_WEB_IMAGE:?CONTRACT_WEB_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
|
||||
kms-gateway-aliyun:
|
||||
image: ${KMS_GATEWAY_IMAGE:?KMS_GATEWAY_IMAGE must reference an immutable UAT image}
|
||||
restart: unless-stopped
|
||||
@@ -133,6 +161,8 @@ services:
|
||||
retries: 8
|
||||
|
||||
contract-api:
|
||||
image: ${CONTRACT_API_IMAGE:?CONTRACT_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
environment: &contract-evidence-trust-uat-environment
|
||||
CONTRACT_API_ENVIRONMENT: staging
|
||||
CONTRACT_API_RELIABLE_EVIDENCE_SIGNING_BACKEND: kms_gateway
|
||||
@@ -165,6 +195,8 @@ services:
|
||||
evidence-witness-tencent: { condition: service_healthy }
|
||||
|
||||
contract-playbook-governance-worker:
|
||||
image: ${CONTRACT_API_IMAGE:?CONTRACT_API_IMAGE must reference the immutable UAT candidate}
|
||||
build: !reset null
|
||||
environment: *contract-evidence-trust-uat-environment
|
||||
volumes: *contract-evidence-trust-uat-volumes
|
||||
depends_on:
|
||||
@@ -176,7 +208,7 @@ services:
|
||||
volumes:
|
||||
postgres-data:
|
||||
name: ${POSTGRES_VOLUME_NAME:?POSTGRES_VOLUME_NAME must be UAT-scoped}
|
||||
minio-data:
|
||||
rustfs-data:
|
||||
name: ${MINIO_VOLUME_NAME:?MINIO_VOLUME_NAME must be UAT-scoped}
|
||||
contract-files:
|
||||
name: ${CONTRACT_FILES_VOLUME_NAME:?CONTRACT_FILES_VOLUME_NAME must be UAT-scoped}
|
||||
|
||||
@@ -63,23 +63,65 @@ services:
|
||||
retries: 5
|
||||
|
||||
minio:
|
||||
image: ${MINIO_IMAGE:-minio/minio:RELEASE.2023-06-23T20-26-00Z}
|
||||
# The service/DNS and environment keys stay stable for the application;
|
||||
# the implementation is RustFS and uses a distinct data volume. The guard
|
||||
# executes in the same container on every start, so an old completed
|
||||
# one-shot container can never bypass the volume check.
|
||||
image: ${MINIO_IMAGE:-rustfs/rustfs:1.0.0}
|
||||
restart: unless-stopped
|
||||
command: server /data --console-address ":9001"
|
||||
user: "10001:10001"
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777
|
||||
- /logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- |
|
||||
set -eu
|
||||
marker=/data/.yuqei-rustfs-volume-v1
|
||||
if [ -e /data/.minio.sys ] || [ -L /data/.minio.sys ]; then
|
||||
echo "Refusing to mount a MinIO-formatted volume in RustFS." >&2
|
||||
exit 42
|
||||
fi
|
||||
if [ -L "$${marker}" ]; then
|
||||
echo "Refusing a linked RustFS volume marker." >&2
|
||||
exit 44
|
||||
elif [ -e "$${marker}" ]; then
|
||||
[ -f "$${marker}" ] && [ "$$(cat "$${marker}")" = "yuqei-rustfs-volume-v1" ] || {
|
||||
echo "Refusing an invalid RustFS volume marker." >&2
|
||||
exit 44
|
||||
}
|
||||
elif [ -n "$$(find /data -mindepth 1 -maxdepth 1 -print -quit)" ]; then
|
||||
echo "Refusing to mount an unowned non-empty object-storage volume." >&2
|
||||
exit 43
|
||||
else
|
||||
umask 077
|
||||
set -C
|
||||
printf '%s\n' 'yuqei-rustfs-volume-v1' > "$${marker}"
|
||||
fi
|
||||
exec /entrypoint.sh rustfs
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:-yuqei-minio}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-yuqei-minio-local}
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_ADDRESS: 0.0.0.0:9000
|
||||
RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001
|
||||
RUSTFS_CONSOLE_ENABLE: "true"
|
||||
RUSTFS_ACCESS_KEY: ${MINIO_ROOT_USER:-yuqei-minio}
|
||||
RUSTFS_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-yuqei-minio-local}
|
||||
RUSTFS_REGION: ${AI_PLATFORM_OBJECT_STORAGE_REGION_NAME:-us-east-1}
|
||||
ports:
|
||||
- "${MINIO_API_BIND_ADDRESS:-0.0.0.0}:${MINIO_API_HOST_PORT:-9000}:9000"
|
||||
- "${MINIO_CONSOLE_BIND_ADDRESS:-0.0.0.0}:${MINIO_CONSOLE_HOST_PORT:-9001}:9001"
|
||||
volumes:
|
||||
- minio-data:/data
|
||||
- rustfs-data:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "test -f /data/.minio.sys/format.json"]
|
||||
test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
start_period: 45s
|
||||
cap_drop: ["ALL"]
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
|
||||
contract-api-migrate:
|
||||
build:
|
||||
@@ -488,5 +530,5 @@ services:
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
minio-data:
|
||||
rustfs-data:
|
||||
contract-files:
|
||||
|
||||
@@ -31,7 +31,12 @@ KEYRING_SCHEMA = "yuqei.first-release-recovery-keyring.v2"
|
||||
APPROVAL_SCHEMA = "yuqei.first-release-recovery-approval.v2"
|
||||
NO_PRIOR_RELEASE_SCHEMA = "yuqei.no-prior-successful-release.v1"
|
||||
SNAPSHOT_RECEIPT_SCHEMA = "yuqei.host-snapshot-receipt.v1"
|
||||
BACKUP_MANIFEST_SCHEMA = "yuqei-production-backup.v2"
|
||||
BACKUP_MANIFEST_SCHEMA_V2 = "yuqei-production-backup.v2"
|
||||
BACKUP_MANIFEST_SCHEMA = "yuqei-production-backup.v3"
|
||||
READ_ONLY_BACKUP_MANIFEST_SCHEMAS = frozenset(
|
||||
{BACKUP_MANIFEST_SCHEMA_V2, BACKUP_MANIFEST_SCHEMA}
|
||||
)
|
||||
SOURCE_BACKUP_MANIFEST_SCHEMA = "yuqei-production-backup-artifacts.v2"
|
||||
RESTORE_TEST_SCHEMA = "yuqei.first-release-restore-test.v1"
|
||||
PLAN_SCHEMA = "yuqei.first-release-snapshot-restore-plan.v2"
|
||||
READINESS_RECEIPT_SCHEMA = "yuqei.first-release-recovery-readiness-receipt.v2"
|
||||
@@ -79,6 +84,8 @@ REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
PACKAGE_VALIDATOR_SOURCE_CLOSURE = (
|
||||
"yuqei-infra/deploy/plan-first-release-snapshot-restore.py",
|
||||
"yuqei-infra/deploy/release-package.py",
|
||||
"yuqei-infra/backup/run-production-backup.py",
|
||||
"yuqei-infra/backup/verify-production-backup.py",
|
||||
"yuqei-infra/deploy/preflight-production.py",
|
||||
"yuqei-infra/deploy/verify-production-compose.py",
|
||||
"yuqei-infra/scripts/build-installable-release.py",
|
||||
@@ -87,6 +94,12 @@ PACKAGE_VALIDATOR_SOURCE_CLOSURE = (
|
||||
"yuqei-infra/release-trust/third-party-materials-baseline.json",
|
||||
"yuqei-infra/scripts/linux_release_common.py",
|
||||
)
|
||||
BACKUP_APPLICATION_IMAGE_KEYS = (
|
||||
"CONTRACT_API_IMAGE",
|
||||
"AI_PLATFORM_API_IMAGE",
|
||||
"AI_WORKER_IMAGE",
|
||||
)
|
||||
BACKUP_OBJECT_STORAGE_IMAGE_KEYS = ("MINIO_IMAGE", "MINIO_MC_IMAGE")
|
||||
MAX_EVIDENCE_AGE = timedelta(hours=24)
|
||||
MAX_CLOCK_SKEW = timedelta(minutes=5)
|
||||
MAX_JSON_BYTES = 8 * 1024 * 1024
|
||||
@@ -170,14 +183,26 @@ def _absolute(path: Path) -> Path:
|
||||
return Path(os.path.abspath(os.fspath(path)))
|
||||
|
||||
|
||||
def _metadata_is_reparse(metadata: os.stat_result) -> bool:
|
||||
reparse_attribute = int(getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400))
|
||||
return bool(
|
||||
stat.S_ISLNK(metadata.st_mode)
|
||||
or int(getattr(metadata, "st_file_attributes", 0)) & reparse_attribute
|
||||
or int(getattr(metadata, "st_reparse_tag", 0)) != 0
|
||||
)
|
||||
|
||||
|
||||
def _reject_symlink_components(path: Path, *, label: str) -> None:
|
||||
absolute = _absolute(path)
|
||||
current = Path(absolute.anchor)
|
||||
for part in absolute.parts[1:]:
|
||||
current /= part
|
||||
try:
|
||||
if current.exists() and current.is_symlink():
|
||||
metadata = os.lstat(current)
|
||||
if _metadata_is_reparse(metadata):
|
||||
raise SnapshotRestorePlanError(f"{label} contains a symbolic link")
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
except OSError as exc:
|
||||
raise SnapshotRestorePlanError(f"cannot inspect {label} path") from exc
|
||||
|
||||
@@ -228,6 +253,135 @@ def load_json(path: Path, *, label: str) -> dict[str, Any]:
|
||||
return payload
|
||||
|
||||
|
||||
def _metadata_state(metadata: os.stat_result) -> tuple[int, ...]:
|
||||
"""Return metadata that must stay stable for one already-open file view."""
|
||||
|
||||
return (
|
||||
metadata.st_dev,
|
||||
metadata.st_ino,
|
||||
metadata.st_mode,
|
||||
metadata.st_nlink,
|
||||
metadata.st_size,
|
||||
metadata.st_mtime_ns,
|
||||
metadata.st_ctime_ns,
|
||||
int(getattr(metadata, "st_file_attributes", 0)),
|
||||
int(getattr(metadata, "st_reparse_tag", 0)),
|
||||
)
|
||||
|
||||
|
||||
def _assert_external_file_metadata(
|
||||
metadata: os.stat_result,
|
||||
*,
|
||||
label: str,
|
||||
max_bytes: int,
|
||||
) -> None:
|
||||
if (
|
||||
not stat.S_ISREG(metadata.st_mode)
|
||||
or metadata.st_nlink != 1
|
||||
or metadata.st_ino == 0
|
||||
or _metadata_is_reparse(metadata)
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
f"{label} must be a single-link non-reparse regular file"
|
||||
)
|
||||
if metadata.st_size <= 0 or metadata.st_size > max_bytes:
|
||||
raise SnapshotRestorePlanError(f"{label} has an invalid size")
|
||||
|
||||
|
||||
def _strict_json_object_from_bytes(raw: bytes, *, label: str) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(
|
||||
raw.decode("utf-8-sig"),
|
||||
object_pairs_hook=_reject_duplicate_pairs,
|
||||
parse_constant=_reject_constant,
|
||||
)
|
||||
except SnapshotRestorePlanError:
|
||||
raise
|
||||
except (UnicodeError, ValueError) as exc:
|
||||
raise SnapshotRestorePlanError(f"{label} is not strict UTF-8 JSON") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise SnapshotRestorePlanError(f"{label} must be a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def _read_stable_external_json(
|
||||
path: Path,
|
||||
*,
|
||||
label: str,
|
||||
expected_sha256: str,
|
||||
max_bytes: int = MAX_JSON_BYTES,
|
||||
) -> tuple[dict[str, Any], Path]:
|
||||
"""Read, hash, parse, and revalidate an external record via one descriptor."""
|
||||
|
||||
lexical = _absolute(path)
|
||||
_reject_symlink_components(lexical, label=label)
|
||||
try:
|
||||
path_before = os.lstat(lexical)
|
||||
except OSError as exc:
|
||||
raise SnapshotRestorePlanError(f"{label} is missing") from exc
|
||||
_assert_external_file_metadata(path_before, label=label, max_bytes=max_bytes)
|
||||
|
||||
flags = (
|
||||
os.O_RDONLY
|
||||
| getattr(os, "O_BINARY", 0)
|
||||
| getattr(os, "O_NOFOLLOW", 0)
|
||||
)
|
||||
try:
|
||||
descriptor = os.open(lexical, flags)
|
||||
except OSError as exc:
|
||||
raise SnapshotRestorePlanError(f"{label} cannot be opened safely") from exc
|
||||
try:
|
||||
opened_before = os.fstat(descriptor)
|
||||
_assert_external_file_metadata(opened_before, label=label, max_bytes=max_bytes)
|
||||
if (
|
||||
not os.path.samestat(path_before, opened_before)
|
||||
or path_before.st_size != opened_before.st_size
|
||||
or path_before.st_mtime_ns != opened_before.st_mtime_ns
|
||||
):
|
||||
raise SnapshotRestorePlanError(f"{label} path identity changed before reading")
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
while True:
|
||||
try:
|
||||
chunk = os.read(descriptor, min(1024 * 1024, max_bytes + 1 - total))
|
||||
except OSError as exc:
|
||||
raise SnapshotRestorePlanError(f"{label} cannot be read safely") from exc
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise SnapshotRestorePlanError(f"{label} has an invalid size")
|
||||
raw = b"".join(chunks)
|
||||
if len(raw) != opened_before.st_size:
|
||||
raise SnapshotRestorePlanError(f"{label} size changed while reading")
|
||||
if hashlib.sha256(raw).hexdigest() != expected_sha256:
|
||||
raise SnapshotRestorePlanError(f"{label} SHA-256 mismatch")
|
||||
payload = _strict_json_object_from_bytes(raw, label=label)
|
||||
|
||||
opened_after = os.fstat(descriptor)
|
||||
_assert_external_file_metadata(opened_after, label=label, max_bytes=max_bytes)
|
||||
if _metadata_state(opened_after) != _metadata_state(opened_before):
|
||||
raise SnapshotRestorePlanError(f"{label} metadata changed while reading")
|
||||
try:
|
||||
path_after = os.lstat(lexical)
|
||||
except OSError as exc:
|
||||
raise SnapshotRestorePlanError(f"{label} path changed while reading") from exc
|
||||
_assert_external_file_metadata(path_after, label=label, max_bytes=max_bytes)
|
||||
if (
|
||||
_metadata_state(path_after) != _metadata_state(path_before)
|
||||
or not os.path.samestat(path_after, opened_after)
|
||||
or path_after.st_size != opened_after.st_size
|
||||
or path_after.st_mtime_ns != opened_after.st_mtime_ns
|
||||
):
|
||||
raise SnapshotRestorePlanError(f"{label} path identity changed while reading")
|
||||
_reject_symlink_components(lexical, label=label)
|
||||
return payload, lexical
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _exact(value: object, fields: set[str], *, label: str) -> dict[str, Any]:
|
||||
if not isinstance(value, dict) or set(value) != fields:
|
||||
raise SnapshotRestorePlanError(f"{label} fields are missing or unknown")
|
||||
@@ -304,10 +458,12 @@ def _external_record(value: object, *, label: str) -> tuple[dict[str, Any], str,
|
||||
if not isinstance(raw_path, str) or not Path(raw_path).is_absolute():
|
||||
raise SnapshotRestorePlanError(f"{label} path must be absolute")
|
||||
expected = _sha(record.get("sha256"), label=f"{label} external SHA-256")
|
||||
path = _regular_file(Path(raw_path), label=label)
|
||||
if file_sha256(path) != expected:
|
||||
raise SnapshotRestorePlanError(f"{label} SHA-256 mismatch")
|
||||
return load_json(path, label=label), expected, path
|
||||
payload, path = _read_stable_external_json(
|
||||
Path(raw_path),
|
||||
label=label,
|
||||
expected_sha256=expected,
|
||||
)
|
||||
return payload, expected, path
|
||||
|
||||
|
||||
def _validate_source_closure(
|
||||
@@ -438,6 +594,53 @@ def _validate_image_map(
|
||||
return result
|
||||
|
||||
|
||||
def _validate_offline_image_ids(
|
||||
value: object,
|
||||
*,
|
||||
application_images: Mapping[str, Mapping[str, str]],
|
||||
infrastructure_images: Mapping[str, Mapping[str, str]],
|
||||
) -> dict[str, str]:
|
||||
expected_images = {**application_images, **infrastructure_images}
|
||||
if not isinstance(value, (list, tuple)) or len(value) != len(expected_images):
|
||||
raise SnapshotRestorePlanError(
|
||||
"official validator Docker import plan does not cover eleven images"
|
||||
)
|
||||
offline_image_ids: dict[str, str] = {}
|
||||
for raw in value:
|
||||
if not isinstance(raw, Mapping):
|
||||
raise SnapshotRestorePlanError(
|
||||
"official validator Docker import plan entry is invalid"
|
||||
)
|
||||
environment_key = raw.get("environment_key")
|
||||
if not isinstance(environment_key, str) or environment_key not in expected_images:
|
||||
raise SnapshotRestorePlanError(
|
||||
"official validator Docker import plan image key is invalid"
|
||||
)
|
||||
if environment_key in offline_image_ids:
|
||||
raise SnapshotRestorePlanError(
|
||||
"official validator Docker import plan repeats an image key"
|
||||
)
|
||||
expected = expected_images[environment_key]
|
||||
loaded_image_id = raw.get("expected_loaded_image_id")
|
||||
if (
|
||||
not isinstance(loaded_image_id, str)
|
||||
or not DIGEST_RE.fullmatch(loaded_image_id)
|
||||
or raw.get("offline_compose_override") != loaded_image_id
|
||||
or raw.get("registry_reference_for_online_release")
|
||||
!= expected["reference"]
|
||||
or raw.get("expected_oci_manifest_digest") != expected["digest"]
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"official validator Docker import plan identity drifted"
|
||||
)
|
||||
offline_image_ids[environment_key] = loaded_image_id
|
||||
if set(offline_image_ids) != set(expected_images):
|
||||
raise SnapshotRestorePlanError(
|
||||
"official validator Docker import plan image inventory is incomplete"
|
||||
)
|
||||
return offline_image_ids
|
||||
|
||||
|
||||
def _validate_package(
|
||||
package_dir: Path,
|
||||
expected_package_sha256: str,
|
||||
@@ -549,6 +752,11 @@ def _validate_package(
|
||||
or getattr(validated, "infrastructure_images", None) != infrastructure_images
|
||||
):
|
||||
raise SnapshotRestorePlanError("official validator eleven-image identity drifted")
|
||||
offline_image_ids = _validate_offline_image_ids(
|
||||
getattr(validated, "docker_import_plan", None),
|
||||
application_images=application_images,
|
||||
infrastructure_images=infrastructure_images,
|
||||
)
|
||||
|
||||
installable = bound_artifacts["installable_manifest"][0]
|
||||
installable_candidate = installable.get("candidate")
|
||||
@@ -626,6 +834,9 @@ def _validate_package(
|
||||
"compose_project_name": package.get("compose_project_name"),
|
||||
"application_image_count": len(application_images),
|
||||
"infrastructure_image_count": len(infrastructure_images),
|
||||
"application_images": application_images,
|
||||
"infrastructure_images": infrastructure_images,
|
||||
"offline_image_ids": offline_image_ids,
|
||||
}
|
||||
|
||||
|
||||
@@ -844,13 +1055,311 @@ def _validate_snapshot_component(
|
||||
return receipt, digest, _timestamp(receipt.get("created_at"), label="snapshot created_at")
|
||||
|
||||
|
||||
def _validate_source_backup_manifest_binding(
|
||||
manifest: Mapping[str, Any],
|
||||
*,
|
||||
package_binding: Mapping[str, Any],
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
source, source_sha, _source_path = _external_record(
|
||||
manifest.get("source_backup_manifest"),
|
||||
label="source artifacts.v2 backup manifest",
|
||||
)
|
||||
if (
|
||||
source_sha != manifest.get("source_backup_manifest_sha256")
|
||||
or source.get("schema_version") != SOURCE_BACKUP_MANIFEST_SCHEMA
|
||||
or manifest.get("source_backup_schema") != SOURCE_BACKUP_MANIFEST_SCHEMA
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"formal backup manifest source artifacts.v2 binding is invalid"
|
||||
)
|
||||
_exact(
|
||||
source,
|
||||
{
|
||||
"schema_version",
|
||||
"backup_id",
|
||||
"created_at",
|
||||
"release_id",
|
||||
"compose_project_name",
|
||||
"consistency_mode",
|
||||
"database_names",
|
||||
"images",
|
||||
"object_storage_mirror",
|
||||
"artifacts",
|
||||
},
|
||||
label="source artifacts.v2 backup manifest",
|
||||
)
|
||||
if (
|
||||
source.get("backup_id") != manifest.get("backup_id")
|
||||
or source.get("created_at") != manifest.get("completed_at")
|
||||
or source.get("release_id") != manifest.get("release_id")
|
||||
or source.get("consistency_mode") != "quiesced"
|
||||
or source.get("compose_project_name")
|
||||
!= package_binding.get("compose_project_name")
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"source artifacts.v2 backup manifest differs from the recovery set"
|
||||
)
|
||||
_timestamp(source.get("created_at"), label="source backup created_at")
|
||||
database_names = _exact(
|
||||
source.get("database_names"),
|
||||
{"contract", "ai_platform", "operations"},
|
||||
label="source backup database names",
|
||||
)
|
||||
for value in database_names.values():
|
||||
_identifier(value, label="source backup database name")
|
||||
|
||||
source_images = _exact(
|
||||
source.get("images"),
|
||||
{"contract_api", "ai_platform_api", "ai_worker"},
|
||||
label="source backup application images",
|
||||
)
|
||||
expected_application = {
|
||||
"CONTRACT_API_IMAGE": source_images["contract_api"],
|
||||
"AI_PLATFORM_API_IMAGE": source_images["ai_platform_api"],
|
||||
"AI_WORKER_IMAGE": source_images["ai_worker"],
|
||||
}
|
||||
if manifest.get("application_runtime_images") != expected_application:
|
||||
raise SnapshotRestorePlanError(
|
||||
"formal backup application images differ from source artifacts.v2"
|
||||
)
|
||||
|
||||
object_storage = _exact(
|
||||
source.get("object_storage_mirror"),
|
||||
{
|
||||
"engine",
|
||||
"engine_version",
|
||||
"engine_image",
|
||||
"client",
|
||||
"client_version",
|
||||
"client_image",
|
||||
"engine_runtime_image",
|
||||
"engine_resolved_image_id",
|
||||
"client_resolved_image_id",
|
||||
"bucket",
|
||||
"path",
|
||||
"object_count",
|
||||
"total_size_bytes",
|
||||
"content_index_sha256",
|
||||
},
|
||||
label="source backup object-storage mirror",
|
||||
)
|
||||
if (
|
||||
object_storage.get("engine") != "rustfs"
|
||||
or object_storage.get("engine_version") != "1.0.0"
|
||||
or object_storage.get("client") != "rustfs-rc"
|
||||
or object_storage.get("client_version") != "0.1.36"
|
||||
or object_storage.get("path") != "minio"
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"source backup object-storage implementation is unsupported"
|
||||
)
|
||||
offline_runtime = isinstance(
|
||||
object_storage.get("engine_runtime_image"), str
|
||||
) and DIGEST_RE.fullmatch(str(object_storage["engine_runtime_image"]))
|
||||
expected_object_storage = {
|
||||
"MINIO_IMAGE": {
|
||||
"candidate_reference": object_storage.get("engine_image"),
|
||||
"runtime_reference": object_storage.get("engine_runtime_image"),
|
||||
"resolved_image_id": object_storage.get("engine_resolved_image_id"),
|
||||
},
|
||||
"MINIO_MC_IMAGE": {
|
||||
"candidate_reference": object_storage.get("client_image"),
|
||||
"runtime_reference": (
|
||||
object_storage.get("client_resolved_image_id")
|
||||
if offline_runtime
|
||||
else object_storage.get("client_image")
|
||||
),
|
||||
"resolved_image_id": object_storage.get("client_resolved_image_id"),
|
||||
},
|
||||
}
|
||||
if manifest.get("object_storage_runtime_images") != expected_object_storage:
|
||||
raise SnapshotRestorePlanError(
|
||||
"formal backup object-storage images differ from source artifacts.v2"
|
||||
)
|
||||
|
||||
raw_artifacts = source.get("artifacts")
|
||||
if not isinstance(raw_artifacts, list) or not raw_artifacts:
|
||||
raise SnapshotRestorePlanError("source backup artifact inventory is missing")
|
||||
artifacts: dict[str, dict[str, Any]] = {}
|
||||
folded_artifact_paths: set[str] = set()
|
||||
for raw in raw_artifacts:
|
||||
record = _exact(
|
||||
raw,
|
||||
{"path", "size_bytes", "sha256"},
|
||||
label="source backup artifact",
|
||||
)
|
||||
relative = _safe_relative(record.get("path"), label="source backup artifact path")
|
||||
size = record.get("size_bytes")
|
||||
digest = record.get("sha256")
|
||||
folded_relative = relative.casefold()
|
||||
if (
|
||||
relative in artifacts
|
||||
or folded_relative in folded_artifact_paths
|
||||
or not isinstance(size, int)
|
||||
or isinstance(size, bool)
|
||||
or size < 0
|
||||
):
|
||||
raise SnapshotRestorePlanError("source backup artifact inventory is invalid")
|
||||
_sha(digest, label="source backup artifact SHA-256")
|
||||
artifacts[relative] = dict(record)
|
||||
folded_artifact_paths.add(folded_relative)
|
||||
|
||||
database_paths = {
|
||||
"contract": "postgres-contract.dump",
|
||||
"ai": "postgres-ai_platform.dump",
|
||||
"operations": "postgres-operations.dump",
|
||||
}
|
||||
databases = manifest.get("databases")
|
||||
if not isinstance(databases, Mapping):
|
||||
raise SnapshotRestorePlanError("formal backup database inventory is missing")
|
||||
for name, relative in database_paths.items():
|
||||
artifact = artifacts.get(relative)
|
||||
record = databases.get(name)
|
||||
if (
|
||||
not isinstance(artifact, Mapping)
|
||||
or not isinstance(record, Mapping)
|
||||
or record.get("artifact_sha256") != artifact.get("sha256")
|
||||
or record.get("size_bytes") != artifact.get("size_bytes")
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
f"formal backup database {name} differs from source artifacts.v2"
|
||||
)
|
||||
contract_artifact = artifacts.get("contract-files.tar.gz")
|
||||
contract_files = manifest.get("contract_files")
|
||||
if (
|
||||
not isinstance(contract_artifact, Mapping)
|
||||
or not isinstance(contract_files, Mapping)
|
||||
or contract_files.get("archive_sha256") != contract_artifact.get("sha256")
|
||||
or contract_files.get("size_bytes") != contract_artifact.get("size_bytes")
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"formal backup contract files differ from source artifacts.v2"
|
||||
)
|
||||
minio_records = [
|
||||
record
|
||||
for path, record in artifacts.items()
|
||||
if path.startswith("minio/")
|
||||
]
|
||||
minio_records.sort(key=lambda item: str(item["path"]).encode("utf-8"))
|
||||
minio = manifest.get("minio")
|
||||
if (
|
||||
not isinstance(minio, Mapping)
|
||||
or minio.get("object_count") != len(minio_records)
|
||||
or minio.get("total_size_bytes")
|
||||
!= sum(int(item["size_bytes"]) for item in minio_records)
|
||||
or minio.get("content_index_sha256") != canonical_sha256(minio_records)
|
||||
or minio.get("object_count") != object_storage.get("object_count")
|
||||
or minio.get("total_size_bytes") != object_storage.get("total_size_bytes")
|
||||
or minio.get("content_index_sha256")
|
||||
!= object_storage.get("content_index_sha256")
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"formal backup object inventory differs from source artifacts.v2"
|
||||
)
|
||||
return source, source_sha
|
||||
|
||||
|
||||
def _validate_formal_backup_image_binding(
|
||||
manifest: Mapping[str, Any],
|
||||
*,
|
||||
package_binding: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
_source, source_backup_sha = _validate_source_backup_manifest_binding(
|
||||
manifest,
|
||||
package_binding=package_binding,
|
||||
)
|
||||
application_images = package_binding.get("application_images")
|
||||
infrastructure_images = package_binding.get("infrastructure_images")
|
||||
offline_image_ids = package_binding.get("offline_image_ids")
|
||||
if (
|
||||
not isinstance(application_images, Mapping)
|
||||
or not isinstance(infrastructure_images, Mapping)
|
||||
or not isinstance(offline_image_ids, Mapping)
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"release package does not expose candidate runtime image identities"
|
||||
)
|
||||
|
||||
application_runtime = _exact(
|
||||
manifest.get("application_runtime_images"),
|
||||
set(BACKUP_APPLICATION_IMAGE_KEYS),
|
||||
label="backup application runtime images",
|
||||
)
|
||||
runtime_modes: set[str] = set()
|
||||
normalized_application: dict[str, str] = {}
|
||||
for key in BACKUP_APPLICATION_IMAGE_KEYS:
|
||||
candidate = application_images.get(key)
|
||||
offline_image_id = offline_image_ids.get(key)
|
||||
runtime_identity = application_runtime.get(key)
|
||||
if (
|
||||
not isinstance(candidate, Mapping)
|
||||
or not isinstance(candidate.get("reference"), str)
|
||||
or not isinstance(offline_image_id, str)
|
||||
or not DIGEST_RE.fullmatch(offline_image_id)
|
||||
or not isinstance(runtime_identity, str)
|
||||
or runtime_identity not in {candidate["reference"], offline_image_id}
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
f"backup application runtime image {key} differs from the release package"
|
||||
)
|
||||
runtime_modes.add(
|
||||
"online" if runtime_identity == candidate["reference"] else "offline"
|
||||
)
|
||||
normalized_application[key] = runtime_identity
|
||||
|
||||
object_storage_runtime = _exact(
|
||||
manifest.get("object_storage_runtime_images"),
|
||||
set(BACKUP_OBJECT_STORAGE_IMAGE_KEYS),
|
||||
label="backup object-storage runtime images",
|
||||
)
|
||||
normalized_object_storage: dict[str, dict[str, str]] = {}
|
||||
for key in BACKUP_OBJECT_STORAGE_IMAGE_KEYS:
|
||||
record = _exact(
|
||||
object_storage_runtime.get(key),
|
||||
{"candidate_reference", "runtime_reference", "resolved_image_id"},
|
||||
label=f"backup object-storage runtime image {key}",
|
||||
)
|
||||
candidate = infrastructure_images.get(key)
|
||||
offline_image_id = offline_image_ids.get(key)
|
||||
if (
|
||||
not isinstance(candidate, Mapping)
|
||||
or not isinstance(candidate.get("reference"), str)
|
||||
or record.get("candidate_reference") != candidate["reference"]
|
||||
or not isinstance(offline_image_id, str)
|
||||
or not DIGEST_RE.fullmatch(offline_image_id)
|
||||
or record.get("resolved_image_id") != offline_image_id
|
||||
or record.get("runtime_reference")
|
||||
not in {candidate["reference"], offline_image_id}
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
f"backup object-storage runtime image {key} differs from the release package"
|
||||
)
|
||||
runtime_modes.add(
|
||||
"online"
|
||||
if record["runtime_reference"] == candidate["reference"]
|
||||
else "offline"
|
||||
)
|
||||
normalized_object_storage[key] = dict(record)
|
||||
if len(runtime_modes) != 1:
|
||||
raise SnapshotRestorePlanError(
|
||||
"backup runtime image identities mix online and offline release modes"
|
||||
)
|
||||
return {
|
||||
"source_backup_manifest_sha256": source_backup_sha,
|
||||
"runtime_mode": next(iter(runtime_modes)),
|
||||
"application_runtime_images": normalized_application,
|
||||
"object_storage_runtime_images": normalized_object_storage,
|
||||
}
|
||||
|
||||
|
||||
def _validate_backup_components(
|
||||
postgres_value: object,
|
||||
object_value: object,
|
||||
*,
|
||||
release_id: str,
|
||||
package_binding: Mapping[str, Any],
|
||||
environment_id: str,
|
||||
consistency_set_id: str,
|
||||
require_formal_backup_manifest: bool = False,
|
||||
) -> tuple[dict[str, Any], str, datetime]:
|
||||
postgres = _exact(
|
||||
postgres_value,
|
||||
@@ -873,27 +1382,44 @@ def _validate_backup_components(
|
||||
manifest, digest, _path = _external_record(
|
||||
postgres.get("backup_manifest"), label="consistent backup manifest"
|
||||
)
|
||||
schema_version = manifest.get("schema_version")
|
||||
if schema_version not in READ_ONLY_BACKUP_MANIFEST_SCHEMAS:
|
||||
raise SnapshotRestorePlanError("unsupported consistent backup manifest schema")
|
||||
if require_formal_backup_manifest and schema_version != BACKUP_MANIFEST_SCHEMA:
|
||||
raise SnapshotRestorePlanError(
|
||||
"legacy v2 backup manifest is read-only and cannot mint new release evidence"
|
||||
)
|
||||
expected_fields = {
|
||||
"schema_version",
|
||||
"backup_id",
|
||||
"release_id",
|
||||
"environment_id",
|
||||
"consistency_set_id",
|
||||
"consistency_mode",
|
||||
"databases",
|
||||
"contract_files",
|
||||
"minio",
|
||||
"completed_at",
|
||||
"status",
|
||||
"manifest_payload_sha256",
|
||||
}
|
||||
if schema_version == BACKUP_MANIFEST_SCHEMA:
|
||||
expected_fields.update(
|
||||
{
|
||||
"source_backup_schema",
|
||||
"source_backup_manifest_sha256",
|
||||
"source_backup_manifest",
|
||||
"application_runtime_images",
|
||||
"object_storage_runtime_images",
|
||||
}
|
||||
)
|
||||
_exact(
|
||||
manifest,
|
||||
{
|
||||
"schema_version",
|
||||
"backup_id",
|
||||
"release_id",
|
||||
"environment_id",
|
||||
"consistency_set_id",
|
||||
"consistency_mode",
|
||||
"databases",
|
||||
"contract_files",
|
||||
"minio",
|
||||
"completed_at",
|
||||
"status",
|
||||
"manifest_payload_sha256",
|
||||
},
|
||||
expected_fields,
|
||||
label="consistent backup manifest",
|
||||
)
|
||||
if manifest.get("schema_version") != BACKUP_MANIFEST_SCHEMA:
|
||||
raise SnapshotRestorePlanError("unsupported consistent backup manifest schema")
|
||||
_embedded_hash(manifest, "manifest_payload_sha256", label="consistent backup manifest")
|
||||
release_id = _identifier(package_binding.get("release_id"), label="release_id")
|
||||
if (
|
||||
manifest.get("release_id") != release_id
|
||||
or manifest.get("environment_id") != environment_id
|
||||
@@ -933,6 +1459,11 @@ def _validate_backup_components(
|
||||
):
|
||||
if any(not isinstance(record.get(field), int) or isinstance(record.get(field), bool) or record[field] < 0 for field in fields):
|
||||
raise SnapshotRestorePlanError(f"{label} backup counters are invalid")
|
||||
if schema_version == BACKUP_MANIFEST_SCHEMA:
|
||||
_validate_formal_backup_image_binding(
|
||||
manifest,
|
||||
package_binding=package_binding,
|
||||
)
|
||||
return manifest, digest, _timestamp(manifest.get("completed_at"), label="backup completed_at")
|
||||
|
||||
|
||||
@@ -1096,6 +1627,7 @@ def validate_evidence(
|
||||
keyring: Mapping[str, Any],
|
||||
trusted_root_public_key: Ed25519PublicKey,
|
||||
now: datetime,
|
||||
require_formal_backup_manifest: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
if now.tzinfo is None:
|
||||
raise SnapshotRestorePlanError("validation time must include a timezone")
|
||||
@@ -1185,9 +1717,10 @@ def validate_evidence(
|
||||
backup, backup_sha, backup_at = _validate_backup_components(
|
||||
components["postgres_consistent_backup"],
|
||||
components["object_storage_state"],
|
||||
release_id=package_binding["release_id"],
|
||||
package_binding=package_binding,
|
||||
environment_id=environment_id,
|
||||
consistency_set_id=consistency_set_id,
|
||||
require_formal_backup_manifest=require_formal_backup_manifest,
|
||||
)
|
||||
bindings = _validate_bindings(components["release_environment_and_tls_bindings"])
|
||||
bindings_sha = canonical_sha256(bindings)
|
||||
@@ -1252,7 +1785,11 @@ def validate_evidence(
|
||||
},
|
||||
"backup": {
|
||||
"backup_id": backup["backup_id"],
|
||||
"schema_version": backup["schema_version"],
|
||||
"manifest_sha256": backup_sha,
|
||||
"source_backup_manifest_sha256": backup.get(
|
||||
"source_backup_manifest_sha256"
|
||||
),
|
||||
"databases": list(REQUIRED_DATABASES),
|
||||
"storage_targets": list(REQUIRED_STORAGE_TARGETS),
|
||||
},
|
||||
@@ -1345,6 +1882,15 @@ def build_install_readiness_receipt(
|
||||
) -> dict[str, Any]:
|
||||
if generated_at.tzinfo is None:
|
||||
raise SnapshotRestorePlanError("receipt time must include a timezone")
|
||||
backup = validated_evidence.get("backup")
|
||||
if (
|
||||
not isinstance(backup, Mapping)
|
||||
or backup.get("schema_version") != BACKUP_MANIFEST_SCHEMA
|
||||
or not isinstance(backup.get("source_backup_manifest_sha256"), str)
|
||||
):
|
||||
raise SnapshotRestorePlanError(
|
||||
"formal install readiness requires a verified v3 backup manifest"
|
||||
)
|
||||
approval_expiries = [
|
||||
_timestamp(item["expires_at"], label="approval expires_at")
|
||||
for item in validated_evidence["approvals"]
|
||||
@@ -1472,6 +2018,7 @@ def _verify_raw_inputs(
|
||||
now: datetime,
|
||||
package_validator_for_tests: Callable[..., Any] | None = None,
|
||||
require_installable_package_v2: bool = False,
|
||||
require_formal_backup_manifest: bool = False,
|
||||
) -> tuple[dict[str, Any], dict[str, Any], str]:
|
||||
package_binding = _validate_package(
|
||||
package_dir,
|
||||
@@ -1489,6 +2036,7 @@ def _verify_raw_inputs(
|
||||
keyring=keyring,
|
||||
trusted_root_public_key=root_key,
|
||||
now=now,
|
||||
require_formal_backup_manifest=require_formal_backup_manifest,
|
||||
)
|
||||
validated["raw_artifact_sha256"] = {
|
||||
"evidence": file_sha256(
|
||||
@@ -1531,6 +2079,7 @@ def verify_and_build_output(
|
||||
now=now,
|
||||
package_validator_for_tests=package_validator_for_tests,
|
||||
require_installable_package_v2=(output_kind == "install-readiness-receipt"),
|
||||
require_formal_backup_manifest=(output_kind == "install-readiness-receipt"),
|
||||
)
|
||||
if output_kind == "plan":
|
||||
return build_plan(
|
||||
@@ -1601,6 +2150,7 @@ def validate_install_readiness(
|
||||
trusted_root_public_key_file=trusted_root_public_key_file,
|
||||
now=now,
|
||||
require_installable_package_v2=True,
|
||||
require_formal_backup_manifest=True,
|
||||
)
|
||||
rebuilt = build_install_readiness_receipt(
|
||||
package_binding=package_binding,
|
||||
|
||||
@@ -159,6 +159,10 @@ def validate_environment(values: dict[str, str], *, template_mode: bool = False)
|
||||
"YUQEI_WORKER_DATABASE_URL",
|
||||
"MINIO_ROOT_USER",
|
||||
"MINIO_ROOT_PASSWORD",
|
||||
"OBJECT_STORAGE_ENGINE",
|
||||
"OBJECT_STORAGE_ENGINE_VERSION",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT_VERSION",
|
||||
"CONTRACT_API_WEB_PROXY_SHARED_SECRET",
|
||||
"CONTRACT_API_IDENTITY_RATE_LIMIT_SECRET",
|
||||
"CONTRACT_API_ANTI_ABUSE_HASH_SECRET",
|
||||
@@ -195,6 +199,10 @@ def validate_environment(values: dict[str, str], *, template_mode: bool = False)
|
||||
"AI_PLATFORM_ANTI_ABUSE_MODE": "enforce",
|
||||
"YUQEI_WORKER_ENVIRONMENT": "production",
|
||||
"YUQEI_WORKER_DATABASE_AUTO_CREATE": "false",
|
||||
"OBJECT_STORAGE_ENGINE": "rustfs",
|
||||
"OBJECT_STORAGE_ENGINE_VERSION": "1.0.0",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT": "rustfs-rc",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT_VERSION": "0.1.36",
|
||||
}
|
||||
invalid_exact = [
|
||||
f"{key}={values.get(key, '<missing>')}"
|
||||
@@ -325,6 +333,12 @@ def validate_environment(values: dict[str, str], *, template_mode: bool = False)
|
||||
|
||||
volume_keys = ("POSTGRES_VOLUME_NAME", "MINIO_VOLUME_NAME", "CONTRACT_FILES_VOLUME_NAME")
|
||||
invalid_volumes = [key for key in volume_keys if "smoke" in values.get(key, "").lower() or not values.get(key)]
|
||||
object_storage_volume = values.get("MINIO_VOLUME_NAME", "").lower()
|
||||
if (
|
||||
"MINIO_VOLUME_NAME" not in invalid_volumes
|
||||
and ("rustfs" not in object_storage_volume or "minio" in object_storage_volume)
|
||||
):
|
||||
invalid_volumes.append("MINIO_VOLUME_NAME")
|
||||
checks.append(_result("production_volumes", not invalid_volumes, "Stable production volume names are configured." if not invalid_volumes else f"Invalid production volume names: {', '.join(invalid_volumes)}."))
|
||||
|
||||
backup_value = values.get("BACKUP_HOST_DIR", "")
|
||||
|
||||
@@ -33,6 +33,7 @@ MAX_ENV_BYTES = 1024 * 1024
|
||||
MAX_CONFIG_BYTES = 8 * 1024 * 1024
|
||||
ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
|
||||
VOLUME_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||
IMAGE_REFERENCE_RE = re.compile(
|
||||
r"^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$"
|
||||
)
|
||||
@@ -49,6 +50,17 @@ APPLICATION_SERVICE_IMAGE_ENV: Mapping[str, str] = {
|
||||
"contract-web": "CONTRACT_WEB_IMAGE",
|
||||
}
|
||||
|
||||
INFRASTRUCTURE_SERVICE_IMAGE_ENV: Mapping[str, str] = {
|
||||
"postgres": "POSTGRES_IMAGE",
|
||||
"postgres-init-databases": "POSTGRES_CLIENT_IMAGE",
|
||||
"redis": "REDIS_IMAGE",
|
||||
"ai-redis": "REDIS_IMAGE",
|
||||
"minio": "MINIO_IMAGE",
|
||||
"edge": "NGINX_IMAGE",
|
||||
"contract-files-backup": "BACKUP_UTILITY_IMAGE",
|
||||
"minio-backup": "MINIO_MC_IMAGE",
|
||||
}
|
||||
|
||||
PRODUCTION_SERVICES = (
|
||||
"postgres",
|
||||
"postgres-init-databases",
|
||||
@@ -201,7 +213,24 @@ def parse_strict_env(path: Path) -> tuple[Path, dict[str, str]]:
|
||||
if project_name is None or not PROJECT_RE.fullmatch(project_name):
|
||||
raise ComposeGateError("COMPOSE_PROJECT_NAME is missing or unsafe in the strict env file.")
|
||||
|
||||
for env_key in sorted(set(APPLICATION_SERVICE_IMAGE_ENV.values())):
|
||||
storage_volume = values.get("MINIO_VOLUME_NAME", "")
|
||||
if (
|
||||
not VOLUME_NAME_RE.fullmatch(storage_volume)
|
||||
or "rustfs" not in storage_volume.lower()
|
||||
or "minio" in storage_volume.lower()
|
||||
):
|
||||
raise ComposeGateError(
|
||||
"MINIO_VOLUME_NAME must identify a dedicated RustFS generation and must "
|
||||
"not reuse a historical MinIO volume name."
|
||||
)
|
||||
for key in ("MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"):
|
||||
if not values.get(key, ""):
|
||||
raise ComposeGateError(f"{key} is required for the RustFS runtime contract.")
|
||||
|
||||
image_env_keys = set(APPLICATION_SERVICE_IMAGE_ENV.values()) | set(
|
||||
INFRASTRUCTURE_SERVICE_IMAGE_ENV.values()
|
||||
)
|
||||
for env_key in sorted(image_env_keys):
|
||||
image = values.get(env_key)
|
||||
if image is None or not IMAGE_REFERENCE_RE.fullmatch(image):
|
||||
raise ComposeGateError(
|
||||
@@ -328,6 +357,83 @@ def _volume_targets(value: object) -> set[str]:
|
||||
return targets
|
||||
|
||||
|
||||
def _validate_object_storage_contract(
|
||||
services: Mapping[str, Any],
|
||||
*,
|
||||
volumes: object,
|
||||
env_values: Mapping[str, str],
|
||||
mode: str,
|
||||
profile_case: ProfileCase,
|
||||
) -> None:
|
||||
label = f"{mode}/{profile_case.name}"
|
||||
storage = services.get("minio")
|
||||
if not isinstance(storage, dict):
|
||||
raise ComposeGateError(f"{label}: object-storage service is missing or invalid.")
|
||||
mounts = storage.get("volumes")
|
||||
valid_mount = (
|
||||
isinstance(mounts, list)
|
||||
and len(mounts) == 1
|
||||
and isinstance(mounts[0], dict)
|
||||
and mounts[0].get("type") == "volume"
|
||||
and mounts[0].get("source") == "rustfs-data"
|
||||
and mounts[0].get("target") == "/data"
|
||||
and mounts[0].get("read_only") in (None, False)
|
||||
)
|
||||
volume = volumes.get("rustfs-data") if isinstance(volumes, dict) else None
|
||||
valid_volume = (
|
||||
isinstance(volume, dict)
|
||||
and volume.get("name") == env_values["MINIO_VOLUME_NAME"]
|
||||
)
|
||||
if not valid_mount or not valid_volume:
|
||||
raise ComposeGateError(
|
||||
f"{label}: object storage must use only the dedicated rustfs-data volume."
|
||||
)
|
||||
|
||||
command = storage.get("command")
|
||||
command_text = "\n".join(command) if isinstance(command, list) else str(command or "")
|
||||
if (
|
||||
storage.get("entrypoint") != ["/bin/sh", "-ec"]
|
||||
or "set -eu" not in command_text
|
||||
or "/data/.minio.sys" not in command_text
|
||||
or ".yuqei-rustfs-volume-v1" not in command_text
|
||||
or "exec /entrypoint.sh rustfs" not in command_text
|
||||
):
|
||||
raise ComposeGateError(
|
||||
f"{label}: object-storage volume guard or fixed RustFS startup is missing."
|
||||
)
|
||||
environment = storage.get("environment")
|
||||
required_environment = {
|
||||
"RUSTFS_VOLUMES": "/data",
|
||||
"RUSTFS_ADDRESS": "0.0.0.0:9000",
|
||||
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
|
||||
"RUSTFS_CONSOLE_ENABLE": "true",
|
||||
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
|
||||
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
|
||||
}
|
||||
if not isinstance(environment, dict) or any(
|
||||
str(environment.get(key, "")) != value
|
||||
for key, value in required_environment.items()
|
||||
):
|
||||
raise ComposeGateError(f"{label}: RustFS runtime environment is invalid.")
|
||||
healthcheck = storage.get("healthcheck")
|
||||
health_test = healthcheck.get("test") if isinstance(healthcheck, dict) else None
|
||||
if not isinstance(health_test, list) or not any(
|
||||
isinstance(token, str) and "/health/ready" in token for token in health_test
|
||||
):
|
||||
raise ComposeGateError(f"{label}: RustFS readiness healthcheck is missing.")
|
||||
if (
|
||||
storage.get("user") != "10001:10001"
|
||||
or storage.get("read_only") is not True
|
||||
or not isinstance(storage.get("tmpfs"), list)
|
||||
or not any(str(item).startswith("/tmp:") for item in storage["tmpfs"])
|
||||
or not any(str(item).startswith("/logs:") for item in storage["tmpfs"])
|
||||
or "ALL" not in storage.get("cap_drop", [])
|
||||
or "no-new-privileges:true" not in storage.get("security_opt", [])
|
||||
or storage.get("privileged") not in (None, False)
|
||||
):
|
||||
raise ComposeGateError(f"{label}: RustFS runtime hardening is incomplete.")
|
||||
|
||||
|
||||
def _validate_redis_isolation(
|
||||
services: Mapping[str, Any],
|
||||
*,
|
||||
@@ -478,6 +584,13 @@ def _validate_model(
|
||||
mode=mode,
|
||||
profile_case=profile_case,
|
||||
)
|
||||
_validate_object_storage_contract(
|
||||
services,
|
||||
volumes=payload.get("volumes"),
|
||||
env_values=env_values,
|
||||
mode=mode,
|
||||
profile_case=profile_case,
|
||||
)
|
||||
|
||||
validated_application_services: list[str] = []
|
||||
for service_name, env_key in APPLICATION_SERVICE_IMAGE_ENV.items():
|
||||
@@ -506,6 +619,25 @@ def _validate_model(
|
||||
)
|
||||
validated_application_services.append(service_name)
|
||||
|
||||
for service_name, env_key in INFRASTRUCTURE_SERVICE_IMAGE_ENV.items():
|
||||
if service_name not in services:
|
||||
continue
|
||||
service = services[service_name]
|
||||
if not isinstance(service, dict):
|
||||
raise ComposeGateError(
|
||||
f"{mode}/{profile_case.name}: service {service_name} is not an object."
|
||||
)
|
||||
if service.get("image") != env_values[env_key]:
|
||||
raise ComposeGateError(
|
||||
f"{mode}/{profile_case.name}: service {service_name} image does not "
|
||||
f"exactly match {env_key}."
|
||||
)
|
||||
if service.get("build") is not None:
|
||||
raise ComposeGateError(
|
||||
f"{mode}/{profile_case.name}: service {service_name} retains a "
|
||||
"build definition."
|
||||
)
|
||||
|
||||
if mode == "offline":
|
||||
for service_name in sorted(actual_names):
|
||||
service = services[service_name]
|
||||
|
||||
@@ -345,8 +345,8 @@ MINIO_CONSOLE_HOST_PORT=$MinioConsoleHostPort
|
||||
CONTRACT_API_HOST_PORT=$ContractApiHostPort
|
||||
AI_PLATFORM_API_HOST_PORT=$AiPlatformApiHostPort
|
||||
CONTRACT_WEB_HOST_PORT=$ContractWebHostPort
|
||||
MINIO_IMAGE=minio/minio:RELEASE.2023-06-23T20-26-00Z
|
||||
MINIO_MC_IMAGE=minio/mc:latest
|
||||
MINIO_IMAGE=rustfs/rustfs:1.0.0
|
||||
MINIO_MC_IMAGE=rustfs/rc:0.1.36
|
||||
MINIO_ROOT_USER=yuqei-minio
|
||||
MINIO_ROOT_PASSWORD=yuqei-minio-local-smoke
|
||||
AI_PLATFORM_OBJECT_STORAGE_BUCKET=ai-platform-uploads
|
||||
|
||||
@@ -678,8 +678,8 @@ def synthetic_compose_gate_env_bytes() -> bytes:
|
||||
"POSTGRES_IMAGE": ("postgres", "5"),
|
||||
"POSTGRES_CLIENT_IMAGE": ("postgres-client", "6"),
|
||||
"REDIS_IMAGE": ("redis", "7"),
|
||||
"MINIO_IMAGE": ("minio", "8"),
|
||||
"MINIO_MC_IMAGE": ("minio-mc", "9"),
|
||||
"MINIO_IMAGE": ("rustfs", "8"),
|
||||
"MINIO_MC_IMAGE": ("rustfs-rc", "9"),
|
||||
"NGINX_IMAGE": ("nginx", "a"),
|
||||
"BACKUP_UTILITY_IMAGE": ("backup-utility", "b"),
|
||||
}
|
||||
@@ -706,6 +706,7 @@ def synthetic_compose_gate_env_bytes() -> bytes:
|
||||
"CONTRACT_API_PUBLIC_POLICY_RELEASE_DIGEST": "sha256:" + "e" * 64,
|
||||
"MINIO_ROOT_PASSWORD": "synthetic-not-a-production-secret",
|
||||
"MINIO_ROOT_USER": "synthetic-not-a-production-user",
|
||||
"MINIO_VOLUME_NAME": "yuqei-rc04-structural-rustfs-v1",
|
||||
"PUBLIC_POLICY_CANDIDATE_DIR_HOST": "/synthetic/public-policy/candidate",
|
||||
"PUBLIC_POLICY_LEGAL_APPROVAL_FILE_HOST": (
|
||||
"/synthetic/public-policy/legal-approval.json"
|
||||
|
||||
@@ -86,6 +86,7 @@ POLICY_SOURCE_RELATIVE = "yuqei-infra/release/linux-release-security-policy.v1.j
|
||||
COMPOSE_CONFIG_MAX_BYTES = 8 * 1024 * 1024
|
||||
COMPOSE_ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
COMPOSE_PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
|
||||
COMPOSE_VOLUME_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
|
||||
COMPOSE_IMAGE_REFERENCE_RE = re.compile(
|
||||
r"^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$"
|
||||
)
|
||||
@@ -100,6 +101,16 @@ TRUSTED_APPLICATION_SERVICE_IMAGE_ENV: Mapping[str, str] = {
|
||||
"worker": "AI_WORKER_IMAGE",
|
||||
"contract-web": "CONTRACT_WEB_IMAGE",
|
||||
}
|
||||
TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV: Mapping[str, str] = {
|
||||
"postgres": "POSTGRES_IMAGE",
|
||||
"postgres-init-databases": "POSTGRES_CLIENT_IMAGE",
|
||||
"redis": "REDIS_IMAGE",
|
||||
"ai-redis": "REDIS_IMAGE",
|
||||
"minio": "MINIO_IMAGE",
|
||||
"edge": "NGINX_IMAGE",
|
||||
"contract-files-backup": "BACKUP_UTILITY_IMAGE",
|
||||
"minio-backup": "MINIO_MC_IMAGE",
|
||||
}
|
||||
TRUSTED_PRODUCTION_SERVICES = (
|
||||
"postgres",
|
||||
"postgres-init-databases",
|
||||
@@ -587,7 +598,24 @@ def _strict_synthetic_compose_env(data: bytes) -> dict[str, str]:
|
||||
raise ReleaseValidationError(
|
||||
"synthetic Compose gate environment lacks a safe COMPOSE_PROJECT_NAME"
|
||||
)
|
||||
for env_key in sorted(set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values())):
|
||||
storage_volume = values.get("MINIO_VOLUME_NAME", "")
|
||||
if (
|
||||
not COMPOSE_VOLUME_NAME_RE.fullmatch(storage_volume)
|
||||
or "rustfs" not in storage_volume.lower()
|
||||
or "minio" in storage_volume.lower()
|
||||
):
|
||||
raise ReleaseValidationError(
|
||||
"synthetic Compose gate environment lacks a dedicated RustFS volume name"
|
||||
)
|
||||
for key in ("MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"):
|
||||
if not values.get(key, ""):
|
||||
raise ReleaseValidationError(
|
||||
f"synthetic Compose gate environment lacks {key}"
|
||||
)
|
||||
image_env_keys = set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values()) | set(
|
||||
TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.values()
|
||||
)
|
||||
for env_key in sorted(image_env_keys):
|
||||
image = values.get(env_key)
|
||||
if not isinstance(image, str) or not COMPOSE_IMAGE_REFERENCE_RE.fullmatch(image):
|
||||
raise ReleaseValidationError(
|
||||
@@ -650,6 +678,85 @@ def _required_compose_services(profiles: Sequence[str]) -> set[str]:
|
||||
}
|
||||
|
||||
|
||||
def _validate_trusted_object_storage_contract(
|
||||
services: Mapping[str, Any],
|
||||
*,
|
||||
volumes: object,
|
||||
env_values: Mapping[str, str],
|
||||
mode: str,
|
||||
profile_name: str,
|
||||
) -> None:
|
||||
label = f"{mode}/{profile_name}"
|
||||
storage = services.get("minio")
|
||||
if not isinstance(storage, dict):
|
||||
raise ReleaseValidationError(
|
||||
f"{label}: object-storage service is missing or invalid"
|
||||
)
|
||||
mounts = storage.get("volumes")
|
||||
valid_mount = (
|
||||
isinstance(mounts, list)
|
||||
and len(mounts) == 1
|
||||
and isinstance(mounts[0], dict)
|
||||
and mounts[0].get("type") == "volume"
|
||||
and mounts[0].get("source") == "rustfs-data"
|
||||
and mounts[0].get("target") == "/data"
|
||||
and mounts[0].get("read_only") in (None, False)
|
||||
)
|
||||
volume = volumes.get("rustfs-data") if isinstance(volumes, dict) else None
|
||||
valid_volume = (
|
||||
isinstance(volume, dict)
|
||||
and volume.get("name") == env_values["MINIO_VOLUME_NAME"]
|
||||
)
|
||||
if not valid_mount or not valid_volume:
|
||||
raise ReleaseValidationError(
|
||||
f"{label}: object storage must use only the dedicated rustfs-data volume"
|
||||
)
|
||||
|
||||
command = storage.get("command")
|
||||
command_text = "\n".join(command) if isinstance(command, list) else str(command or "")
|
||||
if (
|
||||
storage.get("entrypoint") != ["/bin/sh", "-ec"]
|
||||
or "set -eu" not in command_text
|
||||
or "/data/.minio.sys" not in command_text
|
||||
or ".yuqei-rustfs-volume-v1" not in command_text
|
||||
or "exec /entrypoint.sh rustfs" not in command_text
|
||||
):
|
||||
raise ReleaseValidationError(
|
||||
f"{label}: object-storage volume guard or fixed RustFS startup is missing"
|
||||
)
|
||||
environment = storage.get("environment")
|
||||
required_environment = {
|
||||
"RUSTFS_VOLUMES": "/data",
|
||||
"RUSTFS_ADDRESS": "0.0.0.0:9000",
|
||||
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
|
||||
"RUSTFS_CONSOLE_ENABLE": "true",
|
||||
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
|
||||
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
|
||||
}
|
||||
if not isinstance(environment, dict) or any(
|
||||
str(environment.get(key, "")) != value
|
||||
for key, value in required_environment.items()
|
||||
):
|
||||
raise ReleaseValidationError(f"{label}: RustFS runtime environment is invalid")
|
||||
healthcheck = storage.get("healthcheck")
|
||||
health_test = healthcheck.get("test") if isinstance(healthcheck, dict) else None
|
||||
if not isinstance(health_test, list) or not any(
|
||||
isinstance(token, str) and "/health/ready" in token for token in health_test
|
||||
):
|
||||
raise ReleaseValidationError(f"{label}: RustFS readiness healthcheck is missing")
|
||||
if (
|
||||
storage.get("user") != "10001:10001"
|
||||
or storage.get("read_only") is not True
|
||||
or not isinstance(storage.get("tmpfs"), list)
|
||||
or not any(str(item).startswith("/tmp:") for item in storage["tmpfs"])
|
||||
or not any(str(item).startswith("/logs:") for item in storage["tmpfs"])
|
||||
or "ALL" not in storage.get("cap_drop", [])
|
||||
or "no-new-privileges:true" not in storage.get("security_opt", [])
|
||||
or storage.get("privileged") not in (None, False)
|
||||
):
|
||||
raise ReleaseValidationError(f"{label}: RustFS runtime hardening is incomplete")
|
||||
|
||||
|
||||
def _validate_trusted_compose_model(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
@@ -677,6 +784,14 @@ def _validate_trusted_compose_model(
|
||||
f"{mode}/{profile_name}: service {service_name} is not an object"
|
||||
)
|
||||
|
||||
_validate_trusted_object_storage_contract(
|
||||
services,
|
||||
volumes=payload.get("volumes"),
|
||||
env_values=env_values,
|
||||
mode=mode,
|
||||
profile_name=profile_name,
|
||||
)
|
||||
|
||||
validated_application_services: list[str] = []
|
||||
for service_name, env_key in TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.items():
|
||||
if service_name not in services:
|
||||
@@ -697,6 +812,19 @@ def _validate_trusted_compose_model(
|
||||
)
|
||||
validated_application_services.append(service_name)
|
||||
|
||||
for service_name, env_key in TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.items():
|
||||
if service_name not in services:
|
||||
continue
|
||||
service = services[service_name]
|
||||
if service.get("image") != env_values[env_key]:
|
||||
raise ReleaseValidationError(
|
||||
f"{mode}/{profile_name}: service {service_name} image does not exactly match {env_key}"
|
||||
)
|
||||
if service.get("build") is not None:
|
||||
raise ReleaseValidationError(
|
||||
f"{mode}/{profile_name}: service {service_name} retains a build definition"
|
||||
)
|
||||
|
||||
if mode == "offline":
|
||||
for service_name in sorted(actual_names):
|
||||
service = services[service_name]
|
||||
|
||||
@@ -22,6 +22,29 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
IMAGE_REFERENCE_RE = re.compile(r"^[^\s@]+@sha256:[0-9a-f]{64}$")
|
||||
UAT_SERVICE_IMAGE_ENV = {
|
||||
"postgres": "POSTGRES_IMAGE",
|
||||
"postgres-init-databases": "POSTGRES_CLIENT_IMAGE",
|
||||
"redis": "REDIS_IMAGE",
|
||||
"ai-redis": "REDIS_IMAGE",
|
||||
"minio": "MINIO_IMAGE",
|
||||
"contract-api-migrate": "CONTRACT_API_IMAGE",
|
||||
"contract-api": "CONTRACT_API_IMAGE",
|
||||
"contract-review-worker": "CONTRACT_API_IMAGE",
|
||||
"contract-playbook-governance-worker": "CONTRACT_API_IMAGE",
|
||||
"ai-platform-api-migrate": "AI_PLATFORM_API_IMAGE",
|
||||
"ai-platform-prompt-assets-import": "AI_PLATFORM_API_IMAGE",
|
||||
"ai-platform-api": "AI_PLATFORM_API_IMAGE",
|
||||
"worker": "AI_WORKER_IMAGE",
|
||||
"contract-web": "CONTRACT_WEB_IMAGE",
|
||||
"kms-gateway-aliyun": "KMS_GATEWAY_IMAGE",
|
||||
"kms-gateway-tencent": "KMS_GATEWAY_IMAGE",
|
||||
"evidence-witness-aliyun": "EVIDENCE_WITNESS_GATEWAY_IMAGE",
|
||||
"evidence-witness-tencent": "EVIDENCE_WITNESS_GATEWAY_IMAGE",
|
||||
}
|
||||
|
||||
|
||||
POLICY_UPDATE_FIELDS = (
|
||||
"timezone",
|
||||
"business_calendar_enabled",
|
||||
@@ -67,6 +90,16 @@ REQUIRED_VALUES = (
|
||||
"POSTGRES_VOLUME_NAME",
|
||||
"MINIO_VOLUME_NAME",
|
||||
"CONTRACT_FILES_VOLUME_NAME",
|
||||
"MINIO_ROOT_USER",
|
||||
"MINIO_ROOT_PASSWORD",
|
||||
"POSTGRES_IMAGE",
|
||||
"POSTGRES_CLIENT_IMAGE",
|
||||
"REDIS_IMAGE",
|
||||
"MINIO_IMAGE",
|
||||
"CONTRACT_API_IMAGE",
|
||||
"CONTRACT_WEB_IMAGE",
|
||||
"AI_PLATFORM_API_IMAGE",
|
||||
"AI_WORKER_IMAGE",
|
||||
"POSTGRES_BIND_ADDRESS",
|
||||
"REDIS_BIND_ADDRESS",
|
||||
"MINIO_API_BIND_ADDRESS",
|
||||
@@ -164,9 +197,20 @@ def preflight(env: dict[str, str], *, admin_a: str, admin_b: str, static_only: b
|
||||
placeholders = sorted(key for key, value in env.items() if "CHANGE_ME" in value or "example.com" in value)
|
||||
check(checks, "no_placeholders", not placeholders, "no placeholders found" if not placeholders else "placeholder keys: " + ", ".join(placeholders))
|
||||
|
||||
image_pattern = re.compile(r"^.+@sha256:[0-9a-fA-F]{64}$")
|
||||
invalid_images = [key for key in ("KMS_GATEWAY_IMAGE", "EVIDENCE_WITNESS_GATEWAY_IMAGE") if not image_pattern.match(env.get(key, ""))]
|
||||
check(checks, "immutable_images", not invalid_images, "both images use immutable SHA-256 digests" if not invalid_images else "mutable or invalid image keys: " + ", ".join(invalid_images))
|
||||
image_keys = (
|
||||
"POSTGRES_IMAGE",
|
||||
"POSTGRES_CLIENT_IMAGE",
|
||||
"REDIS_IMAGE",
|
||||
"MINIO_IMAGE",
|
||||
"CONTRACT_API_IMAGE",
|
||||
"CONTRACT_WEB_IMAGE",
|
||||
"AI_PLATFORM_API_IMAGE",
|
||||
"AI_WORKER_IMAGE",
|
||||
"KMS_GATEWAY_IMAGE",
|
||||
"EVIDENCE_WITNESS_GATEWAY_IMAGE",
|
||||
)
|
||||
invalid_images = [key for key in image_keys if not IMAGE_REFERENCE_RE.fullmatch(env.get(key, ""))]
|
||||
check(checks, "immutable_images", not invalid_images, "all UAT images use immutable SHA-256 digests" if not invalid_images else "mutable or invalid image keys: " + ", ".join(invalid_images))
|
||||
|
||||
project = env.get("COMPOSE_PROJECT_NAME", "").lower()
|
||||
isolated_project = "uat" in project and not any(marker in project for marker in ("prod", "production"))
|
||||
@@ -174,6 +218,12 @@ def preflight(env: dict[str, str], *, admin_a: str, admin_b: str, static_only: b
|
||||
|
||||
volume_keys = ("POSTGRES_VOLUME_NAME", "MINIO_VOLUME_NAME", "CONTRACT_FILES_VOLUME_NAME")
|
||||
bad_volumes = [key for key in volume_keys if "uat" not in env.get(key, "").lower()]
|
||||
object_storage_volume = env.get("MINIO_VOLUME_NAME", "").lower()
|
||||
if (
|
||||
"MINIO_VOLUME_NAME" not in bad_volumes
|
||||
and ("rustfs" not in object_storage_volume or "minio" in object_storage_volume)
|
||||
):
|
||||
bad_volumes.append("MINIO_VOLUME_NAME")
|
||||
check(checks, "isolated_volumes", not bad_volumes, "all persistent volumes are UAT-scoped" if not bad_volumes else "non-UAT volume keys: " + ", ".join(bad_volumes))
|
||||
|
||||
default_ports = {"5432", "6379", "9000", "9001", "8100", "8101", "3000"}
|
||||
@@ -300,6 +350,79 @@ def run_command(command: list[str], *, cwd: Path, timeout: int = 600) -> str:
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def validate_rendered_compose(raw_json: str, env: dict[str, str]) -> None:
|
||||
try:
|
||||
payload = json.loads(raw_json)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise UatFailure("Compose did not emit valid JSON.") from exc
|
||||
services = payload.get("services") if isinstance(payload, dict) else None
|
||||
volumes = payload.get("volumes") if isinstance(payload, dict) else None
|
||||
if not isinstance(services, dict) or not isinstance(volumes, dict):
|
||||
raise UatFailure("Rendered Compose model is incomplete.")
|
||||
for service_name, env_key in UAT_SERVICE_IMAGE_ENV.items():
|
||||
service = services.get(service_name)
|
||||
if not isinstance(service, dict):
|
||||
raise UatFailure(f"Rendered Compose service is missing: {service_name}.")
|
||||
expected = env.get(env_key, "")
|
||||
if not IMAGE_REFERENCE_RE.fullmatch(expected) or service.get("image") != expected:
|
||||
raise UatFailure(
|
||||
f"Rendered Compose service {service_name} is not bound to {env_key}."
|
||||
)
|
||||
if service.get("build") is not None:
|
||||
raise UatFailure(
|
||||
f"Rendered Compose service {service_name} retains a build definition."
|
||||
)
|
||||
|
||||
storage = services["minio"]
|
||||
mounts = storage.get("volumes")
|
||||
valid_mount = (
|
||||
isinstance(mounts, list)
|
||||
and len(mounts) == 1
|
||||
and isinstance(mounts[0], dict)
|
||||
and mounts[0].get("type") == "volume"
|
||||
and mounts[0].get("source") == "rustfs-data"
|
||||
and mounts[0].get("target") == "/data"
|
||||
)
|
||||
volume = volumes.get("rustfs-data")
|
||||
valid_volume = (
|
||||
isinstance(volume, dict)
|
||||
and volume.get("name") == env.get("MINIO_VOLUME_NAME")
|
||||
and "rustfs" in str(volume.get("name", "")).lower()
|
||||
and "minio" not in str(volume.get("name", "")).lower()
|
||||
)
|
||||
if not valid_mount or not valid_volume:
|
||||
raise UatFailure("Rendered Compose does not use the isolated RustFS volume contract.")
|
||||
environment = storage.get("environment")
|
||||
if (
|
||||
not isinstance(environment, dict)
|
||||
or environment.get("RUSTFS_ACCESS_KEY") != env.get("MINIO_ROOT_USER")
|
||||
or environment.get("RUSTFS_SECRET_KEY") != env.get("MINIO_ROOT_PASSWORD")
|
||||
or environment.get("RUSTFS_VOLUMES") != "/data"
|
||||
):
|
||||
raise UatFailure("Rendered Compose RustFS credentials or data path are not bound.")
|
||||
command = storage.get("command")
|
||||
command_text = "\n".join(command) if isinstance(command, list) else str(command or "")
|
||||
healthcheck = storage.get("healthcheck")
|
||||
health_test = healthcheck.get("test") if isinstance(healthcheck, dict) else None
|
||||
if (
|
||||
storage.get("entrypoint") != ["/bin/sh", "-ec"]
|
||||
or "set -eu" not in command_text
|
||||
or "/data/.minio.sys" not in command_text
|
||||
or ".yuqei-rustfs-volume-v1" not in command_text
|
||||
or "exec /entrypoint.sh rustfs" not in command_text
|
||||
or not isinstance(health_test, list)
|
||||
or not any(
|
||||
isinstance(token, str) and "/health/ready" in token
|
||||
for token in health_test
|
||||
)
|
||||
or storage.get("user") != "10001:10001"
|
||||
or storage.get("read_only") is not True
|
||||
or "ALL" not in storage.get("cap_drop", [])
|
||||
or "no-new-privileges:true" not in storage.get("security_opt", [])
|
||||
):
|
||||
raise UatFailure("Rendered Compose RustFS startup or hardening contract is invalid.")
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise UatFailure(message)
|
||||
@@ -353,7 +476,8 @@ def execute_uat(args: argparse.Namespace, env: dict[str, str], report: dict[str,
|
||||
]
|
||||
steps: list[dict[str, Any]] = report["steps"]
|
||||
|
||||
run_command(compose + ["config", "--quiet"], cwd=infra_root)
|
||||
rendered = run_command(compose + ["config", "--format", "json"], cwd=infra_root)
|
||||
validate_rendered_compose(rendered, env)
|
||||
steps.append({"name": "compose_config", "status": "passed", "at": now_iso()})
|
||||
run_command(compose + ["up", "-d", "--wait", "--wait-timeout", str(args.startup_timeout)], cwd=infra_root, timeout=args.startup_timeout + 120)
|
||||
steps.append({"name": "isolated_stack_started", "status": "passed", "at": now_iso()})
|
||||
|
||||
@@ -5,6 +5,7 @@ import hashlib
|
||||
import importlib.util
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
@@ -109,6 +110,7 @@ def _case(
|
||||
reuse_root_as_approval_key: bool = False,
|
||||
restore_bindings_match: bool = True,
|
||||
empty_image_map: str | None = None,
|
||||
backup_schema: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
release_id = "release-first-1"
|
||||
environment_id = "production-cn-1"
|
||||
@@ -192,10 +194,27 @@ def _case(
|
||||
strict=True,
|
||||
)
|
||||
}
|
||||
backup_application_images = dict(application_images)
|
||||
backup_infrastructure_images = dict(infrastructure_images)
|
||||
if empty_image_map == "application":
|
||||
application_images = {}
|
||||
elif empty_image_map == "infrastructure":
|
||||
infrastructure_images = {}
|
||||
all_images = {**application_images, **infrastructure_images}
|
||||
offline_image_ids = {
|
||||
key: "sha256:" + hashlib.sha256(f"offline:{key}".encode("utf-8")).hexdigest()
|
||||
for key in (*planner.APPLICATION_IMAGE_KEYS, *planner.INFRASTRUCTURE_IMAGE_KEYS)
|
||||
}
|
||||
docker_import_plan = tuple(
|
||||
{
|
||||
"environment_key": key,
|
||||
"expected_loaded_image_id": offline_image_ids[key],
|
||||
"offline_compose_override": offline_image_ids[key],
|
||||
"registry_reference_for_online_release": record["reference"],
|
||||
"expected_oci_manifest_digest": record["digest"],
|
||||
}
|
||||
for key, record in all_images.items()
|
||||
)
|
||||
identity = _embedded(
|
||||
{
|
||||
"schema_version": planner.IMAGE_IDENTITY_SCHEMA,
|
||||
@@ -299,6 +318,7 @@ def _case(
|
||||
trusted_build_receipt_sha256=trusted_build_receipt_sha256,
|
||||
application_images=application_images,
|
||||
infrastructure_images=infrastructure_images,
|
||||
docker_import_plan=docker_import_plan,
|
||||
)
|
||||
|
||||
def package_validator(
|
||||
@@ -356,9 +376,85 @@ def _case(
|
||||
}
|
||||
for name in database_names
|
||||
}
|
||||
backup = _embedded(
|
||||
database_artifact_paths = {
|
||||
"contract": "postgres-contract.dump",
|
||||
"ai": "postgres-ai_platform.dump",
|
||||
"operations": "postgres-operations.dump",
|
||||
}
|
||||
source_artifacts = [
|
||||
{
|
||||
"schema_version": planner.BACKUP_MANIFEST_SCHEMA,
|
||||
"path": database_artifact_paths[name],
|
||||
"size_bytes": record["size_bytes"],
|
||||
"sha256": record["artifact_sha256"],
|
||||
}
|
||||
for name, record in database_records.items()
|
||||
]
|
||||
source_artifacts.extend(
|
||||
[
|
||||
{
|
||||
"path": "contract-files.tar.gz",
|
||||
"size_bytes": 2048,
|
||||
"sha256": "a" * 64,
|
||||
},
|
||||
{
|
||||
"path": "minio/tenant-a/object.bin",
|
||||
"size_bytes": 4096,
|
||||
"sha256": "b" * 64,
|
||||
},
|
||||
]
|
||||
)
|
||||
minio_records = [
|
||||
item for item in source_artifacts if item["path"].startswith("minio/")
|
||||
]
|
||||
minio_content_index = planner.canonical_sha256(minio_records)
|
||||
source_backup = {
|
||||
"schema_version": planner.SOURCE_BACKUP_MANIFEST_SCHEMA,
|
||||
"backup_id": "backup-consistent-1",
|
||||
"created_at": "2026-09-27T00:05:00Z",
|
||||
"release_id": release_id,
|
||||
"compose_project_name": "yuqei-contract",
|
||||
"consistency_mode": "quiesced",
|
||||
"database_names": {
|
||||
"contract": "yuqei_contract",
|
||||
"ai_platform": "yuqei_ai_platform",
|
||||
"operations": "yuqei_operations",
|
||||
},
|
||||
"images": {
|
||||
"contract_api": backup_application_images["CONTRACT_API_IMAGE"][
|
||||
"reference"
|
||||
],
|
||||
"ai_platform_api": backup_application_images["AI_PLATFORM_API_IMAGE"][
|
||||
"reference"
|
||||
],
|
||||
"ai_worker": backup_application_images["AI_WORKER_IMAGE"]["reference"],
|
||||
},
|
||||
"object_storage_mirror": {
|
||||
"engine": "rustfs",
|
||||
"engine_version": "1.0.0",
|
||||
"engine_image": backup_infrastructure_images["MINIO_IMAGE"]["reference"],
|
||||
"client": "rustfs-rc",
|
||||
"client_version": "0.1.36",
|
||||
"client_image": backup_infrastructure_images["MINIO_MC_IMAGE"][
|
||||
"reference"
|
||||
],
|
||||
"engine_runtime_image": backup_infrastructure_images["MINIO_IMAGE"][
|
||||
"reference"
|
||||
],
|
||||
"engine_resolved_image_id": offline_image_ids["MINIO_IMAGE"],
|
||||
"client_resolved_image_id": offline_image_ids["MINIO_MC_IMAGE"],
|
||||
"bucket": "ai-platform-uploads",
|
||||
"path": "minio",
|
||||
"object_count": 1,
|
||||
"total_size_bytes": 4096,
|
||||
"content_index_sha256": minio_content_index,
|
||||
},
|
||||
"artifacts": source_artifacts,
|
||||
}
|
||||
source_backup_path = _write_json(
|
||||
evidence_root / "source-backup-manifest.json", source_backup
|
||||
)
|
||||
backup_payload: dict[str, Any] = {
|
||||
"schema_version": backup_schema or planner.BACKUP_MANIFEST_SCHEMA,
|
||||
"backup_id": "backup-consistent-1",
|
||||
"release_id": release_id,
|
||||
"environment_id": environment_id,
|
||||
@@ -371,15 +467,34 @@ def _case(
|
||||
"file_count": 2,
|
||||
},
|
||||
"minio": {
|
||||
"content_index_sha256": "b" * 64,
|
||||
"object_count": 3,
|
||||
"content_index_sha256": minio_content_index,
|
||||
"object_count": 1,
|
||||
"total_size_bytes": 4096,
|
||||
},
|
||||
"completed_at": "2026-09-27T00:05:00Z",
|
||||
"status": "complete",
|
||||
},
|
||||
"manifest_payload_sha256",
|
||||
)
|
||||
}
|
||||
if backup_payload["schema_version"] == planner.BACKUP_MANIFEST_SCHEMA:
|
||||
backup_payload.update(
|
||||
{
|
||||
"source_backup_schema": planner.SOURCE_BACKUP_MANIFEST_SCHEMA,
|
||||
"source_backup_manifest_sha256": _sha(source_backup_path),
|
||||
"source_backup_manifest": _external(source_backup_path),
|
||||
"application_runtime_images": {
|
||||
key: backup_application_images[key]["reference"]
|
||||
for key in planner.BACKUP_APPLICATION_IMAGE_KEYS
|
||||
},
|
||||
"object_storage_runtime_images": {
|
||||
key: {
|
||||
"candidate_reference": backup_infrastructure_images[key]["reference"],
|
||||
"runtime_reference": backup_infrastructure_images[key]["reference"],
|
||||
"resolved_image_id": offline_image_ids[key],
|
||||
}
|
||||
for key in planner.BACKUP_OBJECT_STORAGE_IMAGE_KEYS
|
||||
},
|
||||
}
|
||||
)
|
||||
backup = _embedded(backup_payload, "manifest_payload_sha256")
|
||||
backup_path = _write_json(evidence_root / "consistent-backup.json", backup)
|
||||
|
||||
release_bindings = {
|
||||
@@ -528,6 +643,8 @@ def _case(
|
||||
"keyring_path": keyring_path,
|
||||
"root_path": root_path,
|
||||
"root_sha256": _sha(root_path),
|
||||
"backup_path": backup_path,
|
||||
"source_backup_path": source_backup_path,
|
||||
}
|
||||
|
||||
|
||||
@@ -595,6 +712,33 @@ def _rewrite_package_schema(case: dict[str, Any], schema: str) -> None:
|
||||
case["validated_package"].package_sha256 = case["package_sha256"]
|
||||
|
||||
|
||||
def _direct_backup_validation(
|
||||
case: dict[str, Any],
|
||||
*,
|
||||
require_formal: bool = True,
|
||||
) -> tuple[dict[str, Any], str, datetime]:
|
||||
package_binding = planner._validate_package(
|
||||
case["package_dir"],
|
||||
case["package_sha256"],
|
||||
repo_root=case["repo_root"],
|
||||
package_validator=case["package_validator"],
|
||||
)
|
||||
components = case["evidence"]["components"]
|
||||
postgres = dict(components["postgres_consistent_backup"])
|
||||
objects = dict(components["object_storage_state"])
|
||||
backup_record = _external(case["backup_path"])
|
||||
postgres["backup_manifest"] = backup_record
|
||||
objects["backup_manifest"] = backup_record
|
||||
return planner._validate_backup_components(
|
||||
postgres,
|
||||
objects,
|
||||
package_binding=package_binding,
|
||||
environment_id=case["evidence"]["target"]["environment_id"],
|
||||
consistency_set_id=case["evidence"]["consistency_set_id"],
|
||||
require_formal_backup_manifest=require_formal,
|
||||
)
|
||||
|
||||
|
||||
def test_valid_first_release_evidence_emits_only_a_non_writing_plan(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -636,6 +780,109 @@ def test_valid_first_release_evidence_emits_only_a_non_writing_plan(
|
||||
)
|
||||
|
||||
|
||||
def test_formal_backup_rejects_drifted_source_artifacts_manifest(tmp_path: Path) -> None:
|
||||
case = _case(tmp_path)
|
||||
case["source_backup_path"].write_bytes(
|
||||
case["source_backup_path"].read_bytes() + b"\n"
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
planner.SnapshotRestorePlanError,
|
||||
match="source artifacts.v2 backup manifest SHA-256 mismatch",
|
||||
):
|
||||
_direct_backup_validation(case)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target_kind", ("formal_v3", "source_artifacts_v2"))
|
||||
def test_formal_backup_rejects_atomic_external_manifest_replacement(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
target_kind: str,
|
||||
) -> None:
|
||||
case = _case(tmp_path)
|
||||
target = (
|
||||
case["backup_path"]
|
||||
if target_kind == "formal_v3"
|
||||
else case["source_backup_path"]
|
||||
)
|
||||
target_absolute = Path(os.path.abspath(target))
|
||||
attacker = target.with_name(f"{target.stem}.replacement.json")
|
||||
attacker.write_bytes(target.read_bytes() + b"\n")
|
||||
real_open = planner.os.open
|
||||
replaced = False
|
||||
|
||||
def replace_after_path_check(path: object, flags: int, *args: object) -> int:
|
||||
nonlocal replaced
|
||||
if not replaced and Path(os.path.abspath(os.fspath(path))) == target_absolute:
|
||||
os.replace(attacker, target)
|
||||
replaced = True
|
||||
return real_open(path, flags, *args)
|
||||
|
||||
monkeypatch.setattr(planner.os, "open", replace_after_path_check)
|
||||
with pytest.raises(
|
||||
planner.SnapshotRestorePlanError,
|
||||
match="path identity changed before reading",
|
||||
):
|
||||
_direct_backup_validation(case)
|
||||
assert replaced is True
|
||||
|
||||
|
||||
def test_formal_backup_rejects_application_runtime_identity_outside_candidate(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
case = _case(tmp_path)
|
||||
source = json.loads(case["source_backup_path"].read_text(encoding="utf-8"))
|
||||
forged_runtime = "sha256:" + "9" * 64
|
||||
source["images"]["contract_api"] = forged_runtime
|
||||
_write_json(case["source_backup_path"], source)
|
||||
backup = json.loads(case["backup_path"].read_text(encoding="utf-8"))
|
||||
backup["source_backup_manifest"] = _external(case["source_backup_path"])
|
||||
backup["source_backup_manifest_sha256"] = _sha(case["source_backup_path"])
|
||||
backup["application_runtime_images"]["CONTRACT_API_IMAGE"] = forged_runtime
|
||||
backup = _embedded(
|
||||
{key: value for key, value in backup.items() if key != "manifest_payload_sha256"},
|
||||
"manifest_payload_sha256",
|
||||
)
|
||||
_write_json(case["backup_path"], backup)
|
||||
|
||||
with pytest.raises(
|
||||
planner.SnapshotRestorePlanError,
|
||||
match="CONTRACT_API_IMAGE differs from the release package",
|
||||
):
|
||||
_direct_backup_validation(case)
|
||||
|
||||
|
||||
def test_formal_backup_rejects_object_runtime_identity_outside_candidate(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
case = _case(tmp_path)
|
||||
source = json.loads(case["source_backup_path"].read_text(encoding="utf-8"))
|
||||
forged_reference = f"registry.example.invalid/forged/rustfs@sha256:{'9' * 64}"
|
||||
source["object_storage_mirror"]["engine_image"] = forged_reference
|
||||
source["object_storage_mirror"]["engine_runtime_image"] = forged_reference
|
||||
_write_json(case["source_backup_path"], source)
|
||||
backup = json.loads(case["backup_path"].read_text(encoding="utf-8"))
|
||||
backup["source_backup_manifest"] = _external(case["source_backup_path"])
|
||||
backup["source_backup_manifest_sha256"] = _sha(case["source_backup_path"])
|
||||
backup["object_storage_runtime_images"]["MINIO_IMAGE"].update(
|
||||
{
|
||||
"candidate_reference": forged_reference,
|
||||
"runtime_reference": forged_reference,
|
||||
}
|
||||
)
|
||||
backup = _embedded(
|
||||
{key: value for key, value in backup.items() if key != "manifest_payload_sha256"},
|
||||
"manifest_payload_sha256",
|
||||
)
|
||||
_write_json(case["backup_path"], backup)
|
||||
|
||||
with pytest.raises(
|
||||
planner.SnapshotRestorePlanError,
|
||||
match="MINIO_IMAGE differs from the release package",
|
||||
):
|
||||
_direct_backup_validation(case)
|
||||
|
||||
|
||||
def test_install_readiness_receipt_binds_every_gate_input_without_allowing_restore(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -700,6 +947,24 @@ def test_v1_package_remains_read_only_and_cannot_mint_install_readiness(
|
||||
)
|
||||
|
||||
|
||||
def test_v2_backup_manifest_remains_plan_only_and_cannot_mint_new_evidence(
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
case = _case(tmp_path, backup_schema=planner.BACKUP_MANIFEST_SCHEMA_V2)
|
||||
_provision_test_pins(monkeypatch, case)
|
||||
|
||||
plan = _build_output(case, "plan")
|
||||
assert plan["backup"]["schema_version"] == planner.BACKUP_MANIFEST_SCHEMA_V2
|
||||
assert plan["backup"]["source_backup_manifest_sha256"] is None
|
||||
|
||||
with pytest.raises(
|
||||
planner.SnapshotRestorePlanError,
|
||||
match="legacy v2 backup manifest is read-only",
|
||||
):
|
||||
_build_output(case, "install-readiness-receipt")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("which", ("application", "infrastructure"))
|
||||
def test_official_package_result_must_cover_all_eleven_images(
|
||||
tmp_path: Path,
|
||||
@@ -752,6 +1017,8 @@ def test_package_rejects_wrong_trusted_build_receipt_from_official_validator(
|
||||
@pytest.mark.parametrize(
|
||||
"relative",
|
||||
(
|
||||
"yuqei-infra/backup/run-production-backup.py",
|
||||
"yuqei-infra/backup/verify-production-backup.py",
|
||||
"yuqei-infra/scripts/build-infrastructure-image-bundle.py",
|
||||
"yuqei-infra/scripts/build-third-party-materials.py",
|
||||
"yuqei-infra/release-trust/third-party-materials-baseline.json",
|
||||
@@ -764,6 +1031,8 @@ def test_package_source_closure_binds_dynamic_bundle_builders(
|
||||
) -> None:
|
||||
case = _case(tmp_path)
|
||||
for required in (
|
||||
"yuqei-infra/backup/run-production-backup.py",
|
||||
"yuqei-infra/backup/verify-production-backup.py",
|
||||
"yuqei-infra/scripts/build-infrastructure-image-bundle.py",
|
||||
"yuqei-infra/scripts/build-third-party-materials.py",
|
||||
"yuqei-infra/release-trust/third-party-materials-baseline.json",
|
||||
|
||||
@@ -685,16 +685,110 @@ def _compose_model(
|
||||
service: dict[str, Any] = {
|
||||
"pull_policy": "never" if mode == "offline" else "always"
|
||||
}
|
||||
env_key = builder.TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
|
||||
if service_name == "minio":
|
||||
service.update(
|
||||
{
|
||||
"entrypoint": ["/bin/sh", "-ec"],
|
||||
"command": [
|
||||
"set -eu\n"
|
||||
"test ! -e /data/.minio.sys\n"
|
||||
"marker=/data/.yuqei-rustfs-volume-v1\n"
|
||||
"exec /entrypoint.sh rustfs\n"
|
||||
],
|
||||
"environment": {
|
||||
"RUSTFS_VOLUMES": "/data",
|
||||
"RUSTFS_ADDRESS": "0.0.0.0:9000",
|
||||
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
|
||||
"RUSTFS_CONSOLE_ENABLE": "true",
|
||||
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
|
||||
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
|
||||
},
|
||||
"volumes": [
|
||||
{
|
||||
"type": "volume",
|
||||
"source": "rustfs-data",
|
||||
"target": "/data",
|
||||
}
|
||||
],
|
||||
"healthcheck": {
|
||||
"test": [
|
||||
"CMD-SHELL",
|
||||
"curl -fsS http://127.0.0.1:9000/health/ready",
|
||||
]
|
||||
},
|
||||
"user": "10001:10001",
|
||||
"read_only": True,
|
||||
"tmpfs": [
|
||||
"/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777",
|
||||
"/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001",
|
||||
],
|
||||
"cap_drop": ["ALL"],
|
||||
"security_opt": ["no-new-privileges:true"],
|
||||
}
|
||||
)
|
||||
env_key = (
|
||||
builder.TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
|
||||
or builder.TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name)
|
||||
)
|
||||
if env_key is not None:
|
||||
service["image"] = env_values[env_key]
|
||||
services[service_name] = service
|
||||
return {
|
||||
"name": f"fixture-{mode}-{profile_name}",
|
||||
"services": services,
|
||||
"volumes": {
|
||||
"rustfs-data": {"name": env_values["MINIO_VOLUME_NAME"]},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_trusted_compose_model_rejects_infrastructure_image_drift() -> None:
|
||||
env_values = _synthetic_compose_env_values()
|
||||
model = _compose_model(mode="online", profile_name="default", profiles=())
|
||||
model["services"]["minio"]["image"] = (
|
||||
"synthetic.invalid/yuqei/unreviewed-rustfs@sha256:" + "f" * 64
|
||||
)
|
||||
|
||||
with pytest.raises(common.ReleaseValidationError, match="exactly match MINIO_IMAGE"):
|
||||
builder._validate_trusted_compose_model(
|
||||
model,
|
||||
env_values=env_values,
|
||||
mode="online",
|
||||
profile_name="default",
|
||||
profiles=(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutation", "message"),
|
||||
[
|
||||
("volume", "dedicated rustfs-data volume"),
|
||||
("guard", "volume guard or fixed RustFS startup"),
|
||||
("hardening", "runtime hardening"),
|
||||
],
|
||||
)
|
||||
def test_trusted_compose_model_rejects_rustfs_contract_drift(
|
||||
mutation: str, message: str
|
||||
) -> None:
|
||||
env_values = _synthetic_compose_env_values()
|
||||
model = _compose_model(mode="online", profile_name="default", profiles=())
|
||||
if mutation == "volume":
|
||||
model["volumes"]["rustfs-data"]["name"] = "legacy-production-minio"
|
||||
elif mutation == "guard":
|
||||
model["services"]["minio"]["command"] = ["exec /entrypoint.sh rustfs"]
|
||||
else:
|
||||
model["services"]["minio"]["read_only"] = False
|
||||
|
||||
with pytest.raises(common.ReleaseValidationError, match=message):
|
||||
builder._validate_trusted_compose_model(
|
||||
model,
|
||||
env_values=env_values,
|
||||
mode="online",
|
||||
profile_name="default",
|
||||
profiles=(),
|
||||
)
|
||||
|
||||
|
||||
def _bundle(tmp_path: Path) -> dict[str, Any]:
|
||||
source = _fixture_source(tmp_path)
|
||||
root = tmp_path / "bundle"
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[1] / "scripts" / "run-p1-07-6-isolated-uat.py"
|
||||
SPEC = importlib.util.spec_from_file_location("p1_07_6_uat", SCRIPT)
|
||||
@@ -15,8 +18,18 @@ def valid_env(tmp_path: Path) -> dict[str, str]:
|
||||
env = {
|
||||
"COMPOSE_PROJECT_NAME": "yuqei-p1-07-6-uat",
|
||||
"POSTGRES_VOLUME_NAME": "yuqei-p1-07-6-uat-postgres",
|
||||
"MINIO_VOLUME_NAME": "yuqei-p1-07-6-uat-minio",
|
||||
"MINIO_VOLUME_NAME": "yuqei-p1-07-6-uat-rustfs-v1",
|
||||
"CONTRACT_FILES_VOLUME_NAME": "yuqei-p1-07-6-uat-contract-files",
|
||||
"MINIO_ROOT_USER": "synthetic-uat-access-key",
|
||||
"MINIO_ROOT_PASSWORD": "synthetic-uat-secret-value",
|
||||
"POSTGRES_IMAGE": "registry.local/postgres@sha256:" + "1" * 64,
|
||||
"POSTGRES_CLIENT_IMAGE": "registry.local/postgres-client@sha256:" + "2" * 64,
|
||||
"REDIS_IMAGE": "registry.local/redis@sha256:" + "3" * 64,
|
||||
"MINIO_IMAGE": "registry.local/rustfs@sha256:" + "4" * 64,
|
||||
"CONTRACT_API_IMAGE": "registry.local/contract-api@sha256:" + "5" * 64,
|
||||
"CONTRACT_WEB_IMAGE": "registry.local/contract-web@sha256:" + "8" * 64,
|
||||
"AI_PLATFORM_API_IMAGE": "registry.local/ai-platform-api@sha256:" + "6" * 64,
|
||||
"AI_WORKER_IMAGE": "registry.local/ai-worker@sha256:" + "7" * 64,
|
||||
"KMS_GATEWAY_IMAGE": "registry.local/kms@sha256:" + "a" * 64,
|
||||
"EVIDENCE_WITNESS_GATEWAY_IMAGE": "registry.local/witness@sha256:" + "b" * 64,
|
||||
"UAT_ALIYUN_KMS_REGION": "cn-hangzhou",
|
||||
@@ -79,3 +92,52 @@ def test_preflight_rejects_shared_admin_and_mutable_image(tmp_path: Path) -> Non
|
||||
)
|
||||
failed = {item["name"] for item in checks if item["status"] == "failed"}
|
||||
assert {"immutable_images", "two_platform_administrators"} <= failed
|
||||
|
||||
|
||||
def _rendered_model(env: dict[str, str]) -> dict[str, object]:
|
||||
services = {
|
||||
service: {"image": env[env_key]}
|
||||
for service, env_key in uat.UAT_SERVICE_IMAGE_ENV.items()
|
||||
}
|
||||
services["minio"].update(
|
||||
{
|
||||
"volumes": [
|
||||
{"type": "volume", "source": "rustfs-data", "target": "/data"}
|
||||
],
|
||||
"entrypoint": ["/bin/sh", "-ec"],
|
||||
"command": [
|
||||
"set -eu\n"
|
||||
"test ! -e /data/.minio.sys\n"
|
||||
"marker=/data/.yuqei-rustfs-volume-v1\n"
|
||||
"exec /entrypoint.sh rustfs\n"
|
||||
],
|
||||
"healthcheck": {
|
||||
"test": ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready"]
|
||||
},
|
||||
"environment": {
|
||||
"RUSTFS_ACCESS_KEY": env["MINIO_ROOT_USER"],
|
||||
"RUSTFS_SECRET_KEY": env["MINIO_ROOT_PASSWORD"],
|
||||
"RUSTFS_VOLUMES": "/data",
|
||||
},
|
||||
"user": "10001:10001",
|
||||
"read_only": True,
|
||||
"cap_drop": ["ALL"],
|
||||
"security_opt": ["no-new-privileges:true"],
|
||||
}
|
||||
)
|
||||
return {
|
||||
"services": services,
|
||||
"volumes": {"rustfs-data": {"name": env["MINIO_VOLUME_NAME"]}},
|
||||
}
|
||||
|
||||
|
||||
def test_rendered_uat_compose_is_bound_to_all_images_and_rustfs_volume(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
env = valid_env(tmp_path)
|
||||
model = _rendered_model(env)
|
||||
uat.validate_rendered_compose(json.dumps(model), env)
|
||||
|
||||
model["services"]["minio"]["image"] = "registry.local/other@sha256:" + "f" * 64
|
||||
with pytest.raises(uat.UatFailure, match="MINIO_IMAGE"):
|
||||
uat.validate_rendered_compose(json.dumps(model), env)
|
||||
|
||||
@@ -56,10 +56,20 @@ def _make_fixture(tmp_path: Path) -> GateFixture:
|
||||
"CONTRACT_WEB_IMAGE": _image("contract-web", "2"),
|
||||
"AI_PLATFORM_API_IMAGE": _image("ai-platform-api", "3"),
|
||||
"AI_WORKER_IMAGE": _image("ai-worker", "4"),
|
||||
"POSTGRES_IMAGE": _image("postgres", "5"),
|
||||
"POSTGRES_CLIENT_IMAGE": _image("postgres-client", "6"),
|
||||
"REDIS_IMAGE": _image("redis", "7"),
|
||||
"MINIO_IMAGE": _image("rustfs", "8"),
|
||||
"NGINX_IMAGE": _image("nginx", "9"),
|
||||
"BACKUP_UTILITY_IMAGE": _image("backup-utility", "a"),
|
||||
"MINIO_MC_IMAGE": _image("rustfs-rc", "b"),
|
||||
}
|
||||
env_file = tmp_path / "strict-production.env"
|
||||
env_file.write_text(
|
||||
"COMPOSE_PROJECT_NAME=yuqei-prod\n"
|
||||
"MINIO_VOLUME_NAME=yuqei-prod-rustfs-v1\n"
|
||||
"MINIO_ROOT_USER=synthetic-access-key\n"
|
||||
"MINIO_ROOT_PASSWORD=synthetic-secret-value-for-compose-gate\n"
|
||||
+ "".join(f"{key}={value}\n" for key, value in images.items()),
|
||||
encoding="utf-8",
|
||||
newline="\n",
|
||||
@@ -109,16 +119,67 @@ class FakeComposeRunner:
|
||||
}
|
||||
if service_name in gate.REDIS_SERVICE_NAMES:
|
||||
service["command"] = list(gate.EXPECTED_REDIS_COMMAND)
|
||||
image_env = gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
|
||||
if service_name == "minio":
|
||||
service.update(
|
||||
{
|
||||
"entrypoint": ["/bin/sh", "-ec"],
|
||||
"command": [
|
||||
"set -eu\n"
|
||||
"test ! -e /data/.minio.sys\n"
|
||||
"marker=/data/.yuqei-rustfs-volume-v1\n"
|
||||
"exec /entrypoint.sh rustfs\n"
|
||||
],
|
||||
"environment": {
|
||||
"RUSTFS_VOLUMES": "/data",
|
||||
"RUSTFS_ADDRESS": "0.0.0.0:9000",
|
||||
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
|
||||
"RUSTFS_CONSOLE_ENABLE": "true",
|
||||
"RUSTFS_ACCESS_KEY": "synthetic-access-key",
|
||||
"RUSTFS_SECRET_KEY": "synthetic-secret-value-for-compose-gate",
|
||||
},
|
||||
"volumes": [
|
||||
{
|
||||
"type": "volume",
|
||||
"source": "rustfs-data",
|
||||
"target": "/data",
|
||||
}
|
||||
],
|
||||
"healthcheck": {
|
||||
"test": [
|
||||
"CMD-SHELL",
|
||||
"curl -fsS http://127.0.0.1:9000/health/ready",
|
||||
]
|
||||
},
|
||||
"user": "10001:10001",
|
||||
"read_only": True,
|
||||
"tmpfs": [
|
||||
"/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777",
|
||||
"/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001",
|
||||
],
|
||||
"cap_drop": ["ALL"],
|
||||
"security_opt": ["no-new-privileges:true"],
|
||||
}
|
||||
)
|
||||
image_env = (
|
||||
gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
|
||||
or gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name)
|
||||
)
|
||||
if image_env is not None:
|
||||
service["image"] = self.fixture.images[image_env]
|
||||
service["pull_policy"] = "never" if offline else "always"
|
||||
if service_name in gate.APPLICATION_SERVICE_IMAGE_ENV:
|
||||
service["pull_policy"] = "never" if offline else "always"
|
||||
elif offline:
|
||||
service["pull_policy"] = "never"
|
||||
elif offline:
|
||||
service["pull_policy"] = "never"
|
||||
services[service_name] = service
|
||||
if self.mutate is not None:
|
||||
self.mutate(services, offline, profiles, command)
|
||||
payload: dict[str, Any] = {"name": "yuqei-prod", "services": services}
|
||||
payload: dict[str, Any] = {
|
||||
"name": "yuqei-prod",
|
||||
"services": services,
|
||||
"volumes": {"rustfs-data": {"name": "yuqei-prod-rustfs-v1"}},
|
||||
}
|
||||
if self.mutate_payload is not None:
|
||||
self.mutate_payload(payload)
|
||||
stdout = json.dumps(payload)
|
||||
@@ -232,6 +293,48 @@ def test_application_build_mutation_fails_closed(tmp_path: Path) -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_infrastructure_image_mutation_fails_closed(tmp_path: Path) -> None:
|
||||
fixture = _make_fixture(tmp_path)
|
||||
|
||||
def mutate(services, _offline, _profiles, _command):
|
||||
services["minio"]["image"] = _image("unreviewed-rustfs", "c")
|
||||
|
||||
with pytest.raises(gate.ComposeGateError, match="exactly match MINIO_IMAGE"):
|
||||
gate.verify_production_compose(
|
||||
env_file=fixture.env_file,
|
||||
repo_root=fixture.repo,
|
||||
runner=FakeComposeRunner(fixture, mutate),
|
||||
)
|
||||
|
||||
|
||||
def test_object_storage_volume_generation_mutation_fails_closed(tmp_path: Path) -> None:
|
||||
fixture = _make_fixture(tmp_path)
|
||||
|
||||
def mutate_payload(payload):
|
||||
payload["volumes"]["rustfs-data"]["name"] = "yuqei-v2-production-minio"
|
||||
|
||||
with pytest.raises(gate.ComposeGateError, match="dedicated rustfs-data volume"):
|
||||
gate.verify_production_compose(
|
||||
env_file=fixture.env_file,
|
||||
repo_root=fixture.repo,
|
||||
runner=FakeComposeRunner(fixture, mutate_payload=mutate_payload),
|
||||
)
|
||||
|
||||
|
||||
def test_object_storage_runtime_hardening_mutation_fails_closed(tmp_path: Path) -> None:
|
||||
fixture = _make_fixture(tmp_path)
|
||||
|
||||
def mutate(services, _offline, _profiles, _command):
|
||||
services["minio"]["read_only"] = False
|
||||
|
||||
with pytest.raises(gate.ComposeGateError, match="runtime hardening"):
|
||||
gate.verify_production_compose(
|
||||
env_file=fixture.env_file,
|
||||
repo_root=fixture.repo,
|
||||
runner=FakeComposeRunner(fixture, mutate),
|
||||
)
|
||||
|
||||
|
||||
def test_online_pull_policy_mutation_fails_closed(tmp_path: Path) -> None:
|
||||
fixture = _make_fixture(tmp_path)
|
||||
|
||||
@@ -253,7 +356,6 @@ def test_offline_pull_policy_and_build_are_checked_for_infrastructure(tmp_path:
|
||||
def mutate(services, offline, _profiles, _command):
|
||||
if offline:
|
||||
services["edge"]["pull_policy"] = "always"
|
||||
services["postgres"]["build"] = "."
|
||||
|
||||
with pytest.raises(gate.ComposeGateError, match="service edge pull_policy must be never"):
|
||||
gate.verify_production_compose(
|
||||
@@ -263,6 +365,20 @@ def test_offline_pull_policy_and_build_are_checked_for_infrastructure(tmp_path:
|
||||
)
|
||||
|
||||
|
||||
def test_infrastructure_build_mutation_fails_closed(tmp_path: Path) -> None:
|
||||
fixture = _make_fixture(tmp_path)
|
||||
|
||||
def mutate(services, _offline, _profiles, _command):
|
||||
services["postgres"]["build"] = "."
|
||||
|
||||
with pytest.raises(gate.ComposeGateError, match="service postgres retains a build"):
|
||||
gate.verify_production_compose(
|
||||
env_file=fixture.env_file,
|
||||
repo_root=fixture.repo,
|
||||
runner=FakeComposeRunner(fixture, mutate),
|
||||
)
|
||||
|
||||
|
||||
def test_both_profiles_require_all_sixteen_services(tmp_path: Path) -> None:
|
||||
fixture = _make_fixture(tmp_path)
|
||||
|
||||
|
||||
@@ -162,6 +162,26 @@ def test_ai_and_contract_anti_abuse_use_independent_redis_services() -> None:
|
||||
assert "ai-redis" in services["worker"]["depends_on"]
|
||||
|
||||
|
||||
def test_rustfs_uses_a_distinct_fail_closed_volume_contract() -> None:
|
||||
base = _load_compose(BASE_COMPOSE)
|
||||
model = _production_model()
|
||||
services = base["services"]
|
||||
storage = services["minio"]
|
||||
|
||||
assert "rustfs/rustfs" in storage["image"]
|
||||
assert storage["volumes"] == ["rustfs-data:/data"]
|
||||
assert storage["environment"]["RUSTFS_VOLUMES"] == "/data"
|
||||
assert storage["environment"]["RUSTFS_ACCESS_KEY"].startswith("${MINIO_ROOT_USER")
|
||||
startup_command = "\n".join(storage["command"])
|
||||
assert "set -eu" in startup_command
|
||||
assert "/data/.minio.sys" in startup_command
|
||||
assert ".yuqei-rustfs-volume-v1" in startup_command
|
||||
assert "exec /entrypoint.sh rustfs" in startup_command
|
||||
assert "rustfs-data" in model["volumes"]
|
||||
assert "minio-data" not in model["volumes"]
|
||||
assert "yuqei-v2-production-rustfs-v1" in model["volumes"]["rustfs-data"]["name"]
|
||||
|
||||
|
||||
def test_edge_healthcheck_verifies_hostname_and_pinned_ca() -> None:
|
||||
edge = _production_model()["services"]["edge"]
|
||||
command = " ".join(edge["healthcheck"]["test"])
|
||||
@@ -208,6 +228,28 @@ def test_contract_web_image_is_required_and_must_be_immutable() -> None:
|
||||
assert "CONTRACT_WEB_IMAGE" in checks["immutable_images"].message
|
||||
|
||||
|
||||
def test_object_storage_runtime_contract_is_required_before_backup() -> None:
|
||||
values = preflight.parse_env_file(ENV_EXAMPLE)
|
||||
for key in (
|
||||
"OBJECT_STORAGE_ENGINE",
|
||||
"OBJECT_STORAGE_ENGINE_VERSION",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT_VERSION",
|
||||
):
|
||||
values.pop(key)
|
||||
checks = {check.key: check for check in preflight.validate_environment(values)}
|
||||
assert checks["required_values"].status == "fail"
|
||||
assert "OBJECT_STORAGE_ENGINE" in checks["required_values"].message
|
||||
|
||||
|
||||
def test_production_preflight_rejects_historical_minio_volume_name() -> None:
|
||||
values = preflight.parse_env_file(ENV_EXAMPLE)
|
||||
values["MINIO_VOLUME_NAME"] = "yuqei-v2-production-minio"
|
||||
checks = {check.key: check for check in preflight.validate_environment(values)}
|
||||
assert checks["production_volumes"].status == "fail"
|
||||
assert "MINIO_VOLUME_NAME" in checks["production_volumes"].message
|
||||
|
||||
|
||||
def _write_tls_material(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
|
||||
@@ -5,7 +5,9 @@ import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from datetime import datetime, timezone
|
||||
@@ -36,6 +38,10 @@ def load_script(name: str, relative_path: str) -> ModuleType:
|
||||
preflight = load_script("yuqei_production_preflight", "yuqei-infra/deploy/preflight-production.py")
|
||||
backup_run = load_script("yuqei_backup_run", "yuqei-infra/backup/run-production-backup.py")
|
||||
backup_verify = load_script("yuqei_backup_verify", "yuqei-infra/backup/verify-production-backup.py")
|
||||
snapshot_restore_planner = load_script(
|
||||
"yuqei_snapshot_restore_planner_for_backup",
|
||||
"yuqei-infra/deploy/plan-first-release-snapshot-restore.py",
|
||||
)
|
||||
provider_acceptance = load_script("yuqei_provider_acceptance", "yuqei-infra/scripts/run-ai-provider-acceptance.py")
|
||||
contract_product_golden = load_script(
|
||||
"yuqei_contract_product_golden",
|
||||
@@ -69,8 +75,8 @@ def production_values() -> dict[str, str]:
|
||||
"POSTGRES_IMAGE": f"registry.internal/pgvector@sha256:{'4' * 64}",
|
||||
"POSTGRES_CLIENT_IMAGE": f"registry.internal/postgres@sha256:{'5' * 64}",
|
||||
"REDIS_IMAGE": f"registry.internal/redis@sha256:{'6' * 64}",
|
||||
"MINIO_IMAGE": f"registry.internal/minio@sha256:{'7' * 64}",
|
||||
"MINIO_MC_IMAGE": f"registry.internal/minio-mc@sha256:{'8' * 64}",
|
||||
"MINIO_IMAGE": f"registry.internal/rustfs@sha256:{'7' * 64}",
|
||||
"MINIO_MC_IMAGE": f"registry.internal/rustfs-rc@sha256:{'8' * 64}",
|
||||
"NGINX_IMAGE": f"registry.internal/nginx@sha256:{'9' * 64}",
|
||||
"BACKUP_UTILITY_IMAGE": f"registry.internal/alpine@sha256:{'a' * 64}",
|
||||
"BACKEND_SERVER_NAME": "api.contracts.acme.cn",
|
||||
@@ -107,34 +113,52 @@ def production_values() -> dict[str, str]:
|
||||
return values
|
||||
|
||||
|
||||
def test_minio_backup_invokes_pinned_mc_directly_without_a_shell(tmp_path: Path) -> None:
|
||||
def test_object_storage_backup_injects_rc_config_via_stdin_without_secret_argv(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
partial_dir = tmp_path / ".backup-20260811T120000Z.partial"
|
||||
credentials = tmp_path / "minio.env"
|
||||
values = {
|
||||
"COMPOSE_PROJECT_NAME": "yuqei-v2-production",
|
||||
"AI_PLATFORM_OBJECT_STORAGE_BUCKET": "ai-platform-uploads",
|
||||
"MINIO_MC_IMAGE": f"minio/mc@sha256:{'a' * 64}",
|
||||
"MINIO_ROOT_USER": "backup-access-key",
|
||||
"MINIO_ROOT_PASSWORD": "backup-secret-value",
|
||||
"MINIO_MC_IMAGE": f"rustfs/rc@sha256:{'a' * 64}",
|
||||
}
|
||||
|
||||
command = backup_run.minio_mirror_command(
|
||||
command = backup_run.object_storage_mirror_command(
|
||||
values=values,
|
||||
partial_dir=partial_dir,
|
||||
credential_env_file=credentials,
|
||||
)
|
||||
config = backup_run._rc_config_bytes(values).decode("utf-8")
|
||||
|
||||
assert command[:2] == ["docker", "run"]
|
||||
assert "--entrypoint" not in command
|
||||
assert "/bin/sh" not in command
|
||||
assert command[-6:] == [
|
||||
"--config-dir",
|
||||
"/tmp/.mc",
|
||||
"mirror",
|
||||
"--overwrite",
|
||||
"source/ai-platform-uploads",
|
||||
"/backup/minio",
|
||||
assert command[command.index("--pull") + 1] == "never"
|
||||
assert "--interactive" in command
|
||||
assert command[command.index("--entrypoint") + 1] == "/bin/sh"
|
||||
assert "--env-file" not in command
|
||||
assert "HOME=/tmp/rc-home" in command
|
||||
assert command[-3:] == [
|
||||
"--",
|
||||
"source/ai-platform-uploads/",
|
||||
"/backup/minio/",
|
||||
]
|
||||
assert command[command.index("--env-file") + 1] == str(credentials)
|
||||
assert all("secret" not in token for token in command)
|
||||
read_only_index = command.index("--read-only")
|
||||
assert ["--read-only", "--cap-drop", "ALL"] == command[
|
||||
read_only_index : read_only_index + 3
|
||||
]
|
||||
assert "no-new-privileges:true" in command
|
||||
assert "cat > \"$HOME/.config/rc/config.toml\"" in command[-4]
|
||||
assert "exec rc --no-color --no-progress mirror --overwrite" in command[-4]
|
||||
assert all("backup-secret-value" not in token for token in command)
|
||||
assert all("backup-access-key" not in token for token in command)
|
||||
assert 'name = "source"' in config
|
||||
assert 'endpoint = "http://minio:9000"' in config
|
||||
assert 'access_key = "backup-access-key"' in config
|
||||
assert 'secret_key = "backup-secret-value"' in config
|
||||
|
||||
|
||||
def test_object_storage_backup_rejects_unsafe_bind_source(tmp_path: Path) -> None:
|
||||
with pytest.raises(RuntimeError, match="unsafe character"):
|
||||
backup_run._docker_bind_source(tmp_path / "comma,path")
|
||||
|
||||
|
||||
def test_backup_quiesces_every_long_running_application_writer() -> None:
|
||||
@@ -152,7 +176,7 @@ def test_backup_quiesces_every_long_running_application_writer() -> None:
|
||||
[
|
||||
("contract", "contract_api_alembic_version", "contract_api_alembic_version"),
|
||||
("ai_platform", "ai_platform_alembic_version", "ai_platform_alembic_version"),
|
||||
("operations", "public_table_count", "information_schema.tables"),
|
||||
("operations", "public_schema_fingerprint", "information_schema.columns"),
|
||||
],
|
||||
)
|
||||
def test_restore_check_uses_each_database_owned_schema_marker(
|
||||
@@ -720,10 +744,32 @@ def _complete_backup_fixture(tmp_path: Path) -> tuple[Path, dict[str, object]]:
|
||||
item for item in artifacts if str(item["path"]).startswith("minio/")
|
||||
]
|
||||
manifest: dict[str, object] = {
|
||||
"schema_version": "yuqei-production-backup.v1",
|
||||
"schema_version": "yuqei-production-backup-artifacts.v2",
|
||||
"backup_id": backup_dir.name,
|
||||
"created_at": "2026-07-16T12:00:00+00:00",
|
||||
"minio_mirror": {
|
||||
"release_id": "release-2026-07-16",
|
||||
"compose_project_name": "yuqei-production",
|
||||
"consistency_mode": "quiesced",
|
||||
"database_names": {
|
||||
"contract": "yuqei_contract",
|
||||
"ai_platform": "yuqei_ai",
|
||||
"operations": "yuqei_ops",
|
||||
},
|
||||
"images": {
|
||||
"contract_api": f"registry.internal/contract-api@sha256:{'1' * 64}",
|
||||
"ai_platform_api": f"registry.internal/ai-platform-api@sha256:{'2' * 64}",
|
||||
"ai_worker": f"registry.internal/ai-worker@sha256:{'3' * 64}",
|
||||
},
|
||||
"object_storage_mirror": {
|
||||
"engine": "rustfs",
|
||||
"engine_version": "1.0.0",
|
||||
"engine_image": f"registry.internal/rustfs@sha256:{'7' * 64}",
|
||||
"client": "rustfs-rc",
|
||||
"client_version": "0.1.36",
|
||||
"client_image": f"registry.internal/rustfs-rc@sha256:{'8' * 64}",
|
||||
"engine_runtime_image": f"registry.internal/rustfs@sha256:{'7' * 64}",
|
||||
"engine_resolved_image_id": f"sha256:{'9' * 64}",
|
||||
"client_resolved_image_id": f"sha256:{'a' * 64}",
|
||||
"bucket": "ai-platform-uploads",
|
||||
"path": "minio",
|
||||
"object_count": len(minio_records),
|
||||
@@ -739,6 +785,37 @@ def _complete_backup_fixture(tmp_path: Path) -> tuple[Path, dict[str, object]]:
|
||||
return backup_dir, manifest
|
||||
|
||||
|
||||
def _restore_check_fixture(source: dict[str, object]) -> list[dict[str, str]]:
|
||||
database_names = source["database_names"]
|
||||
assert isinstance(database_names, dict)
|
||||
return [
|
||||
{
|
||||
"label": "contract",
|
||||
"database": database_names["contract"],
|
||||
"restore_database": "yuqei_restore_check_contract_20260927010000",
|
||||
"validation": "contract_api_alembic_version",
|
||||
"validation_value": "20260926_0081",
|
||||
"schema_revision": "20260926_0081",
|
||||
},
|
||||
{
|
||||
"label": "ai_platform",
|
||||
"database": database_names["ai_platform"],
|
||||
"restore_database": "yuqei_restore_check_ai_platform_20260927010000",
|
||||
"validation": "ai_platform_alembic_version",
|
||||
"validation_value": "20260925_0042",
|
||||
"schema_revision": "20260925_0042",
|
||||
},
|
||||
{
|
||||
"label": "operations",
|
||||
"database": database_names["operations"],
|
||||
"restore_database": "yuqei_restore_check_operations_20260927010000",
|
||||
"validation": "public_schema_fingerprint",
|
||||
"validation_value": "a" * 32,
|
||||
"schema_revision": "a" * 32,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None:
|
||||
backup_dir, _ = _complete_backup_fixture(tmp_path)
|
||||
|
||||
@@ -750,6 +827,663 @@ def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> No
|
||||
assert len(verified) == 5
|
||||
|
||||
|
||||
def test_artifacts_v2_adapter_is_consumed_by_first_release_recovery_planner(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
backup_dir, source = _complete_backup_fixture(tmp_path)
|
||||
source, verified = backup_verify.validate_backup(backup_dir)
|
||||
restored = _restore_check_fixture(source)
|
||||
adapted = backup_verify.build_planner_backup_manifest(
|
||||
backup_dir=backup_dir,
|
||||
source_manifest=source,
|
||||
restore_checks=restored,
|
||||
environment_id="isolated-uat-01",
|
||||
consistency_set_id="consistency-backup-20260716T120000Z",
|
||||
)
|
||||
adapted_path = tmp_path / "consistent-backup-manifest.json"
|
||||
adapted_path.write_text(json.dumps(adapted), encoding="utf-8")
|
||||
record = {
|
||||
"path": str(adapted_path.resolve()),
|
||||
"sha256": hashlib.sha256(adapted_path.read_bytes()).hexdigest(),
|
||||
}
|
||||
application_images = {
|
||||
"CONTRACT_API_IMAGE": source["images"]["contract_api"],
|
||||
"AI_PLATFORM_API_IMAGE": source["images"]["ai_platform_api"],
|
||||
"AI_WORKER_IMAGE": source["images"]["ai_worker"],
|
||||
}
|
||||
object_storage = source["object_storage_mirror"]
|
||||
infrastructure_images = {
|
||||
"MINIO_IMAGE": object_storage["engine_image"],
|
||||
"MINIO_MC_IMAGE": object_storage["client_image"],
|
||||
}
|
||||
package_binding = {
|
||||
"release_id": source["release_id"],
|
||||
"compose_project_name": source["compose_project_name"],
|
||||
"application_images": {
|
||||
key: {"reference": reference, "digest": reference.rsplit("@", 1)[1]}
|
||||
for key, reference in application_images.items()
|
||||
},
|
||||
"infrastructure_images": {
|
||||
key: {"reference": reference, "digest": reference.rsplit("@", 1)[1]}
|
||||
for key, reference in infrastructure_images.items()
|
||||
},
|
||||
"offline_image_ids": {
|
||||
"CONTRACT_API_IMAGE": f"sha256:{'4' * 64}",
|
||||
"AI_PLATFORM_API_IMAGE": f"sha256:{'5' * 64}",
|
||||
"AI_WORKER_IMAGE": f"sha256:{'6' * 64}",
|
||||
"MINIO_IMAGE": object_storage["engine_resolved_image_id"],
|
||||
"MINIO_MC_IMAGE": object_storage["client_resolved_image_id"],
|
||||
},
|
||||
}
|
||||
|
||||
validated, digest, _completed = snapshot_restore_planner._validate_backup_components(
|
||||
{
|
||||
"backup_manifest": record,
|
||||
"consistency_mode": "quiesced",
|
||||
"databases": list(snapshot_restore_planner.REQUIRED_DATABASES),
|
||||
},
|
||||
{
|
||||
"backup_manifest": record,
|
||||
"stores": list(snapshot_restore_planner.REQUIRED_STORAGE_TARGETS),
|
||||
},
|
||||
package_binding=package_binding,
|
||||
environment_id="isolated-uat-01",
|
||||
consistency_set_id="consistency-backup-20260716T120000Z",
|
||||
require_formal_backup_manifest=True,
|
||||
)
|
||||
|
||||
assert validated == adapted
|
||||
assert digest == record["sha256"]
|
||||
assert adapted["schema_version"] == backup_verify.PLANNER_BACKUP_MANIFEST_SCHEMA
|
||||
assert adapted["source_backup_manifest_sha256"] == hashlib.sha256(
|
||||
(backup_dir / "manifest.json").read_bytes()
|
||||
).hexdigest()
|
||||
assert adapted["application_runtime_images"] == application_images
|
||||
assert adapted["object_storage_runtime_images"]["MINIO_IMAGE"] == {
|
||||
"candidate_reference": object_storage["engine_image"],
|
||||
"runtime_reference": object_storage["engine_runtime_image"],
|
||||
"resolved_image_id": object_storage["engine_resolved_image_id"],
|
||||
}
|
||||
assert adapted["databases"]["ai"]["schema_revision"] == "20260925_0042"
|
||||
summary = backup_verify.build_restore_check_summary(
|
||||
checked_at=datetime(2026, 9, 27, 1, 0, tzinfo=timezone.utc),
|
||||
source_manifest=source,
|
||||
verified=verified,
|
||||
restored=restored,
|
||||
source_manifest_sha256=hashlib.sha256(
|
||||
(backup_dir / "manifest.json").read_bytes()
|
||||
).hexdigest(),
|
||||
planner_manifest_record={
|
||||
"schema_version": adapted["schema_version"],
|
||||
"path": str(adapted_path.resolve()),
|
||||
"sha256": record["sha256"],
|
||||
"size_bytes": adapted_path.stat().st_size,
|
||||
},
|
||||
)
|
||||
unsigned_summary = {
|
||||
key: value for key, value in summary.items() if key != "summary_payload_sha256"
|
||||
}
|
||||
assert summary["schema_version"] == backup_verify.RESTORE_CHECK_SUMMARY_SCHEMA
|
||||
assert summary["summary_payload_sha256"] == backup_verify.canonical_sha256(
|
||||
unsigned_summary
|
||||
)
|
||||
assert summary["independent_restore_test_claimed"] is False
|
||||
|
||||
|
||||
def test_recovery_adapter_rejects_restore_check_for_another_source_database(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
backup_dir, source = _complete_backup_fixture(tmp_path)
|
||||
source, _verified = backup_verify.validate_backup(backup_dir)
|
||||
restored = _restore_check_fixture(source)
|
||||
restored[0]["database"] = "another_contract_database"
|
||||
|
||||
with pytest.raises(RuntimeError, match="does not bind the source database"):
|
||||
backup_verify.build_planner_backup_manifest(
|
||||
backup_dir=backup_dir,
|
||||
source_manifest=source,
|
||||
restore_checks=restored,
|
||||
environment_id="isolated-uat-01",
|
||||
consistency_set_id="consistency-backup-20260716T120000Z",
|
||||
)
|
||||
|
||||
|
||||
def test_recovery_adapter_rejects_legacy_v1_instead_of_relabeling_it(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
backup_dir, source = _complete_backup_fixture(tmp_path)
|
||||
source["schema_version"] = "yuqei-production-backup.v1"
|
||||
|
||||
with pytest.raises(RuntimeError, match="Only artifacts.v2"):
|
||||
backup_verify.build_planner_backup_manifest(
|
||||
backup_dir=backup_dir,
|
||||
source_manifest=source,
|
||||
restore_checks=[],
|
||||
environment_id="isolated-uat-01",
|
||||
consistency_set_id="consistency-1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "message"),
|
||||
(
|
||||
("engine", "minio", "engine or backup client"),
|
||||
("client", "minio-mc", "engine or backup client"),
|
||||
("engine_image", "rustfs/rustfs:1.0.0", "image identity is not immutable"),
|
||||
("client_image", "rustfs/rc:0.1.36", "image identity is not immutable"),
|
||||
("engine_version", "1.0.1", "version metadata is invalid"),
|
||||
(
|
||||
"engine_runtime_image",
|
||||
f"registry.internal/other@sha256:{'b' * 64}",
|
||||
"running image does not match",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_v2_backup_manifest_rejects_unbound_object_storage_identity(
|
||||
tmp_path: Path,
|
||||
field: str,
|
||||
value: str,
|
||||
message: str,
|
||||
) -> None:
|
||||
backup_dir, manifest = _complete_backup_fixture(tmp_path)
|
||||
manifest["object_storage_mirror"][field] = value
|
||||
(backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
backup_run.verify_manifest(backup_dir)
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
backup_verify.validate_backup(backup_dir)
|
||||
|
||||
|
||||
def test_object_storage_runtime_identity_is_bound_to_running_and_client_images(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
outputs = iter(
|
||||
(
|
||||
b"a" * 64 + b"\n",
|
||||
values["MINIO_IMAGE"].encode("utf-8") + b"\n",
|
||||
b"sha256:" + b"b" * 64 + b"\n",
|
||||
b"sha256:" + b"b" * 64 + b"\n",
|
||||
b"sha256:" + b"c" * 64 + b"\n",
|
||||
)
|
||||
)
|
||||
commands: list[list[str]] = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
commands.append(command)
|
||||
return subprocess.CompletedProcess(command, 0, stdout=next(outputs), stderr=b"")
|
||||
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
identity = backup_run._inspect_object_storage_runtime(
|
||||
env_file=tmp_path / "production.env",
|
||||
values=values,
|
||||
)
|
||||
|
||||
assert identity == {
|
||||
"engine_runtime_image": values["MINIO_IMAGE"],
|
||||
"engine_resolved_image_id": "sha256:" + "b" * 64,
|
||||
"client_runtime_image": values["MINIO_MC_IMAGE"],
|
||||
"client_resolved_image_id": "sha256:" + "c" * 64,
|
||||
}
|
||||
assert commands[1][:3] == ["docker", "inspect", "--format"]
|
||||
assert commands[2][:3] == ["docker", "inspect", "--format"]
|
||||
assert commands[3][:3] == ["docker", "image", "inspect"]
|
||||
assert commands[4][:3] == ["docker", "image", "inspect"]
|
||||
|
||||
|
||||
def test_offline_object_storage_runtime_uses_receipt_bound_config_digests(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
engine_id = "sha256:" + "d" * 64
|
||||
client_id = "sha256:" + "e" * 64
|
||||
outputs = iter(
|
||||
(
|
||||
b"a" * 64 + b"\n",
|
||||
engine_id.encode("ascii") + b"\n",
|
||||
engine_id.encode("ascii") + b"\n",
|
||||
engine_id.encode("ascii") + b"\n",
|
||||
client_id.encode("ascii") + b"\n",
|
||||
)
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
return subprocess.CompletedProcess(command, 0, stdout=next(outputs), stderr=b"")
|
||||
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
identity = backup_run._inspect_object_storage_runtime(
|
||||
env_file=tmp_path / "production.env",
|
||||
values=values,
|
||||
runtime_image_overrides={
|
||||
"MINIO_IMAGE": engine_id,
|
||||
"MINIO_MC_IMAGE": client_id,
|
||||
},
|
||||
)
|
||||
|
||||
assert identity["engine_runtime_image"] == engine_id
|
||||
assert identity["engine_resolved_image_id"] == engine_id
|
||||
assert identity["client_runtime_image"] == client_id
|
||||
assert identity["client_resolved_image_id"] == client_id
|
||||
|
||||
|
||||
def test_offline_config_digest_without_candidate_bound_receipt_fails_closed(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
outputs = iter(
|
||||
(
|
||||
b"a" * 64 + b"\n",
|
||||
("sha256:" + "d" * 64).encode("ascii") + b"\n",
|
||||
)
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
return subprocess.CompletedProcess(command, 0, stdout=next(outputs), stderr=b"")
|
||||
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
with pytest.raises(RuntimeError, match="selected runtime identity"):
|
||||
backup_run._inspect_object_storage_runtime(
|
||||
env_file=tmp_path / "production.env",
|
||||
values=values,
|
||||
)
|
||||
|
||||
|
||||
def test_release_receipt_binds_offline_ids_to_frozen_candidate_environment(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
state = tmp_path / "release-state"
|
||||
receipts = state / "receipts"
|
||||
receipts.mkdir(parents=True)
|
||||
idempotency_key = "install-identity-001"
|
||||
snapshot = (
|
||||
state
|
||||
/ "snapshots"
|
||||
/ hashlib.sha256(idempotency_key.encode("utf-8")).hexdigest()
|
||||
/ "production.env"
|
||||
)
|
||||
snapshot.parent.mkdir(parents=True)
|
||||
snapshot.write_text(
|
||||
"\n".join(
|
||||
f"{key}={values[key]}"
|
||||
for key in ("RELEASE_ID", "COMPOSE_PROJECT_NAME", *backup_run.RUNTIME_IMAGE_KEYS)
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
compose_files = []
|
||||
for relative in (
|
||||
"yuqei-infra/compose/docker-compose.yml",
|
||||
"yuqei-infra/compose/docker-compose.production.yml",
|
||||
"yuqei-infra/compose/docker-compose.production.offline.yml",
|
||||
):
|
||||
target = snapshot.parent.joinpath(*Path(relative).parts)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text("services: {}\n", encoding="utf-8")
|
||||
compose_files.append(target)
|
||||
|
||||
def canonical_bytes(payload):
|
||||
return json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
).encode("utf-8")
|
||||
|
||||
snapshot_manifest = {
|
||||
"schema_version": "yuqei.release-input-snapshot.v2",
|
||||
"plan_sha256": "1" * 64,
|
||||
"tls_material_binding_sha256": "2" * 64,
|
||||
"execution_env_sha256": hashlib.sha256(snapshot.read_bytes()).hexdigest(),
|
||||
"files": [
|
||||
{
|
||||
"path": path.as_posix(),
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"size_bytes": path.stat().st_size,
|
||||
}
|
||||
for path in (snapshot, *compose_files)
|
||||
],
|
||||
}
|
||||
snapshot_manifest["snapshot_payload_sha256"] = hashlib.sha256(
|
||||
canonical_bytes(snapshot_manifest)
|
||||
).hexdigest()
|
||||
snapshot_manifest_path = snapshot.parent / "snapshot-manifest.json"
|
||||
snapshot_manifest_path.write_bytes(canonical_bytes(snapshot_manifest) + b"\n")
|
||||
overrides = {
|
||||
key: "sha256:" + f"{index:x}"[-1] * 64
|
||||
for index, key in enumerate(backup_run.RUNTIME_IMAGE_KEYS, start=1)
|
||||
}
|
||||
overrides["AI_PLATFORM_RELEASE_IMAGE_DIGEST"] = "sha256:" + "f" * 64
|
||||
receipt = {
|
||||
"execution_mode": "builtin_fixed_subprocess_runner",
|
||||
"status": "backend_verified",
|
||||
"result": {"returncode": 0},
|
||||
"target_release_id": values["RELEASE_ID"],
|
||||
"compose_project_name": values["COMPOSE_PROJECT_NAME"],
|
||||
"idempotency_key": idempotency_key,
|
||||
"execution_env_sha256": hashlib.sha256(snapshot.read_bytes()).hexdigest(),
|
||||
"snapshot_payload_sha256": snapshot_manifest["snapshot_payload_sha256"],
|
||||
"environment_overrides": overrides,
|
||||
"post_apply_check": {
|
||||
"services": [{"service": "minio", "image_id": overrides["MINIO_IMAGE"]}],
|
||||
},
|
||||
}
|
||||
receipt_path = receipts / "receipt.json"
|
||||
receipt_path.write_text("{}\n", encoding="utf-8")
|
||||
monkeypatch.setattr(
|
||||
backup_run,
|
||||
"_load_release_package_module",
|
||||
lambda: SimpleNamespace(
|
||||
_read_existing_receipt=lambda _path: receipt,
|
||||
_read_json=lambda path, **_kwargs: json.loads(path.read_text(encoding="utf-8")),
|
||||
canonical_json_bytes=canonical_bytes,
|
||||
canonical_sha256=lambda payload: hashlib.sha256(canonical_bytes(payload)).hexdigest(),
|
||||
),
|
||||
)
|
||||
|
||||
actual_overrides, actual_compose = backup_run._release_runtime_overrides(
|
||||
receipt_path=receipt_path,
|
||||
values=values,
|
||||
)
|
||||
assert actual_overrides == overrides
|
||||
assert actual_compose == tuple(path.resolve() for path in compose_files)
|
||||
|
||||
discovered = backup_run._discover_runtime_identity(
|
||||
state_root=state,
|
||||
values=values,
|
||||
)
|
||||
assert discovered is not None
|
||||
assert discovered[0] == receipt_path
|
||||
second_receipt = receipts / "second.json"
|
||||
second_receipt.write_text("{}\n", encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="exactly one fully validated receipt"):
|
||||
backup_run._discover_runtime_identity(
|
||||
state_root=state,
|
||||
values=values,
|
||||
)
|
||||
second_receipt.unlink()
|
||||
|
||||
values["MINIO_IMAGE"] = f"registry.internal/other@sha256:{'9' * 64}"
|
||||
with pytest.raises(RuntimeError, match="candidate binding differs for MINIO_IMAGE"):
|
||||
backup_run._release_runtime_overrides(
|
||||
receipt_path=receipt_path,
|
||||
values=values,
|
||||
)
|
||||
|
||||
|
||||
def test_offline_writer_restore_uses_frozen_three_layer_compose_and_never_pulls(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
compose_files = (
|
||||
tmp_path / "docker-compose.yml",
|
||||
tmp_path / "docker-compose.production.yml",
|
||||
tmp_path / "docker-compose.production.offline.yml",
|
||||
)
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
calls.append(command)
|
||||
stdout = b"contract-api\n" if "ps" in command else b""
|
||||
return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr=b"")
|
||||
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
attempts = backup_run._restore_stopped_writers(
|
||||
env_file=tmp_path / "production.env",
|
||||
project_name="yuqei-v2-production",
|
||||
stopped_writers=("contract-api",),
|
||||
runtime_env={"MINIO_IMAGE": "sha256:" + "1" * 64},
|
||||
compose_files=compose_files,
|
||||
)
|
||||
|
||||
assert attempts[0]["command_succeeded"] is True
|
||||
assert attempts[0]["running_verified"] is True
|
||||
for command in calls:
|
||||
assert command.count("-f") == 3
|
||||
assert str(compose_files[2]) in command
|
||||
restore_command = calls[0]
|
||||
assert restore_command[restore_command.index("--pull") + 1] == "never"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("backup_id", "created_at", "release_id", "images"))
|
||||
def test_v2_backup_manifest_requires_complete_envelope(
|
||||
tmp_path: Path,
|
||||
field: str,
|
||||
) -> None:
|
||||
backup_dir, manifest = _complete_backup_fixture(tmp_path)
|
||||
manifest.pop(field)
|
||||
(backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with pytest.raises(RuntimeError, match="manifest fields"):
|
||||
backup_run.verify_manifest(backup_dir)
|
||||
with pytest.raises(RuntimeError, match="manifest fields"):
|
||||
backup_verify.validate_backup(backup_dir)
|
||||
|
||||
|
||||
def test_create_backup_verifies_partial_before_publishing_completed_name(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
values.update(
|
||||
{
|
||||
"BACKUP_HOST_DIR": str(tmp_path),
|
||||
"OBJECT_STORAGE_ENGINE": "rustfs",
|
||||
"OBJECT_STORAGE_ENGINE_VERSION": "1.0.0",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT": "rustfs-rc",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT_VERSION": "0.1.36",
|
||||
}
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
return subprocess.CompletedProcess(command, 0, stdout=b"", stderr=b"")
|
||||
|
||||
def fake_run_to_file(_command, target, **_kwargs):
|
||||
target.write_bytes(b"database-dump")
|
||||
|
||||
def reject_partial(path):
|
||||
assert path.name.startswith(".backup-") and path.name.endswith(".partial")
|
||||
raise RuntimeError("synthetic manifest rejection")
|
||||
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
monkeypatch.setattr(backup_run, "_run_to_file", fake_run_to_file)
|
||||
monkeypatch.setattr(backup_run, "_verify_contract_archive", lambda _path: None)
|
||||
monkeypatch.setattr(
|
||||
backup_run,
|
||||
"_inspect_object_storage_runtime",
|
||||
lambda **_kwargs: {
|
||||
"engine_runtime_image": values["MINIO_IMAGE"],
|
||||
"engine_resolved_image_id": f"sha256:{'b' * 64}",
|
||||
"client_runtime_image": values["MINIO_MC_IMAGE"],
|
||||
"client_resolved_image_id": f"sha256:{'c' * 64}",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
backup_run,
|
||||
"_object_storage_mirror_metadata",
|
||||
lambda *_args, **_kwargs: {},
|
||||
)
|
||||
monkeypatch.setattr(backup_run, "verify_manifest", reject_partial)
|
||||
|
||||
with pytest.raises(RuntimeError, match="synthetic manifest rejection"):
|
||||
backup_run.create_backup(
|
||||
env_file=tmp_path / "production.env",
|
||||
values=values,
|
||||
backup_root=tmp_path,
|
||||
quiesce_writers=False,
|
||||
)
|
||||
|
||||
assert not list(tmp_path.glob("backup-*"))
|
||||
partials = list(tmp_path.glob(".backup-*.partial"))
|
||||
assert len(partials) == 1
|
||||
assert (partials[0] / "FAILED").is_file()
|
||||
|
||||
|
||||
def test_writer_restore_failure_quarantines_completed_backup_and_attempts_all(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
values.update(
|
||||
{
|
||||
"BACKUP_HOST_DIR": str(tmp_path),
|
||||
"OBJECT_STORAGE_ENGINE": "rustfs",
|
||||
"OBJECT_STORAGE_ENGINE_VERSION": "1.0.0",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT": "rustfs-rc",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT_VERSION": "0.1.36",
|
||||
}
|
||||
)
|
||||
restored: list[str] = []
|
||||
compose_calls: list[list[str]] = []
|
||||
ps_calls = 0
|
||||
frozen_compose = (
|
||||
tmp_path / "docker-compose.yml",
|
||||
tmp_path / "docker-compose.production.yml",
|
||||
tmp_path / "docker-compose.production.offline.yml",
|
||||
)
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
nonlocal ps_calls
|
||||
if command[:2] == ["docker", "compose"]:
|
||||
compose_calls.append(command)
|
||||
if "ps" in command and "--status" in command:
|
||||
ps_calls += 1
|
||||
stdout = ("\n".join(backup_run.WRITER_SERVICES) + "\n").encode("utf-8")
|
||||
return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr=b"")
|
||||
if "up" in command and "--no-build" in command:
|
||||
service = command[-1]
|
||||
restored.append(service)
|
||||
return subprocess.CompletedProcess(
|
||||
command,
|
||||
19 if service == backup_run.WRITER_SERVICES[0] else 0,
|
||||
stdout=b"",
|
||||
stderr=b"synthetic restore failure" if not restored[:-1] else b"",
|
||||
)
|
||||
return subprocess.CompletedProcess(command, 0, stdout=b"", stderr=b"")
|
||||
|
||||
def fake_run_to_file(_command, target, **_kwargs):
|
||||
compose_calls.append(_command)
|
||||
target.write_bytes(b"database-dump")
|
||||
|
||||
monkeypatch.setattr(
|
||||
backup_run,
|
||||
"_release_runtime_overrides",
|
||||
lambda **_kwargs: (
|
||||
{
|
||||
**{
|
||||
key: "sha256:" + "d" * 64
|
||||
for key in backup_run.RUNTIME_IMAGE_KEYS
|
||||
},
|
||||
"AI_PLATFORM_RELEASE_IMAGE_DIGEST": "sha256:" + "e" * 64,
|
||||
},
|
||||
frozen_compose,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
monkeypatch.setattr(backup_run, "_run_to_file", fake_run_to_file)
|
||||
monkeypatch.setattr(backup_run, "_verify_contract_archive", lambda _path: None)
|
||||
monkeypatch.setattr(
|
||||
backup_run,
|
||||
"_inspect_object_storage_runtime",
|
||||
lambda **_kwargs: {
|
||||
"engine_runtime_image": values["MINIO_IMAGE"],
|
||||
"engine_resolved_image_id": f"sha256:{'b' * 64}",
|
||||
"client_runtime_image": values["MINIO_MC_IMAGE"],
|
||||
"client_resolved_image_id": f"sha256:{'c' * 64}",
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(backup_run, "_object_storage_mirror_metadata", lambda *_a, **_k: {})
|
||||
monkeypatch.setattr(backup_run, "verify_manifest", lambda _path: {})
|
||||
|
||||
with pytest.raises(RuntimeError, match="could not be restored and verified"):
|
||||
backup_run.create_backup(
|
||||
env_file=tmp_path / "production.env",
|
||||
values=values,
|
||||
backup_root=tmp_path,
|
||||
quiesce_writers=True,
|
||||
runtime_receipt=tmp_path / "receipt.json",
|
||||
)
|
||||
|
||||
assert restored == list(backup_run.WRITER_SERVICES)
|
||||
assert ps_calls == 2
|
||||
assert not list(tmp_path.glob("backup-*"))
|
||||
partials = list(tmp_path.glob(".backup-*.partial"))
|
||||
assert len(partials) == 1
|
||||
failure = json.loads((partials[0] / "failure.json").read_text(encoding="utf-8"))
|
||||
backup_manifest = json.loads(
|
||||
(partials[0] / "manifest.json").read_text(encoding="utf-8")
|
||||
)
|
||||
assert failure["status"] == "failed"
|
||||
assert set(backup_manifest["images"].values()) == {f"sha256:{'d' * 64}"}
|
||||
assert failure["writer_restore_attempts"][0]["command_succeeded"] is False
|
||||
assert len(failure["writer_restore_attempts"]) == len(backup_run.WRITER_SERVICES)
|
||||
assert compose_calls
|
||||
for command in compose_calls:
|
||||
assert command.count("-f") == 3
|
||||
assert str(frozen_compose[2]) in command
|
||||
|
||||
|
||||
def test_sigterm_enters_writer_restore_path_and_cannot_publish_success(
|
||||
tmp_path: Path,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
values = production_values()
|
||||
values.update(
|
||||
{
|
||||
"BACKUP_HOST_DIR": str(tmp_path),
|
||||
"OBJECT_STORAGE_ENGINE": "rustfs",
|
||||
"OBJECT_STORAGE_ENGINE_VERSION": "1.0.0",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT": "rustfs-rc",
|
||||
"OBJECT_STORAGE_BACKUP_CLIENT_VERSION": "0.1.36",
|
||||
}
|
||||
)
|
||||
installed_handlers: dict[int, object] = {}
|
||||
restored: list[str] = []
|
||||
|
||||
def fake_signal(signum, handler):
|
||||
if callable(handler):
|
||||
installed_handlers[signum] = handler
|
||||
|
||||
def fake_run(command, **_kwargs):
|
||||
if "ps" in command and "--status" in command:
|
||||
stdout = ("\n".join(backup_run.WRITER_SERVICES) + "\n").encode("utf-8")
|
||||
return subprocess.CompletedProcess(command, 0, stdout=stdout, stderr=b"")
|
||||
if "up" in command and "--no-build" in command:
|
||||
restored.append(command[-1])
|
||||
return subprocess.CompletedProcess(command, 0, stdout=b"", stderr=b"")
|
||||
|
||||
def interrupting_dump(_command, _target, **_kwargs):
|
||||
handler = installed_handlers[signal.SIGTERM]
|
||||
assert callable(handler)
|
||||
handler(signal.SIGTERM, None)
|
||||
|
||||
monkeypatch.setattr(backup_run.signal, "signal", fake_signal)
|
||||
monkeypatch.setattr(backup_run, "_run", fake_run)
|
||||
monkeypatch.setattr(backup_run, "_run_to_file", interrupting_dump)
|
||||
|
||||
with pytest.raises(backup_run.BackupInterrupted, match="SIGTERM"):
|
||||
backup_run.create_backup(
|
||||
env_file=tmp_path / "production.env",
|
||||
values=values,
|
||||
backup_root=tmp_path,
|
||||
quiesce_writers=True,
|
||||
)
|
||||
|
||||
assert restored == list(backup_run.WRITER_SERVICES)
|
||||
assert not list(tmp_path.glob("backup-*"))
|
||||
partials = list(tmp_path.glob(".backup-*.partial"))
|
||||
assert len(partials) == 1
|
||||
failure = json.loads((partials[0] / "failure.json").read_text(encoding="utf-8"))
|
||||
assert failure["signal"] == "SIGTERM"
|
||||
assert all(item["running_verified"] for item in failure["writer_restore_attempts"])
|
||||
|
||||
|
||||
def test_backup_manifest_requires_all_databases_contract_files_and_minio(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -764,7 +1498,8 @@ def test_backup_manifest_requires_all_databases_contract_files_and_minio(
|
||||
backup_verify.validate_backup(backup_dir)
|
||||
|
||||
legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy")
|
||||
legacy_manifest.pop("minio_mirror")
|
||||
legacy_manifest["schema_version"] = "yuqei-production-backup.v1"
|
||||
legacy_manifest.pop("object_storage_mirror")
|
||||
legacy_manifest["artifacts"] = [
|
||||
item
|
||||
for item in legacy_manifest["artifacts"]
|
||||
@@ -790,20 +1525,21 @@ def test_backup_manifest_supports_explicit_empty_minio_and_legacy_content(
|
||||
for item in empty_manifest["artifacts"]
|
||||
if not str(item["path"]).startswith("minio/")
|
||||
]
|
||||
empty_manifest["minio_mirror"] = {
|
||||
empty_manifest["object_storage_mirror"].update({
|
||||
"bucket": "ai-platform-uploads",
|
||||
"path": "minio",
|
||||
"object_count": 0,
|
||||
"total_size_bytes": 0,
|
||||
"content_index_sha256": backup_run._minio_content_index([]),
|
||||
}
|
||||
})
|
||||
(empty_dir / "manifest.json").write_text(
|
||||
json.dumps(empty_manifest), encoding="utf-8"
|
||||
)
|
||||
backup_verify.validate_backup(empty_dir)
|
||||
|
||||
legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy-content")
|
||||
legacy_manifest.pop("minio_mirror")
|
||||
legacy_manifest["schema_version"] = "yuqei-production-backup.v1"
|
||||
legacy_manifest.pop("object_storage_mirror")
|
||||
(legacy_dir / "manifest.json").write_text(
|
||||
json.dumps(legacy_manifest), encoding="utf-8"
|
||||
)
|
||||
|
||||
@@ -1497,7 +1497,10 @@ class InspectionRunner:
|
||||
compose_gate = release_package._load_compose_gate_module()
|
||||
env_file = Path(command[command.index("--env-file") + 1])
|
||||
_target, env_values = compose_gate.parse_strict_env(env_file)
|
||||
for env_key in set(compose_gate.APPLICATION_SERVICE_IMAGE_ENV.values()):
|
||||
runtime_image_env_keys = set(
|
||||
compose_gate.APPLICATION_SERVICE_IMAGE_ENV.values()
|
||||
) | set(compose_gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.values())
|
||||
for env_key in runtime_image_env_keys:
|
||||
if env_key in kwargs["env"]:
|
||||
env_values[env_key] = kwargs["env"][env_key]
|
||||
profiles = {
|
||||
@@ -1524,17 +1527,72 @@ class InspectionRunner:
|
||||
}
|
||||
if service_name in compose_gate.REDIS_SERVICE_NAMES:
|
||||
service["command"] = list(compose_gate.EXPECTED_REDIS_COMMAND)
|
||||
image_env = compose_gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
|
||||
if service_name == "minio":
|
||||
service.update(
|
||||
{
|
||||
"entrypoint": ["/bin/sh", "-ec"],
|
||||
"command": [
|
||||
"set -eu\n"
|
||||
"test ! -e /data/.minio.sys\n"
|
||||
"marker=/data/.yuqei-rustfs-volume-v1\n"
|
||||
"exec /entrypoint.sh rustfs\n"
|
||||
],
|
||||
"environment": {
|
||||
"RUSTFS_VOLUMES": "/data",
|
||||
"RUSTFS_ADDRESS": "0.0.0.0:9000",
|
||||
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
|
||||
"RUSTFS_CONSOLE_ENABLE": "true",
|
||||
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
|
||||
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
|
||||
},
|
||||
"volumes": [
|
||||
{
|
||||
"type": "volume",
|
||||
"source": "rustfs-data",
|
||||
"target": "/data",
|
||||
}
|
||||
],
|
||||
"healthcheck": {
|
||||
"test": [
|
||||
"CMD-SHELL",
|
||||
"curl -fsS http://127.0.0.1:9000/health/ready",
|
||||
]
|
||||
},
|
||||
"user": "10001:10001",
|
||||
"read_only": True,
|
||||
"tmpfs": [
|
||||
"/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777",
|
||||
"/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001",
|
||||
],
|
||||
"cap_drop": ["ALL"],
|
||||
"security_opt": ["no-new-privileges:true"],
|
||||
}
|
||||
)
|
||||
image_env = (
|
||||
compose_gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
|
||||
or compose_gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name)
|
||||
)
|
||||
if image_env is not None:
|
||||
service["image"] = env_values[image_env]
|
||||
service["pull_policy"] = "never" if offline else "always"
|
||||
if service_name in compose_gate.APPLICATION_SERVICE_IMAGE_ENV:
|
||||
service["pull_policy"] = "never" if offline else "always"
|
||||
elif offline:
|
||||
service["pull_policy"] = "never"
|
||||
elif offline:
|
||||
service["pull_policy"] = "never"
|
||||
services[service_name] = service
|
||||
return subprocess.CompletedProcess(
|
||||
command,
|
||||
0,
|
||||
stdout=json.dumps({"name": env_values["COMPOSE_PROJECT_NAME"], "services": services}),
|
||||
stdout=json.dumps(
|
||||
{
|
||||
"name": env_values["COMPOSE_PROJECT_NAME"],
|
||||
"services": services,
|
||||
"volumes": {
|
||||
"rustfs-data": {"name": env_values["MINIO_VOLUME_NAME"]}
|
||||
},
|
||||
}
|
||||
),
|
||||
stderr="",
|
||||
)
|
||||
|
||||
@@ -2936,7 +2994,10 @@ def test_preflight_failure_stops_before_state_or_apply_runner(tmp_path: Path) ->
|
||||
"registry.example.test/yuqei/postgres:latest",
|
||||
)
|
||||
|
||||
with pytest.raises(release_package.ReleasePackageError, match="Production preflight failed"):
|
||||
with pytest.raises(
|
||||
release_package.ReleasePackageError,
|
||||
match="POSTGRES_IMAGE must be an explicit lowercase immutable",
|
||||
):
|
||||
_execute(
|
||||
fixture,
|
||||
operation="install",
|
||||
|
||||
Reference in New Issue
Block a user