From 438b17b1ae6bdd27974d30445f6ef96379a947eb Mon Sep 17 00:00:00 2001 From: yuqei Date: Sun, 27 Sep 2026 13:36:39 +0800 Subject: [PATCH] fix(release): isolate RustFS storage and backups --- yuqei-infra/backup/README.md | 14 +- yuqei-infra/backup/run-production-backup.py | 215 ++++++++++++++---- .../backup/verify-production-backup.py | 71 +++++- yuqei-infra/compose/.env.production.example | 10 +- .../docker-compose.production.offline.yml | 3 + .../compose/docker-compose.production.yml | 13 +- .../docker-compose.uat.evidence-trust.yml | 2 +- yuqei-infra/compose/docker-compose.yml | 59 ++++- .../deploy/verify-production-compose.py | 1 + .../test_production_compose_merge_gate.py | 6 +- .../test_production_compose_preflight.py | 24 ++ .../tests/test_production_release_tools.py | 109 ++++++--- 12 files changed, 433 insertions(+), 94 deletions(-) diff --git a/yuqei-infra/backup/README.md b/yuqei-infra/backup/README.md index 25c35131..e756c452 100644 --- a/yuqei-infra/backup/README.md +++ b/yuqei-infra/backup/README.md @@ -1,6 +1,6 @@ # Production backup and restore verification -Application backup APIs are not infrastructure backups. Production evidence requires PostgreSQL custom dumps, contract-file archives, MinIO object exports, checksums, and a disposable database restore rehearsal. +Application backup APIs are not infrastructure backups. Production evidence requires PostgreSQL custom dumps, contract-file archives, S3 object exports, checksums, and a disposable database restore rehearsal. ## Backup @@ -27,8 +27,10 @@ Each completed directory contains: - three PostgreSQL custom-format dumps; - `contract-files.tar.gz`; -- a MinIO bucket mirror; -- `manifest.json` with release ID, image digests, sizes, and SHA-256 values. +- an object-storage bucket mirror (the `minio/` directory name is retained only for package compatibility); +- a v2 `manifest.json` with the RustFS engine/client versions and immutable image identities, release ID, sizes, and SHA-256 values. + +The backup client is the pinned RustFS `rc` image. Credentials are written to a mode-`0600` temporary `config.toml`, mounted read-only, omitted from command arguments, and removed after the bounded run. The backup container runs as the invoking host UID/GID with a read-only root filesystem, no Linux capabilities, and `no-new-privileges`. Incomplete runs remain as `.backup-*.partial` with a `FAILED` marker and are never treated as completed backups. @@ -51,7 +53,11 @@ python3 yuqei-infra/backup/verify-production-backup.py \ --apply ``` -The script creates only generated `yuqei_restore_check_*` databases and drops them in `finally`. It never restores into the configured production database names. Contract-file and MinIO artifacts are integrity-checked but are not written into production volumes by this command. +The script creates only generated `yuqei_restore_check_*` databases and drops them in `finally`. It never restores into the configured production database names. Contract-file and object-storage artifacts are integrity-checked but are not written into production volumes by this command. The verifier remains read-compatible with legacy v1 MinIO manifests but new backups write only v2 manifests. + +## MinIO-to-RustFS volume boundary + +RustFS uses the new `rustfs-data` Compose volume and the production example names it `yuqei-v2-production-rustfs-v1`. Never point that volume at the historical MinIO volume. Before RustFS starts, `object-storage-volume-preflight` rejects `.minio.sys` and any unowned non-empty volume. Migration must be a logical S3 export/import with the old MinIO volume retained unchanged for rollback; direct block/file-level reuse is prohibited. ## Scheduling diff --git a/yuqei-infra/backup/run-production-backup.py b/yuqei-infra/backup/run-production-backup.py index c423209e..7f0cbbda 100644 --- a/yuqei-infra/backup/run-production-backup.py +++ b/yuqei-infra/backup/run-production-backup.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Create a checksummed PostgreSQL, contract-file, and MinIO production backup.""" +"""Create a checksummed PostgreSQL, contract-file, and object-storage backup.""" from __future__ import annotations @@ -17,7 +17,6 @@ import tempfile from datetime import datetime, timezone from pathlib import Path, PurePosixPath from typing import BinaryIO -from urllib.parse import quote REPO_ROOT = Path(__file__).resolve().parents[2] @@ -146,26 +145,27 @@ def create_backup( ) ) _verify_contract_archive(partial_dir / "contract-files.tar.gz") - minio_env_path = _write_minio_alias_env(values) + rc_config_home = _write_rc_config_home(values) try: _run( - minio_mirror_command( + object_storage_mirror_command( values=values, partial_dir=partial_dir, - credential_env_file=minio_env_path, + rc_config_home=rc_config_home, ) ) finally: - minio_env_path.unlink(missing_ok=True) + shutil.rmtree(rc_config_home.parent) artifacts = _artifact_metadata(partial_dir) - minio_mirror = _minio_mirror_metadata( + object_storage_mirror = _object_storage_mirror_metadata( partial_dir, artifacts=artifacts, bucket=values.get("AI_PLATFORM_OBJECT_STORAGE_BUCKET", "ai-platform-uploads"), + values=values, ) manifest = { - "schema_version": "yuqei-production-backup.v1", + "schema_version": "yuqei-production-backup-artifacts.v2", "backup_id": run_id, "created_at": datetime.now(timezone.utc).isoformat(), "release_id": values.get("RELEASE_ID"), @@ -177,7 +177,7 @@ def create_backup( "ai_platform_api": values.get("AI_PLATFORM_API_IMAGE"), "ai_worker": values.get("AI_WORKER_IMAGE"), }, - "minio_mirror": minio_mirror, + "object_storage_mirror": object_storage_mirror, "artifacts": artifacts, } (partial_dir / "manifest.json").write_text( @@ -231,7 +231,13 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]: missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths) if missing: raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}") - _verify_minio_mirror(backup_dir, payload, minio_records) + schema_version = payload.get("schema_version") + if schema_version == "yuqei-production-backup.v1": + _verify_minio_mirror(backup_dir, payload, minio_records) + elif schema_version == "yuqei-production-backup-artifacts.v2": + _verify_object_storage_mirror(backup_dir, payload, minio_records) + else: + raise RuntimeError("Unsupported backup manifest schema.") _verify_contract_archive(backup_dir / "contract-files.tar.gz") return payload @@ -294,15 +300,16 @@ def _verify_contract_archive(path: Path) -> None: seen_folded.add(folded) -def _minio_mirror_metadata( +def _object_storage_mirror_metadata( root: Path, *, artifacts: list[dict[str, object]], bucket: str, + values: dict[str, str], ) -> dict[str, object]: minio_root = root / "minio" if minio_root.is_symlink() or not minio_root.is_dir(): - raise RuntimeError("MinIO mirror directory is missing or unsafe.") + raise RuntimeError("Object-storage mirror directory is missing or unsafe.") records = [ { "path": str(item["path"]), @@ -313,6 +320,12 @@ def _minio_mirror_metadata( if str(item.get("path", "")).startswith("minio/") ] return { + "engine": values["OBJECT_STORAGE_ENGINE"], + "engine_version": values["OBJECT_STORAGE_ENGINE_VERSION"], + "engine_image": values["MINIO_IMAGE"], + "client": values["OBJECT_STORAGE_BACKUP_CLIENT"], + "client_version": values["OBJECT_STORAGE_BACKUP_CLIENT_VERSION"], + "client_image": values["MINIO_MC_IMAGE"], "bucket": bucket, "path": "minio", "object_count": len(records), @@ -373,52 +386,159 @@ def _verify_minio_mirror( raise RuntimeError("MinIO mirror content index mismatch.") -def minio_mirror_command( +def _verify_object_storage_mirror( + backup_dir: Path, + manifest: dict[str, object], + records: list[dict[str, object]], +) -> None: + mirror_root = backup_dir / "minio" + if mirror_root.is_symlink() or not mirror_root.is_dir(): + raise RuntimeError("Object-storage mirror directory is missing or unsafe.") + actual_files: set[str] = set() + for path in mirror_root.rglob("*"): + relative = path.relative_to(backup_dir).as_posix() + if path.is_symlink(): + raise RuntimeError(f"Object-storage mirror contains a symbolic link: {relative}") + if path.is_file(): + actual_files.add(relative) + elif not path.is_dir(): + raise RuntimeError(f"Object-storage mirror contains a special entry: {relative}") + if actual_files != {str(item["path"]) for item in records}: + raise RuntimeError("Object-storage mirror content differs from the backup manifest.") + + metadata = manifest.get("object_storage_mirror") + if not isinstance(metadata, dict): + raise RuntimeError("Object-storage mirror metadata is invalid.") + expected = { + "engine", + "engine_version", + "engine_image", + "client", + "client_version", + "client_image", + "bucket", + "path", + "object_count", + "total_size_bytes", + "content_index_sha256", + } + if set(metadata) != expected or metadata.get("path") != "minio": + raise RuntimeError("Object-storage mirror metadata fields are invalid.") + if metadata.get("engine") != "rustfs" or metadata.get("client") != "rustfs-rc": + raise RuntimeError("Object-storage engine or backup client is unsupported.") + for key in ("engine_version", "client_version"): + if not isinstance(metadata.get(key), str) or not metadata[key].strip(): + raise RuntimeError("Object-storage version metadata is invalid.") + for key in ("engine_image", "client_image"): + value = metadata.get(key) + if not isinstance(value, str) or not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", value): + raise RuntimeError("Object-storage image identity is not immutable.") + if ( + not isinstance(metadata.get("bucket"), str) + or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"]) + ): + raise RuntimeError("Object-storage bucket metadata is invalid.") + if metadata.get("object_count") != len(records): + raise RuntimeError("Object-storage mirror object count mismatch.") + if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records): + raise RuntimeError("Object-storage mirror total size mismatch.") + if metadata.get("content_index_sha256") != _minio_content_index(records): + raise RuntimeError("Object-storage mirror content index mismatch.") + + +def object_storage_mirror_command( *, values: dict[str, str], partial_dir: Path, - credential_env_file: Path, + rc_config_home: Path, ) -> list[str]: project_name = values["COMPOSE_PROJECT_NAME"] bucket = values.get("AI_PLATFORM_OBJECT_STORAGE_BUCKET", "ai-platform-uploads") if not DOCKER_PROJECT_PATTERN.fullmatch(project_name): raise RuntimeError("Compose project name is unsafe for the Docker network.") if not S3_BUCKET_PATTERN.fullmatch(bucket): - raise RuntimeError("MinIO bucket name is invalid.") + raise RuntimeError("Object-storage bucket name is invalid.") + config_source = _docker_bind_source(rc_config_home) + backup_source = _docker_bind_source(partial_dir) + uid = getattr(os, "getuid", lambda: 10001)() + gid = getattr(os, "getgid", lambda: 10001)() return [ "docker", "run", "--rm", "--network", f"{project_name}_default", - "--env-file", - str(credential_env_file), - "-v", - f"{partial_dir.resolve()}:/backup", + "--user", + f"{uid}:{gid}", + "--read-only", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges:true", + "--tmpfs", + "/tmp:rw,noexec,nosuid,nodev,size=16m", + "--env", + "HOME=/home/rc", + "--mount", + f"type=bind,src={config_source},dst=/home/rc,readonly", + "--mount", + f"type=bind,src={backup_source},dst=/backup", values["MINIO_MC_IMAGE"], - "--config-dir", - "/tmp/.mc", + "--no-color", + "--no-progress", "mirror", "--overwrite", - f"source/{bucket}", - "/backup/minio", + f"source/{bucket}/", + "/backup/minio/", ] -def _write_minio_alias_env(values: dict[str, str]) -> Path: - username = quote(values["MINIO_ROOT_USER"], safe="") - password = quote(values["MINIO_ROOT_PASSWORD"], safe="") - with tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - prefix="yuqei-minio-backup-", - suffix=".env", - delete=False, - ) as stream: - stream.write(f"MC_HOST_source=http://{username}:{password}@minio:9000\n") - path = Path(stream.name) - os.chmod(path, 0o600) - return path +def _docker_bind_source(path: Path) -> str: + value = str(path.resolve()) + if any(character in value for character in (",", "\r", "\n")): + raise RuntimeError("Docker bind source contains an unsafe character.") + return value + + +def _write_rc_config_home(values: dict[str, str]) -> Path: + access_key = values["MINIO_ROOT_USER"] + secret_key = values["MINIO_ROOT_PASSWORD"] + if not access_key or not secret_key or "\x00" in access_key or "\x00" in secret_key: + raise RuntimeError("Object-storage backup credentials are invalid.") + root = Path(tempfile.mkdtemp(prefix="yuqei-rustfs-rc-")) + try: + home = root / "home" + config_dir = home / ".config" / "rc" + config_dir.mkdir(parents=True, mode=0o700) + config = "\n".join( + ( + "schema_version = 1", + "", + "[defaults]", + 'output = "human"', + 'color = "never"', + "progress = false", + "", + "[[aliases]]", + 'name = "source"', + 'endpoint = "http://minio:9000"', + f"access_key = {json.dumps(access_key, ensure_ascii=False)}", + f"secret_key = {json.dumps(secret_key, ensure_ascii=False)}", + 'region = "us-east-1"', + "", + ) + ) + config_path = config_dir / "config.toml" + config_path.write_text(config, encoding="utf-8", newline="\n") + os.chmod(root, 0o700) + os.chmod(home, 0o700) + os.chmod(home / ".config", 0o700) + os.chmod(config_dir, 0o700) + os.chmod(config_path, 0o600) + return home + except BaseException: + shutil.rmtree(root, ignore_errors=True) + raise def _sha256(path: Path) -> str: @@ -479,12 +599,31 @@ def main() -> int: "OPS_POSTGRES_DB", "MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD", + "MINIO_IMAGE", "MINIO_MC_IMAGE", + "OBJECT_STORAGE_ENGINE", + "OBJECT_STORAGE_ENGINE_VERSION", + "OBJECT_STORAGE_BACKUP_CLIENT", + "OBJECT_STORAGE_BACKUP_CLIENT_VERSION", ) missing = [key for key in required if not values.get(key)] if missing: print(f"PRODUCTION_BACKUP_FAIL missing_keys={','.join(missing)}") return 2 + if ( + values["OBJECT_STORAGE_ENGINE"] != "rustfs" + or values["OBJECT_STORAGE_BACKUP_CLIENT"] != "rustfs-rc" + ): + print("PRODUCTION_BACKUP_FAIL unsupported_object_storage_runtime") + return 2 + mutable_images = [ + key + for key in ("MINIO_IMAGE", "MINIO_MC_IMAGE") + if not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", values[key]) + ] + if mutable_images: + print(f"PRODUCTION_BACKUP_FAIL mutable_image_keys={','.join(mutable_images)}") + return 2 backup_root = Path(values["BACKUP_HOST_DIR"]).resolve() separate_mount_required = values.get("BACKUP_REQUIRE_SEPARATE_MOUNT", "true").lower() == "true" plan = { @@ -494,7 +633,7 @@ def main() -> int: "backup_root": str(backup_root), "separate_mount_required": separate_mount_required, "quiesce_writers": args.quiesce_writers, - "stores": ["contract_postgresql", "ai_platform_postgresql", "operations_postgresql", "contract_files", "minio"], + "stores": ["contract_postgresql", "ai_platform_postgresql", "operations_postgresql", "contract_files", "object_storage"], } if not args.apply: print(json.dumps(plan, ensure_ascii=False, indent=2)) diff --git a/yuqei-infra/backup/verify-production-backup.py b/yuqei-infra/backup/verify-production-backup.py index 9d04d1d3..38474410 100644 --- a/yuqei-infra/backup/verify-production-backup.py +++ b/yuqei-infra/backup/verify-production-backup.py @@ -46,7 +46,11 @@ def read_env(path: Path) -> dict[str, str]: def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str, object]]]: manifest = json.loads((backup_dir / "manifest.json").read_text(encoding="utf-8")) - if manifest.get("schema_version") != "yuqei-production-backup.v1": + schema_version = manifest.get("schema_version") + if schema_version not in { + "yuqei-production-backup.v1", + "yuqei-production-backup-artifacts.v2", + }: raise RuntimeError("Unsupported backup manifest schema.") artifacts = manifest.get("artifacts") if not isinstance(artifacts, list) or not artifacts: @@ -79,7 +83,10 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str, missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths) if missing: raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}") - _verify_minio_mirror(backup_dir, manifest, minio_records) + if schema_version == "yuqei-production-backup.v1": + _verify_minio_mirror(backup_dir, manifest, minio_records) + else: + _verify_object_storage_mirror(backup_dir, manifest, minio_records) contract_archive = backup_dir / "contract-files.tar.gz" with tarfile.open(contract_archive, mode="r:gz") as archive: @@ -166,6 +173,66 @@ def _verify_minio_mirror( raise RuntimeError("MinIO mirror content index mismatch.") +def _verify_object_storage_mirror( + backup_dir: Path, + manifest: dict[str, object], + records: list[dict[str, object]], +) -> None: + mirror_root = backup_dir / "minio" + if mirror_root.is_symlink() or not mirror_root.is_dir(): + raise RuntimeError("Object-storage mirror directory is missing or unsafe.") + actual_files: set[str] = set() + for path in mirror_root.rglob("*"): + relative = path.relative_to(backup_dir).as_posix() + if path.is_symlink(): + raise RuntimeError(f"Object-storage mirror contains a symbolic link: {relative}") + if path.is_file(): + actual_files.add(relative) + elif not path.is_dir(): + raise RuntimeError(f"Object-storage mirror contains a special entry: {relative}") + if actual_files != {str(item["path"]) for item in records}: + raise RuntimeError("Object-storage mirror content differs from the backup manifest.") + + metadata = manifest.get("object_storage_mirror") + if not isinstance(metadata, dict): + raise RuntimeError("Object-storage mirror metadata is invalid.") + expected = { + "engine", + "engine_version", + "engine_image", + "client", + "client_version", + "client_image", + "bucket", + "path", + "object_count", + "total_size_bytes", + "content_index_sha256", + } + if set(metadata) != expected or metadata.get("path") != "minio": + raise RuntimeError("Object-storage mirror metadata fields are invalid.") + if metadata.get("engine") != "rustfs" or metadata.get("client") != "rustfs-rc": + raise RuntimeError("Object-storage engine or backup client is unsupported.") + for key in ("engine_version", "client_version"): + if not isinstance(metadata.get(key), str) or not metadata[key].strip(): + raise RuntimeError("Object-storage version metadata is invalid.") + for key in ("engine_image", "client_image"): + value = metadata.get(key) + if not isinstance(value, str) or not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", value): + raise RuntimeError("Object-storage image identity is not immutable.") + if ( + not isinstance(metadata.get("bucket"), str) + or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"]) + ): + raise RuntimeError("Object-storage bucket metadata is invalid.") + if metadata.get("object_count") != len(records): + raise RuntimeError("Object-storage mirror object count mismatch.") + if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records): + raise RuntimeError("Object-storage mirror total size mismatch.") + if metadata.get("content_index_sha256") != _minio_content_index(records): + raise RuntimeError("Object-storage mirror content index mismatch.") + + def restore_database_checks( *, backup_dir: Path, diff --git a/yuqei-infra/compose/.env.production.example b/yuqei-infra/compose/.env.production.example index e1ea808d..8fe02fde 100644 --- a/yuqei-infra/compose/.env.production.example +++ b/yuqei-infra/compose/.env.production.example @@ -13,8 +13,8 @@ AI_WORKER_IMAGE=registry.example.com/yuqei/ai-worker@sha256:CHANGE_ME_64_HEX_DIG POSTGRES_IMAGE=pgvector/pgvector@sha256:CHANGE_ME_64_HEX_DIGEST POSTGRES_CLIENT_IMAGE=postgres@sha256:CHANGE_ME_64_HEX_DIGEST REDIS_IMAGE=redis@sha256:CHANGE_ME_64_HEX_DIGEST -MINIO_IMAGE=minio/minio@sha256:CHANGE_ME_64_HEX_DIGEST -MINIO_MC_IMAGE=minio/mc@sha256:CHANGE_ME_64_HEX_DIGEST +MINIO_IMAGE=rustfs/rustfs@sha256:CHANGE_ME_64_HEX_DIGEST +MINIO_MC_IMAGE=registry.example.invalid/yuqei/rustfs-rc@sha256:CHANGE_ME_64_HEX_DIGEST NGINX_IMAGE=nginx@sha256:CHANGE_ME_64_HEX_DIGEST BACKUP_UTILITY_IMAGE=alpine@sha256:CHANGE_ME_64_HEX_DIGEST @@ -38,7 +38,7 @@ AI_PLATFORM_API_BIND_ADDRESS=127.0.0.1 CONTRACT_WEB_BIND_ADDRESS=127.0.0.1 POSTGRES_VOLUME_NAME=yuqei-v2-production-postgres -MINIO_VOLUME_NAME=yuqei-v2-production-minio +MINIO_VOLUME_NAME=yuqei-v2-production-rustfs-v1 CONTRACT_FILES_VOLUME_NAME=yuqei-v2-production-contract-files BACKUP_HOST_DIR=/mnt/yuqei-v2-backups BACKUP_REQUIRE_SEPARATE_MOUNT=true @@ -83,6 +83,10 @@ AI_PLATFORM_API_HOST_PORT=8101 CONTRACT_WEB_HOST_PORT=3000 MINIO_ROOT_USER=CHANGE_ME MINIO_ROOT_PASSWORD=CHANGE_ME +OBJECT_STORAGE_ENGINE=rustfs +OBJECT_STORAGE_ENGINE_VERSION=1.0.0 +OBJECT_STORAGE_BACKUP_CLIENT=rustfs-rc +OBJECT_STORAGE_BACKUP_CLIENT_VERSION=0.1.36 AI_PLATFORM_OBJECT_STORAGE_BUCKET=ai-platform-uploads CONTRACT_API_ENVIRONMENT=production diff --git a/yuqei-infra/compose/docker-compose.production.offline.yml b/yuqei-infra/compose/docker-compose.production.offline.yml index 94b11859..a16cfcf6 100644 --- a/yuqei-infra/compose/docker-compose.production.offline.yml +++ b/yuqei-infra/compose/docker-compose.production.offline.yml @@ -1,4 +1,7 @@ services: + object-storage-volume-preflight: + pull_policy: never + postgres: pull_policy: never diff --git a/yuqei-infra/compose/docker-compose.production.yml b/yuqei-infra/compose/docker-compose.production.yml index 463b4da9..fff78793 100644 --- a/yuqei-infra/compose/docker-compose.production.yml +++ b/yuqei-infra/compose/docker-compose.production.yml @@ -18,7 +18,7 @@ services: minio: image: ${MINIO_IMAGE:?MINIO_IMAGE must reference an immutable production image} volumes: - - minio-data:/data + - rustfs-data:/data contract-api-migrate: image: ${CONTRACT_API_IMAGE:?CONTRACT_API_IMAGE must reference an immutable production image} @@ -152,10 +152,7 @@ services: entrypoint: ["/bin/sh", "-c"] command: ["exit 0"] environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required} - AI_PLATFORM_OBJECT_STORAGE_BUCKET: ${AI_PLATFORM_OBJECT_STORAGE_BUCKET:-ai-platform-uploads} - MC_CONFIG_DIR: /tmp/.mc + HOME: /tmp/home volumes: - ${BACKUP_HOST_DIR:?BACKUP_HOST_DIR is required}:/backup read_only: true @@ -170,8 +167,10 @@ services: volumes: postgres-data: name: ${POSTGRES_VOLUME_NAME:-yuqei-v2-production-postgres} - minio-data: - name: ${MINIO_VOLUME_NAME:-yuqei-v2-production-minio} + rustfs-data: + # This default is intentionally different from the historical MinIO + # volume. The preflight service also fails closed on .minio.sys. + name: ${MINIO_VOLUME_NAME:-yuqei-v2-production-rustfs-v1} contract-files: name: ${CONTRACT_FILES_VOLUME_NAME:-yuqei-v2-production-contract-files} diff --git a/yuqei-infra/compose/docker-compose.uat.evidence-trust.yml b/yuqei-infra/compose/docker-compose.uat.evidence-trust.yml index 01635337..559d790e 100644 --- a/yuqei-infra/compose/docker-compose.uat.evidence-trust.yml +++ b/yuqei-infra/compose/docker-compose.uat.evidence-trust.yml @@ -176,7 +176,7 @@ services: volumes: postgres-data: name: ${POSTGRES_VOLUME_NAME:?POSTGRES_VOLUME_NAME must be UAT-scoped} - minio-data: + rustfs-data: name: ${MINIO_VOLUME_NAME:?MINIO_VOLUME_NAME must be UAT-scoped} contract-files: name: ${CONTRACT_FILES_VOLUME_NAME:?CONTRACT_FILES_VOLUME_NAME must be UAT-scoped} diff --git a/yuqei-infra/compose/docker-compose.yml b/yuqei-infra/compose/docker-compose.yml index 097697da..46e18c31 100644 --- a/yuqei-infra/compose/docker-compose.yml +++ b/yuqei-infra/compose/docker-compose.yml @@ -62,24 +62,69 @@ services: timeout: 5s retries: 5 + object-storage-volume-preflight: + # Keep the legacy image variable for package compatibility, but never let + # RustFS open a MinIO-formatted or otherwise unknown data volume. + image: ${MINIO_IMAGE:-rustfs/rustfs:1.0.0} + entrypoint: ["/bin/sh", "-ec"] + command: + - | + marker=/data/.yuqei-rustfs-volume-v1 + if [ -e /data/.minio.sys ] || [ -L /data/.minio.sys ]; then + echo "Refusing to mount a MinIO-formatted volume in RustFS." >&2 + exit 42 + fi + if [ -L "$${marker}" ]; then + echo "Refusing a linked RustFS volume marker." >&2 + exit 44 + elif [ -e "$${marker}" ]; then + test -f "$${marker}" + test "$$(cat "$${marker}")" = "yuqei-rustfs-volume-v1" + elif [ -n "$$(find /data -mindepth 1 -maxdepth 1 -print -quit)" ]; then + echo "Refusing to mount an unowned non-empty object-storage volume." >&2 + exit 43 + else + umask 077 + set -C + printf '%s\n' 'yuqei-rustfs-volume-v1' > "$${marker}" + fi + volumes: + - rustfs-data:/data + restart: "no" + cap_drop: ["ALL"] + security_opt: + - no-new-privileges:true + minio: - image: ${MINIO_IMAGE:-minio/minio:RELEASE.2023-06-23T20-26-00Z} + # The service/DNS and environment keys stay stable for the application; + # the implementation is RustFS and uses a distinct data volume. + image: ${MINIO_IMAGE:-rustfs/rustfs:1.0.0} restart: unless-stopped - command: server /data --console-address ":9001" environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-yuqei-minio} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-yuqei-minio-local} + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: 0.0.0.0:9000 + RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001 + RUSTFS_CONSOLE_ENABLE: "true" + RUSTFS_ACCESS_KEY: ${MINIO_ROOT_USER:-yuqei-minio} + RUSTFS_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-yuqei-minio-local} + RUSTFS_REGION: ${AI_PLATFORM_OBJECT_STORAGE_REGION_NAME:-us-east-1} ports: - "${MINIO_API_BIND_ADDRESS:-0.0.0.0}:${MINIO_API_HOST_PORT:-9000}:9000" - "${MINIO_CONSOLE_BIND_ADDRESS:-0.0.0.0}:${MINIO_CONSOLE_HOST_PORT:-9001}:9001" volumes: - - minio-data:/data + - rustfs-data:/data healthcheck: - test: ["CMD-SHELL", "test -f /data/.minio.sys/format.json"] + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"] interval: 10s timeout: 5s retries: 12 start_period: 45s + depends_on: + object-storage-volume-preflight: + condition: service_completed_successfully + cap_drop: ["ALL"] + security_opt: + - no-new-privileges:true contract-api-migrate: build: @@ -488,5 +533,5 @@ services: volumes: postgres-data: - minio-data: + rustfs-data: contract-files: diff --git a/yuqei-infra/deploy/verify-production-compose.py b/yuqei-infra/deploy/verify-production-compose.py index e354a0ed..a408b7f5 100644 --- a/yuqei-infra/deploy/verify-production-compose.py +++ b/yuqei-infra/deploy/verify-production-compose.py @@ -54,6 +54,7 @@ PRODUCTION_SERVICES = ( "postgres-init-databases", "redis", "ai-redis", + "object-storage-volume-preflight", "minio", "contract-api-migrate", "contract-api", diff --git a/yuqei-infra/tests/test_production_compose_merge_gate.py b/yuqei-infra/tests/test_production_compose_merge_gate.py index deb0b72f..4f50f09e 100644 --- a/yuqei-infra/tests/test_production_compose_merge_gate.py +++ b/yuqei-infra/tests/test_production_compose_merge_gate.py @@ -137,7 +137,7 @@ def test_gate_renders_online_and_offline_for_all_profile_combinations(tmp_path: assert report["status"] == "passed" assert report["application_service_count"] == 9 - assert report["production_service_count"] == 17 + assert report["production_service_count"] == 18 assert len(report["cases"]) == 8 assert [(case["profile_case"], case["mode"]) for case in report["cases"]] == [ ("default", "online"), @@ -150,14 +150,14 @@ def test_gate_renders_online_and_offline_for_all_profile_combinations(tmp_path: ("backend-hosted-web+ops", "offline"), ] assert [case["service_count"] for case in report["cases"]] == [ - 14, - 14, 15, 15, 16, 16, 17, 17, + 18, + 18, ] assert len(runner.calls) == 8 diff --git a/yuqei-infra/tests/test_production_compose_preflight.py b/yuqei-infra/tests/test_production_compose_preflight.py index 5d8278fc..cc3e7e69 100644 --- a/yuqei-infra/tests/test_production_compose_preflight.py +++ b/yuqei-infra/tests/test_production_compose_preflight.py @@ -40,6 +40,7 @@ PRODUCTION_IMAGE_SERVICES = APPLICATION_SERVICES | { "postgres-init-databases", "redis", "ai-redis", + "object-storage-volume-preflight", "minio", "edge", "contract-files-backup", @@ -162,6 +163,29 @@ def test_ai_and_contract_anti_abuse_use_independent_redis_services() -> None: assert "ai-redis" in services["worker"]["depends_on"] +def test_rustfs_uses_a_distinct_fail_closed_volume_contract() -> None: + base = _load_compose(BASE_COMPOSE) + model = _production_model() + services = base["services"] + storage = services["minio"] + preflight_service = services["object-storage-volume-preflight"] + + assert "rustfs/rustfs" in storage["image"] + assert storage["volumes"] == ["rustfs-data:/data"] + assert storage["environment"]["RUSTFS_VOLUMES"] == "/data" + assert storage["environment"]["RUSTFS_ACCESS_KEY"].startswith("${MINIO_ROOT_USER") + assert storage["depends_on"]["object-storage-volume-preflight"]["condition"] == ( + "service_completed_successfully" + ) + assert preflight_service["volumes"] == ["rustfs-data:/data"] + preflight_command = "\n".join(preflight_service["command"]) + assert "/data/.minio.sys" in preflight_command + assert ".yuqei-rustfs-volume-v1" in preflight_command + assert "rustfs-data" in model["volumes"] + assert "minio-data" not in model["volumes"] + assert "yuqei-v2-production-rustfs-v1" in model["volumes"]["rustfs-data"]["name"] + + def test_edge_healthcheck_verifies_hostname_and_pinned_ca() -> None: edge = _production_model()["services"]["edge"] command = " ".join(edge["healthcheck"]["test"]) diff --git a/yuqei-infra/tests/test_production_release_tools.py b/yuqei-infra/tests/test_production_release_tools.py index 00b92aaf..da931777 100644 --- a/yuqei-infra/tests/test_production_release_tools.py +++ b/yuqei-infra/tests/test_production_release_tools.py @@ -69,8 +69,8 @@ def production_values() -> dict[str, str]: "POSTGRES_IMAGE": f"registry.internal/pgvector@sha256:{'4' * 64}", "POSTGRES_CLIENT_IMAGE": f"registry.internal/postgres@sha256:{'5' * 64}", "REDIS_IMAGE": f"registry.internal/redis@sha256:{'6' * 64}", - "MINIO_IMAGE": f"registry.internal/minio@sha256:{'7' * 64}", - "MINIO_MC_IMAGE": f"registry.internal/minio-mc@sha256:{'8' * 64}", + "MINIO_IMAGE": f"registry.internal/rustfs@sha256:{'7' * 64}", + "MINIO_MC_IMAGE": f"registry.internal/rustfs-rc@sha256:{'8' * 64}", "NGINX_IMAGE": f"registry.internal/nginx@sha256:{'9' * 64}", "BACKUP_UTILITY_IMAGE": f"registry.internal/alpine@sha256:{'a' * 64}", "BACKEND_SERVER_NAME": "api.contracts.acme.cn", @@ -107,34 +107,52 @@ def production_values() -> dict[str, str]: return values -def test_minio_backup_invokes_pinned_mc_directly_without_a_shell(tmp_path: Path) -> None: +def test_object_storage_backup_invokes_pinned_rc_without_shell_or_secret_argv( + tmp_path: Path, +) -> None: partial_dir = tmp_path / ".backup-20260811T120000Z.partial" - credentials = tmp_path / "minio.env" values = { "COMPOSE_PROJECT_NAME": "yuqei-v2-production", "AI_PLATFORM_OBJECT_STORAGE_BUCKET": "ai-platform-uploads", - "MINIO_MC_IMAGE": f"minio/mc@sha256:{'a' * 64}", + "MINIO_ROOT_USER": "backup-access-key", + "MINIO_ROOT_PASSWORD": "backup-secret-value", + "MINIO_MC_IMAGE": f"rustfs/rc@sha256:{'a' * 64}", } + rc_home = backup_run._write_rc_config_home(values) + try: + command = backup_run.object_storage_mirror_command( + values=values, + partial_dir=partial_dir, + rc_config_home=rc_home, + ) - command = backup_run.minio_mirror_command( - values=values, - partial_dir=partial_dir, - credential_env_file=credentials, - ) + assert command[:2] == ["docker", "run"] + assert "--entrypoint" not in command + assert "/bin/sh" not in command + assert "--env-file" not in command + assert command[-6:] == [ + "--no-color", + "--no-progress", + "mirror", + "--overwrite", + "source/ai-platform-uploads/", + "/backup/minio/", + ] + assert ["--read-only", "--cap-drop", "ALL"] == command[7:10] + assert "no-new-privileges:true" in command + assert all("backup-secret-value" not in token for token in command) + config = (rc_home / ".config" / "rc" / "config.toml").read_text(encoding="utf-8") + assert 'name = "source"' in config + assert 'endpoint = "http://minio:9000"' in config + assert 'access_key = "backup-access-key"' in config + assert 'secret_key = "backup-secret-value"' in config + finally: + shutil.rmtree(rc_home.parent) - assert command[:2] == ["docker", "run"] - assert "--entrypoint" not in command - assert "/bin/sh" not in command - assert command[-6:] == [ - "--config-dir", - "/tmp/.mc", - "mirror", - "--overwrite", - "source/ai-platform-uploads", - "/backup/minio", - ] - assert command[command.index("--env-file") + 1] == str(credentials) - assert all("secret" not in token for token in command) + +def test_object_storage_backup_rejects_unsafe_bind_source(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="unsafe character"): + backup_run._docker_bind_source(tmp_path / "comma,path") def test_backup_quiesces_every_long_running_application_writer() -> None: @@ -720,10 +738,16 @@ def _complete_backup_fixture(tmp_path: Path) -> tuple[Path, dict[str, object]]: item for item in artifacts if str(item["path"]).startswith("minio/") ] manifest: dict[str, object] = { - "schema_version": "yuqei-production-backup.v1", + "schema_version": "yuqei-production-backup-artifacts.v2", "backup_id": backup_dir.name, "created_at": "2026-07-16T12:00:00+00:00", - "minio_mirror": { + "object_storage_mirror": { + "engine": "rustfs", + "engine_version": "1.0.0", + "engine_image": f"registry.internal/rustfs@sha256:{'7' * 64}", + "client": "rustfs-rc", + "client_version": "0.1.36", + "client_image": f"registry.internal/rustfs-rc@sha256:{'8' * 64}", "bucket": "ai-platform-uploads", "path": "minio", "object_count": len(minio_records), @@ -750,6 +774,31 @@ def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> No assert len(verified) == 5 +@pytest.mark.parametrize( + ("field", "value", "message"), + ( + ("engine", "minio", "engine or backup client"), + ("client", "minio-mc", "engine or backup client"), + ("engine_image", "rustfs/rustfs:1.0.0", "image identity is not immutable"), + ("client_image", "rustfs/rc:0.1.36", "image identity is not immutable"), + ), +) +def test_v2_backup_manifest_rejects_unbound_object_storage_identity( + tmp_path: Path, + field: str, + value: str, + message: str, +) -> None: + backup_dir, manifest = _complete_backup_fixture(tmp_path) + manifest["object_storage_mirror"][field] = value + (backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + + with pytest.raises(RuntimeError, match=message): + backup_run.verify_manifest(backup_dir) + with pytest.raises(RuntimeError, match=message): + backup_verify.validate_backup(backup_dir) + + def test_backup_manifest_requires_all_databases_contract_files_and_minio( tmp_path: Path, ) -> None: @@ -764,7 +813,8 @@ def test_backup_manifest_requires_all_databases_contract_files_and_minio( backup_verify.validate_backup(backup_dir) legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy") - legacy_manifest.pop("minio_mirror") + legacy_manifest["schema_version"] = "yuqei-production-backup.v1" + legacy_manifest.pop("object_storage_mirror") legacy_manifest["artifacts"] = [ item for item in legacy_manifest["artifacts"] @@ -790,20 +840,21 @@ def test_backup_manifest_supports_explicit_empty_minio_and_legacy_content( for item in empty_manifest["artifacts"] if not str(item["path"]).startswith("minio/") ] - empty_manifest["minio_mirror"] = { + empty_manifest["object_storage_mirror"].update({ "bucket": "ai-platform-uploads", "path": "minio", "object_count": 0, "total_size_bytes": 0, "content_index_sha256": backup_run._minio_content_index([]), - } + }) (empty_dir / "manifest.json").write_text( json.dumps(empty_manifest), encoding="utf-8" ) backup_verify.validate_backup(empty_dir) legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy-content") - legacy_manifest.pop("minio_mirror") + legacy_manifest["schema_version"] = "yuqei-production-backup.v1" + legacy_manifest.pop("object_storage_mirror") (legacy_dir / "manifest.json").write_text( json.dumps(legacy_manifest), encoding="utf-8" )