From dcb53c55c19f09a5cab05f2c1830f0102c03e825 Mon Sep 17 00:00:00 2001 From: yuqei Date: Sun, 27 Sep 2026 15:07:36 +0800 Subject: [PATCH] fix(release): bind RustFS into formal compose gate --- .../deploy/verify-production-compose.py | 4 - yuqei-infra/scripts/linux_release_common.py | 5 +- .../scripts/run-linux-release-build.py | 130 +++++++++++++++++- .../tests/test_linux_release_build_tools.py | 96 ++++++++++++- .../test_production_compose_merge_gate.py | 1 - .../tests/test_release_package_executor.py | 71 +++++++++- 6 files changed, 293 insertions(+), 14 deletions(-) diff --git a/yuqei-infra/deploy/verify-production-compose.py b/yuqei-infra/deploy/verify-production-compose.py index 92c6beec..8ba2f474 100644 --- a/yuqei-infra/deploy/verify-production-compose.py +++ b/yuqei-infra/deploy/verify-production-compose.py @@ -619,7 +619,6 @@ def _validate_model( ) validated_application_services.append(service_name) - validated_infrastructure_services: list[str] = [] for service_name, env_key in INFRASTRUCTURE_SERVICE_IMAGE_ENV.items(): if service_name not in services: continue @@ -638,7 +637,6 @@ def _validate_model( f"{mode}/{profile_case.name}: service {service_name} retains a " "build definition." ) - validated_infrastructure_services.append(service_name) if mode == "offline": for service_name in sorted(actual_names): @@ -664,7 +662,6 @@ def _validate_model( "profiles": list(profile_case.profiles), "service_count": len(actual_names), "validated_application_services": validated_application_services, - "validated_infrastructure_services": validated_infrastructure_services, "config_sha256": hashlib.sha256(_canonical_json_bytes(payload)).hexdigest(), } @@ -798,7 +795,6 @@ def verify_production_compose( for path in (base, production, offline) ], "application_service_count": len(APPLICATION_SERVICE_IMAGE_ENV), - "infrastructure_service_count": len(INFRASTRUCTURE_SERVICE_IMAGE_ENV), "production_service_count": len(PRODUCTION_SERVICES), "cases": results, } diff --git a/yuqei-infra/scripts/linux_release_common.py b/yuqei-infra/scripts/linux_release_common.py index c0066a19..fa6415e6 100644 --- a/yuqei-infra/scripts/linux_release_common.py +++ b/yuqei-infra/scripts/linux_release_common.py @@ -678,8 +678,8 @@ def synthetic_compose_gate_env_bytes() -> bytes: "POSTGRES_IMAGE": ("postgres", "5"), "POSTGRES_CLIENT_IMAGE": ("postgres-client", "6"), "REDIS_IMAGE": ("redis", "7"), - "MINIO_IMAGE": ("minio", "8"), - "MINIO_MC_IMAGE": ("minio-mc", "9"), + "MINIO_IMAGE": ("rustfs", "8"), + "MINIO_MC_IMAGE": ("rustfs-rc", "9"), "NGINX_IMAGE": ("nginx", "a"), "BACKUP_UTILITY_IMAGE": ("backup-utility", "b"), } @@ -706,6 +706,7 @@ def synthetic_compose_gate_env_bytes() -> bytes: "CONTRACT_API_PUBLIC_POLICY_RELEASE_DIGEST": "sha256:" + "e" * 64, "MINIO_ROOT_PASSWORD": "synthetic-not-a-production-secret", "MINIO_ROOT_USER": "synthetic-not-a-production-user", + "MINIO_VOLUME_NAME": "yuqei-rc04-structural-rustfs-v1", "PUBLIC_POLICY_CANDIDATE_DIR_HOST": "/synthetic/public-policy/candidate", "PUBLIC_POLICY_LEGAL_APPROVAL_FILE_HOST": ( "/synthetic/public-policy/legal-approval.json" diff --git a/yuqei-infra/scripts/run-linux-release-build.py b/yuqei-infra/scripts/run-linux-release-build.py index ad5fd3dc..99d167b1 100644 --- a/yuqei-infra/scripts/run-linux-release-build.py +++ b/yuqei-infra/scripts/run-linux-release-build.py @@ -86,6 +86,7 @@ POLICY_SOURCE_RELATIVE = "yuqei-infra/release/linux-release-security-policy.v1.j COMPOSE_CONFIG_MAX_BYTES = 8 * 1024 * 1024 COMPOSE_ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") COMPOSE_PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$") +COMPOSE_VOLUME_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") COMPOSE_IMAGE_REFERENCE_RE = re.compile( r"^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$" ) @@ -100,6 +101,16 @@ TRUSTED_APPLICATION_SERVICE_IMAGE_ENV: Mapping[str, str] = { "worker": "AI_WORKER_IMAGE", "contract-web": "CONTRACT_WEB_IMAGE", } +TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV: Mapping[str, str] = { + "postgres": "POSTGRES_IMAGE", + "postgres-init-databases": "POSTGRES_CLIENT_IMAGE", + "redis": "REDIS_IMAGE", + "ai-redis": "REDIS_IMAGE", + "minio": "MINIO_IMAGE", + "edge": "NGINX_IMAGE", + "contract-files-backup": "BACKUP_UTILITY_IMAGE", + "minio-backup": "MINIO_MC_IMAGE", +} TRUSTED_PRODUCTION_SERVICES = ( "postgres", "postgres-init-databases", @@ -587,7 +598,24 @@ def _strict_synthetic_compose_env(data: bytes) -> dict[str, str]: raise ReleaseValidationError( "synthetic Compose gate environment lacks a safe COMPOSE_PROJECT_NAME" ) - for env_key in sorted(set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values())): + storage_volume = values.get("MINIO_VOLUME_NAME", "") + if ( + not COMPOSE_VOLUME_NAME_RE.fullmatch(storage_volume) + or "rustfs" not in storage_volume.lower() + or "minio" in storage_volume.lower() + ): + raise ReleaseValidationError( + "synthetic Compose gate environment lacks a dedicated RustFS volume name" + ) + for key in ("MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"): + if not values.get(key, ""): + raise ReleaseValidationError( + f"synthetic Compose gate environment lacks {key}" + ) + image_env_keys = set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values()) | set( + TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.values() + ) + for env_key in sorted(image_env_keys): image = values.get(env_key) if not isinstance(image, str) or not COMPOSE_IMAGE_REFERENCE_RE.fullmatch(image): raise ReleaseValidationError( @@ -650,6 +678,85 @@ def _required_compose_services(profiles: Sequence[str]) -> set[str]: } +def _validate_trusted_object_storage_contract( + services: Mapping[str, Any], + *, + volumes: object, + env_values: Mapping[str, str], + mode: str, + profile_name: str, +) -> None: + label = f"{mode}/{profile_name}" + storage = services.get("minio") + if not isinstance(storage, dict): + raise ReleaseValidationError( + f"{label}: object-storage service is missing or invalid" + ) + mounts = storage.get("volumes") + valid_mount = ( + isinstance(mounts, list) + and len(mounts) == 1 + and isinstance(mounts[0], dict) + and mounts[0].get("type") == "volume" + and mounts[0].get("source") == "rustfs-data" + and mounts[0].get("target") == "/data" + and mounts[0].get("read_only") in (None, False) + ) + volume = volumes.get("rustfs-data") if isinstance(volumes, dict) else None + valid_volume = ( + isinstance(volume, dict) + and volume.get("name") == env_values["MINIO_VOLUME_NAME"] + ) + if not valid_mount or not valid_volume: + raise ReleaseValidationError( + f"{label}: object storage must use only the dedicated rustfs-data volume" + ) + + command = storage.get("command") + command_text = "\n".join(command) if isinstance(command, list) else str(command or "") + if ( + storage.get("entrypoint") != ["/bin/sh", "-ec"] + or "set -eu" not in command_text + or "/data/.minio.sys" not in command_text + or ".yuqei-rustfs-volume-v1" not in command_text + or "exec /entrypoint.sh rustfs" not in command_text + ): + raise ReleaseValidationError( + f"{label}: object-storage volume guard or fixed RustFS startup is missing" + ) + environment = storage.get("environment") + required_environment = { + "RUSTFS_VOLUMES": "/data", + "RUSTFS_ADDRESS": "0.0.0.0:9000", + "RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001", + "RUSTFS_CONSOLE_ENABLE": "true", + "RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"], + "RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"], + } + if not isinstance(environment, dict) or any( + str(environment.get(key, "")) != value + for key, value in required_environment.items() + ): + raise ReleaseValidationError(f"{label}: RustFS runtime environment is invalid") + healthcheck = storage.get("healthcheck") + health_test = healthcheck.get("test") if isinstance(healthcheck, dict) else None + if not isinstance(health_test, list) or not any( + isinstance(token, str) and "/health/ready" in token for token in health_test + ): + raise ReleaseValidationError(f"{label}: RustFS readiness healthcheck is missing") + if ( + storage.get("user") != "10001:10001" + or storage.get("read_only") is not True + or not isinstance(storage.get("tmpfs"), list) + or not any(str(item).startswith("/tmp:") for item in storage["tmpfs"]) + or not any(str(item).startswith("/logs:") for item in storage["tmpfs"]) + or "ALL" not in storage.get("cap_drop", []) + or "no-new-privileges:true" not in storage.get("security_opt", []) + or storage.get("privileged") not in (None, False) + ): + raise ReleaseValidationError(f"{label}: RustFS runtime hardening is incomplete") + + def _validate_trusted_compose_model( payload: Mapping[str, Any], *, @@ -677,6 +784,14 @@ def _validate_trusted_compose_model( f"{mode}/{profile_name}: service {service_name} is not an object" ) + _validate_trusted_object_storage_contract( + services, + volumes=payload.get("volumes"), + env_values=env_values, + mode=mode, + profile_name=profile_name, + ) + validated_application_services: list[str] = [] for service_name, env_key in TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.items(): if service_name not in services: @@ -697,6 +812,19 @@ def _validate_trusted_compose_model( ) validated_application_services.append(service_name) + for service_name, env_key in TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.items(): + if service_name not in services: + continue + service = services[service_name] + if service.get("image") != env_values[env_key]: + raise ReleaseValidationError( + f"{mode}/{profile_name}: service {service_name} image does not exactly match {env_key}" + ) + if service.get("build") is not None: + raise ReleaseValidationError( + f"{mode}/{profile_name}: service {service_name} retains a build definition" + ) + if mode == "offline": for service_name in sorted(actual_names): service = services[service_name] diff --git a/yuqei-infra/tests/test_linux_release_build_tools.py b/yuqei-infra/tests/test_linux_release_build_tools.py index eb5a653c..104b7d03 100644 --- a/yuqei-infra/tests/test_linux_release_build_tools.py +++ b/yuqei-infra/tests/test_linux_release_build_tools.py @@ -685,16 +685,110 @@ def _compose_model( service: dict[str, Any] = { "pull_policy": "never" if mode == "offline" else "always" } - env_key = builder.TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.get(service_name) + if service_name == "minio": + service.update( + { + "entrypoint": ["/bin/sh", "-ec"], + "command": [ + "set -eu\n" + "test ! -e /data/.minio.sys\n" + "marker=/data/.yuqei-rustfs-volume-v1\n" + "exec /entrypoint.sh rustfs\n" + ], + "environment": { + "RUSTFS_VOLUMES": "/data", + "RUSTFS_ADDRESS": "0.0.0.0:9000", + "RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001", + "RUSTFS_CONSOLE_ENABLE": "true", + "RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"], + "RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"], + }, + "volumes": [ + { + "type": "volume", + "source": "rustfs-data", + "target": "/data", + } + ], + "healthcheck": { + "test": [ + "CMD-SHELL", + "curl -fsS http://127.0.0.1:9000/health/ready", + ] + }, + "user": "10001:10001", + "read_only": True, + "tmpfs": [ + "/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777", + "/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001", + ], + "cap_drop": ["ALL"], + "security_opt": ["no-new-privileges:true"], + } + ) + env_key = ( + builder.TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.get(service_name) + or builder.TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name) + ) if env_key is not None: service["image"] = env_values[env_key] services[service_name] = service return { "name": f"fixture-{mode}-{profile_name}", "services": services, + "volumes": { + "rustfs-data": {"name": env_values["MINIO_VOLUME_NAME"]}, + }, } +def test_trusted_compose_model_rejects_infrastructure_image_drift() -> None: + env_values = _synthetic_compose_env_values() + model = _compose_model(mode="online", profile_name="default", profiles=()) + model["services"]["minio"]["image"] = ( + "synthetic.invalid/yuqei/unreviewed-rustfs@sha256:" + "f" * 64 + ) + + with pytest.raises(common.ReleaseValidationError, match="exactly match MINIO_IMAGE"): + builder._validate_trusted_compose_model( + model, + env_values=env_values, + mode="online", + profile_name="default", + profiles=(), + ) + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ("volume", "dedicated rustfs-data volume"), + ("guard", "volume guard or fixed RustFS startup"), + ("hardening", "runtime hardening"), + ], +) +def test_trusted_compose_model_rejects_rustfs_contract_drift( + mutation: str, message: str +) -> None: + env_values = _synthetic_compose_env_values() + model = _compose_model(mode="online", profile_name="default", profiles=()) + if mutation == "volume": + model["volumes"]["rustfs-data"]["name"] = "legacy-production-minio" + elif mutation == "guard": + model["services"]["minio"]["command"] = ["exec /entrypoint.sh rustfs"] + else: + model["services"]["minio"]["read_only"] = False + + with pytest.raises(common.ReleaseValidationError, match=message): + builder._validate_trusted_compose_model( + model, + env_values=env_values, + mode="online", + profile_name="default", + profiles=(), + ) + + def _bundle(tmp_path: Path) -> dict[str, Any]: source = _fixture_source(tmp_path) root = tmp_path / "bundle" diff --git a/yuqei-infra/tests/test_production_compose_merge_gate.py b/yuqei-infra/tests/test_production_compose_merge_gate.py index cdd61eb9..500807c1 100644 --- a/yuqei-infra/tests/test_production_compose_merge_gate.py +++ b/yuqei-infra/tests/test_production_compose_merge_gate.py @@ -198,7 +198,6 @@ def test_gate_renders_online_and_offline_for_all_profile_combinations(tmp_path: assert report["status"] == "passed" assert report["application_service_count"] == 9 - assert report["infrastructure_service_count"] == 8 assert report["production_service_count"] == 17 assert len(report["cases"]) == 8 assert [(case["profile_case"], case["mode"]) for case in report["cases"]] == [ diff --git a/yuqei-infra/tests/test_release_package_executor.py b/yuqei-infra/tests/test_release_package_executor.py index da6eb8db..7f9ed232 100644 --- a/yuqei-infra/tests/test_release_package_executor.py +++ b/yuqei-infra/tests/test_release_package_executor.py @@ -1497,7 +1497,10 @@ class InspectionRunner: compose_gate = release_package._load_compose_gate_module() env_file = Path(command[command.index("--env-file") + 1]) _target, env_values = compose_gate.parse_strict_env(env_file) - for env_key in set(compose_gate.APPLICATION_SERVICE_IMAGE_ENV.values()): + runtime_image_env_keys = set( + compose_gate.APPLICATION_SERVICE_IMAGE_ENV.values() + ) | set(compose_gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.values()) + for env_key in runtime_image_env_keys: if env_key in kwargs["env"]: env_values[env_key] = kwargs["env"][env_key] profiles = { @@ -1524,17 +1527,72 @@ class InspectionRunner: } if service_name in compose_gate.REDIS_SERVICE_NAMES: service["command"] = list(compose_gate.EXPECTED_REDIS_COMMAND) - image_env = compose_gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name) + if service_name == "minio": + service.update( + { + "entrypoint": ["/bin/sh", "-ec"], + "command": [ + "set -eu\n" + "test ! -e /data/.minio.sys\n" + "marker=/data/.yuqei-rustfs-volume-v1\n" + "exec /entrypoint.sh rustfs\n" + ], + "environment": { + "RUSTFS_VOLUMES": "/data", + "RUSTFS_ADDRESS": "0.0.0.0:9000", + "RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001", + "RUSTFS_CONSOLE_ENABLE": "true", + "RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"], + "RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"], + }, + "volumes": [ + { + "type": "volume", + "source": "rustfs-data", + "target": "/data", + } + ], + "healthcheck": { + "test": [ + "CMD-SHELL", + "curl -fsS http://127.0.0.1:9000/health/ready", + ] + }, + "user": "10001:10001", + "read_only": True, + "tmpfs": [ + "/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777", + "/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001", + ], + "cap_drop": ["ALL"], + "security_opt": ["no-new-privileges:true"], + } + ) + image_env = ( + compose_gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name) + or compose_gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name) + ) if image_env is not None: service["image"] = env_values[image_env] - service["pull_policy"] = "never" if offline else "always" + if service_name in compose_gate.APPLICATION_SERVICE_IMAGE_ENV: + service["pull_policy"] = "never" if offline else "always" + elif offline: + service["pull_policy"] = "never" elif offline: service["pull_policy"] = "never" services[service_name] = service return subprocess.CompletedProcess( command, 0, - stdout=json.dumps({"name": env_values["COMPOSE_PROJECT_NAME"], "services": services}), + stdout=json.dumps( + { + "name": env_values["COMPOSE_PROJECT_NAME"], + "services": services, + "volumes": { + "rustfs-data": {"name": env_values["MINIO_VOLUME_NAME"]} + }, + } + ), stderr="", ) @@ -2936,7 +2994,10 @@ def test_preflight_failure_stops_before_state_or_apply_runner(tmp_path: Path) -> "registry.example.test/yuqei/postgres:latest", ) - with pytest.raises(release_package.ReleasePackageError, match="Production preflight failed"): + with pytest.raises( + release_package.ReleasePackageError, + match="POSTGRES_IMAGE must be an explicit lowercase immutable", + ): _execute( fixture, operation="install",