yuqei/ai-contract-platform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(release): bind RustFS into formal compose gate

Browse Source
This commit is contained in:
yuqei
2026-09-27 15:07:36 +08:00
parent 3d17266f79
commit dcb53c55c1
6 changed files with 293 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yuqei-infra/deploy/verify-production-compose.py
-4
View File
@@ -619,7 +619,6 @@ def _validate_model(
)
validated_application_services.append(service_name)
validated_infrastructure_services: list[str] = []
for service_name, env_key in INFRASTRUCTURE_SERVICE_IMAGE_ENV.items():
if service_name not in services:
continue
@@ -638,7 +637,6 @@ def _validate_model(
f"{mode}/{profile_case.name}: service {service_name} retains a "
"build definition."
)
validated_infrastructure_services.append(service_name)
if mode == "offline":
for service_name in sorted(actual_names):
@@ -664,7 +662,6 @@ def _validate_model(
"profiles": list(profile_case.profiles),
"service_count": len(actual_names),
"validated_application_services": validated_application_services,
"validated_infrastructure_services": validated_infrastructure_services,
"config_sha256": hashlib.sha256(_canonical_json_bytes(payload)).hexdigest(),
}
@@ -798,7 +795,6 @@ def verify_production_compose(
for path in (base, production, offline)
],
"application_service_count": len(APPLICATION_SERVICE_IMAGE_ENV),
"infrastructure_service_count": len(INFRASTRUCTURE_SERVICE_IMAGE_ENV),
"production_service_count": len(PRODUCTION_SERVICES),
"cases": results,
}
yuqei-infra/scripts/linux_release_common.py
+3 -2
View File
@@ -678,8 +678,8 @@ def synthetic_compose_gate_env_bytes() -> bytes:
"POSTGRES_IMAGE": ("postgres", "5"),
"POSTGRES_CLIENT_IMAGE": ("postgres-client", "6"),
"REDIS_IMAGE": ("redis", "7"),
"MINIO_IMAGE": ("minio", "8"),
"MINIO_MC_IMAGE": ("minio-mc", "9"),
"MINIO_IMAGE": ("rustfs", "8"),
"MINIO_MC_IMAGE": ("rustfs-rc", "9"),
"NGINX_IMAGE": ("nginx", "a"),
"BACKUP_UTILITY_IMAGE": ("backup-utility", "b"),
}
@@ -706,6 +706,7 @@ def synthetic_compose_gate_env_bytes() -> bytes:
"CONTRACT_API_PUBLIC_POLICY_RELEASE_DIGEST": "sha256:" + "e" * 64,
"MINIO_ROOT_PASSWORD": "synthetic-not-a-production-secret",
"MINIO_ROOT_USER": "synthetic-not-a-production-user",
"MINIO_VOLUME_NAME": "yuqei-rc04-structural-rustfs-v1",
"PUBLIC_POLICY_CANDIDATE_DIR_HOST": "/synthetic/public-policy/candidate",
"PUBLIC_POLICY_LEGAL_APPROVAL_FILE_HOST": (
"/synthetic/public-policy/legal-approval.json"
yuqei-infra/scripts/run-linux-release-build.py
+129 -1
View File
@@ -86,6 +86,7 @@ POLICY_SOURCE_RELATIVE = "yuqei-infra/release/linux-release-security-policy.v1.j
COMPOSE_CONFIG_MAX_BYTES = 8 * 1024 * 1024
COMPOSE_ENV_KEY_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
COMPOSE_PROJECT_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
COMPOSE_VOLUME_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$")
COMPOSE_IMAGE_REFERENCE_RE = re.compile(
r"^[a-z0-9](?:[a-z0-9._:/-]{0,510}[a-z0-9])?@sha256:[0-9a-f]{64}$"
)
@@ -100,6 +101,16 @@ TRUSTED_APPLICATION_SERVICE_IMAGE_ENV: Mapping[str, str] = {
"worker": "AI_WORKER_IMAGE",
"contract-web": "CONTRACT_WEB_IMAGE",
}
TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV: Mapping[str, str] = {
"postgres": "POSTGRES_IMAGE",
"postgres-init-databases": "POSTGRES_CLIENT_IMAGE",
"redis": "REDIS_IMAGE",
"ai-redis": "REDIS_IMAGE",
"minio": "MINIO_IMAGE",
"edge": "NGINX_IMAGE",
"contract-files-backup": "BACKUP_UTILITY_IMAGE",
"minio-backup": "MINIO_MC_IMAGE",
}
TRUSTED_PRODUCTION_SERVICES = (
"postgres",
"postgres-init-databases",
@@ -587,7 +598,24 @@ def _strict_synthetic_compose_env(data: bytes) -> dict[str, str]:
raise ReleaseValidationError(
"synthetic Compose gate environment lacks a safe COMPOSE_PROJECT_NAME"
)
for env_key in sorted(set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values())):
storage_volume = values.get("MINIO_VOLUME_NAME", "")
if (
not COMPOSE_VOLUME_NAME_RE.fullmatch(storage_volume)
or "rustfs" not in storage_volume.lower()
or "minio" in storage_volume.lower()
):
raise ReleaseValidationError(
"synthetic Compose gate environment lacks a dedicated RustFS volume name"
)
for key in ("MINIO_ROOT_USER", "MINIO_ROOT_PASSWORD"):
if not values.get(key, ""):
raise ReleaseValidationError(
f"synthetic Compose gate environment lacks {key}"
)
image_env_keys = set(TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.values()) | set(
TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.values()
)
for env_key in sorted(image_env_keys):
image = values.get(env_key)
if not isinstance(image, str) or not COMPOSE_IMAGE_REFERENCE_RE.fullmatch(image):
raise ReleaseValidationError(
@@ -650,6 +678,85 @@ def _required_compose_services(profiles: Sequence[str]) -> set[str]:
}
def _validate_trusted_object_storage_contract(
services: Mapping[str, Any],
*,
volumes: object,
env_values: Mapping[str, str],
mode: str,
profile_name: str,
) -> None:
label = f"{mode}/{profile_name}"
storage = services.get("minio")
if not isinstance(storage, dict):
raise ReleaseValidationError(
f"{label}: object-storage service is missing or invalid"
)
mounts = storage.get("volumes")
valid_mount = (
isinstance(mounts, list)
and len(mounts) == 1
and isinstance(mounts[0], dict)
and mounts[0].get("type") == "volume"
and mounts[0].get("source") == "rustfs-data"
and mounts[0].get("target") == "/data"
and mounts[0].get("read_only") in (None, False)
)
volume = volumes.get("rustfs-data") if isinstance(volumes, dict) else None
valid_volume = (
isinstance(volume, dict)
and volume.get("name") == env_values["MINIO_VOLUME_NAME"]
)
if not valid_mount or not valid_volume:
raise ReleaseValidationError(
f"{label}: object storage must use only the dedicated rustfs-data volume"
)
command = storage.get("command")
command_text = "\n".join(command) if isinstance(command, list) else str(command or "")
if (
storage.get("entrypoint") != ["/bin/sh", "-ec"]
or "set -eu" not in command_text
or "/data/.minio.sys" not in command_text
or ".yuqei-rustfs-volume-v1" not in command_text
or "exec /entrypoint.sh rustfs" not in command_text
):
raise ReleaseValidationError(
f"{label}: object-storage volume guard or fixed RustFS startup is missing"
)
environment = storage.get("environment")
required_environment = {
"RUSTFS_VOLUMES": "/data",
"RUSTFS_ADDRESS": "0.0.0.0:9000",
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
}
if not isinstance(environment, dict) or any(
str(environment.get(key, "")) != value
for key, value in required_environment.items()
):
raise ReleaseValidationError(f"{label}: RustFS runtime environment is invalid")
healthcheck = storage.get("healthcheck")
health_test = healthcheck.get("test") if isinstance(healthcheck, dict) else None
if not isinstance(health_test, list) or not any(
isinstance(token, str) and "/health/ready" in token for token in health_test
):
raise ReleaseValidationError(f"{label}: RustFS readiness healthcheck is missing")
if (
storage.get("user") != "10001:10001"
or storage.get("read_only") is not True
or not isinstance(storage.get("tmpfs"), list)
or not any(str(item).startswith("/tmp:") for item in storage["tmpfs"])
or not any(str(item).startswith("/logs:") for item in storage["tmpfs"])
or "ALL" not in storage.get("cap_drop", [])
or "no-new-privileges:true" not in storage.get("security_opt", [])
or storage.get("privileged") not in (None, False)
):
raise ReleaseValidationError(f"{label}: RustFS runtime hardening is incomplete")
def _validate_trusted_compose_model(
payload: Mapping[str, Any],
*,
@@ -677,6 +784,14 @@ def _validate_trusted_compose_model(
f"{mode}/{profile_name}: service {service_name} is not an object"
)
_validate_trusted_object_storage_contract(
services,
volumes=payload.get("volumes"),
env_values=env_values,
mode=mode,
profile_name=profile_name,
)
validated_application_services: list[str] = []
for service_name, env_key in TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.items():
if service_name not in services:
@@ -697,6 +812,19 @@ def _validate_trusted_compose_model(
)
validated_application_services.append(service_name)
for service_name, env_key in TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.items():
if service_name not in services:
continue
service = services[service_name]
if service.get("image") != env_values[env_key]:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} image does not exactly match {env_key}"
)
if service.get("build") is not None:
raise ReleaseValidationError(
f"{mode}/{profile_name}: service {service_name} retains a build definition"
)
if mode == "offline":
for service_name in sorted(actual_names):
service = services[service_name]
yuqei-infra/tests/test_linux_release_build_tools.py
+95 -1
View File
@@ -685,16 +685,110 @@ def _compose_model(
service: dict[str, Any] = {
"pull_policy": "never" if mode == "offline" else "always"
}
env_key = builder.TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
if service_name == "minio":
service.update(
{
"entrypoint": ["/bin/sh", "-ec"],
"command": [
"set -eu\n"
"test ! -e /data/.minio.sys\n"
"marker=/data/.yuqei-rustfs-volume-v1\n"
"exec /entrypoint.sh rustfs\n"
],
"environment": {
"RUSTFS_VOLUMES": "/data",
"RUSTFS_ADDRESS": "0.0.0.0:9000",
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
},
"volumes": [
{
"type": "volume",
"source": "rustfs-data",
"target": "/data",
}
],
"healthcheck": {
"test": [
"CMD-SHELL",
"curl -fsS http://127.0.0.1:9000/health/ready",
]
},
"user": "10001:10001",
"read_only": True,
"tmpfs": [
"/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777",
"/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001",
],
"cap_drop": ["ALL"],
"security_opt": ["no-new-privileges:true"],
}
)
env_key = (
builder.TRUSTED_APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
or builder.TRUSTED_INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name)
)
if env_key is not None:
service["image"] = env_values[env_key]
services[service_name] = service
return {
"name": f"fixture-{mode}-{profile_name}",
"services": services,
"volumes": {
"rustfs-data": {"name": env_values["MINIO_VOLUME_NAME"]},
},
}
def test_trusted_compose_model_rejects_infrastructure_image_drift() -> None:
env_values = _synthetic_compose_env_values()
model = _compose_model(mode="online", profile_name="default", profiles=())
model["services"]["minio"]["image"] = (
"synthetic.invalid/yuqei/unreviewed-rustfs@sha256:" + "f" * 64
)
with pytest.raises(common.ReleaseValidationError, match="exactly match MINIO_IMAGE"):
builder._validate_trusted_compose_model(
model,
env_values=env_values,
mode="online",
profile_name="default",
profiles=(),
)
@pytest.mark.parametrize(
("mutation", "message"),
[
("volume", "dedicated rustfs-data volume"),
("guard", "volume guard or fixed RustFS startup"),
("hardening", "runtime hardening"),
],
)
def test_trusted_compose_model_rejects_rustfs_contract_drift(
mutation: str, message: str
) -> None:
env_values = _synthetic_compose_env_values()
model = _compose_model(mode="online", profile_name="default", profiles=())
if mutation == "volume":
model["volumes"]["rustfs-data"]["name"] = "legacy-production-minio"
elif mutation == "guard":
model["services"]["minio"]["command"] = ["exec /entrypoint.sh rustfs"]
else:
model["services"]["minio"]["read_only"] = False
with pytest.raises(common.ReleaseValidationError, match=message):
builder._validate_trusted_compose_model(
model,
env_values=env_values,
mode="online",
profile_name="default",
profiles=(),
)
def _bundle(tmp_path: Path) -> dict[str, Any]:
source = _fixture_source(tmp_path)
root = tmp_path / "bundle"
yuqei-infra/tests/test_production_compose_merge_gate.py
-1
View File
@@ -198,7 +198,6 @@ def test_gate_renders_online_and_offline_for_all_profile_combinations(tmp_path:
assert report["status"] == "passed"
assert report["application_service_count"] == 9
assert report["infrastructure_service_count"] == 8
assert report["production_service_count"] == 17
assert len(report["cases"]) == 8
assert [(case["profile_case"], case["mode"]) for case in report["cases"]] == [
yuqei-infra/tests/test_release_package_executor.py
+66 -5
View File
@@ -1497,7 +1497,10 @@ class InspectionRunner:
compose_gate = release_package._load_compose_gate_module()
env_file = Path(command[command.index("--env-file") + 1])
_target, env_values = compose_gate.parse_strict_env(env_file)
for env_key in set(compose_gate.APPLICATION_SERVICE_IMAGE_ENV.values()):
runtime_image_env_keys = set(
compose_gate.APPLICATION_SERVICE_IMAGE_ENV.values()
) | set(compose_gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.values())
for env_key in runtime_image_env_keys:
if env_key in kwargs["env"]:
env_values[env_key] = kwargs["env"][env_key]
profiles = {
@@ -1524,17 +1527,72 @@ class InspectionRunner:
}
if service_name in compose_gate.REDIS_SERVICE_NAMES:
service["command"] = list(compose_gate.EXPECTED_REDIS_COMMAND)
image_env = compose_gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
if service_name == "minio":
service.update(
{
"entrypoint": ["/bin/sh", "-ec"],
"command": [
"set -eu\n"
"test ! -e /data/.minio.sys\n"
"marker=/data/.yuqei-rustfs-volume-v1\n"
"exec /entrypoint.sh rustfs\n"
],
"environment": {
"RUSTFS_VOLUMES": "/data",
"RUSTFS_ADDRESS": "0.0.0.0:9000",
"RUSTFS_CONSOLE_ADDRESS": "0.0.0.0:9001",
"RUSTFS_CONSOLE_ENABLE": "true",
"RUSTFS_ACCESS_KEY": env_values["MINIO_ROOT_USER"],
"RUSTFS_SECRET_KEY": env_values["MINIO_ROOT_PASSWORD"],
},
"volumes": [
{
"type": "volume",
"source": "rustfs-data",
"target": "/data",
}
],
"healthcheck": {
"test": [
"CMD-SHELL",
"curl -fsS http://127.0.0.1:9000/health/ready",
]
},
"user": "10001:10001",
"read_only": True,
"tmpfs": [
"/tmp:rw,noexec,nosuid,nodev,size=16m,mode=1777",
"/logs:rw,noexec,nosuid,nodev,size=64m,mode=0700,uid=10001,gid=10001",
],
"cap_drop": ["ALL"],
"security_opt": ["no-new-privileges:true"],
}
)
image_env = (
compose_gate.APPLICATION_SERVICE_IMAGE_ENV.get(service_name)
or compose_gate.INFRASTRUCTURE_SERVICE_IMAGE_ENV.get(service_name)
)
if image_env is not None:
service["image"] = env_values[image_env]
service["pull_policy"] = "never" if offline else "always"
if service_name in compose_gate.APPLICATION_SERVICE_IMAGE_ENV:
service["pull_policy"] = "never" if offline else "always"
elif offline:
service["pull_policy"] = "never"
elif offline:
service["pull_policy"] = "never"
services[service_name] = service
return subprocess.CompletedProcess(
command,
0,
stdout=json.dumps({"name": env_values["COMPOSE_PROJECT_NAME"], "services": services}),
stdout=json.dumps(
{
"name": env_values["COMPOSE_PROJECT_NAME"],
"services": services,
"volumes": {
"rustfs-data": {"name": env_values["MINIO_VOLUME_NAME"]}
},
}
),
stderr="",
)
@@ -2936,7 +2994,10 @@ def test_preflight_failure_stops_before_state_or_apply_runner(tmp_path: Path) ->
"registry.example.test/yuqei/postgres:latest",
)
with pytest.raises(release_package.ReleasePackageError, match="Production preflight failed"):
with pytest.raises(
release_package.ReleasePackageError,
match="POSTGRES_IMAGE must be an explicit lowercase immutable",
):
_execute(
fixture,
operation="install",