fix: harden RC-04 cold install and UAT recovery
This commit is contained in:
+2
-2
@@ -70,8 +70,8 @@ def upgrade() -> None:
|
||||
sa.CheckConstraint("status in ('pending', 'approved', 'rejected')", name="ck_reliable_audit_change_status"),
|
||||
)
|
||||
for name, columns in (
|
||||
("ix_reliable_evidence_audit_client_change_requests_owner_tenant_id", ["owner_tenant_id"]),
|
||||
("ix_reliable_evidence_audit_client_change_requests_audit_client_id", ["audit_client_id"]),
|
||||
("ix_reliable_audit_change_owner_tenant_id", ["owner_tenant_id"]),
|
||||
("ix_reliable_audit_change_audit_client_id", ["audit_client_id"]),
|
||||
("ix_reliable_evidence_audit_client_change_requests_action", ["action"]),
|
||||
("ix_reliable_evidence_audit_client_change_requests_status", ["status"]),
|
||||
("ix_reliable_evidence_audit_client_change_requests_proposed_by", ["proposed_by"]),
|
||||
|
||||
@@ -12,9 +12,10 @@ import shutil
|
||||
# Docker commands use fixed argv lists and never invoke a host shell.
|
||||
import subprocess # nosec B404
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import BinaryIO
|
||||
from urllib.parse import quote
|
||||
|
||||
@@ -26,7 +27,19 @@ IDENTIFIER_PATTERN = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
|
||||
DOCKER_PROJECT_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,62}$")
|
||||
S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$")
|
||||
BACKUP_RUN_PATTERN = re.compile(r"^backup-[0-9]{8}T[0-9]{6}Z$")
|
||||
WRITER_SERVICES = ("contract-api", "contract-review-worker", "ai-platform-api", "worker")
|
||||
WRITER_SERVICES = (
|
||||
"contract-api",
|
||||
"contract-review-worker",
|
||||
"contract-playbook-governance-worker",
|
||||
"ai-platform-api",
|
||||
"worker",
|
||||
)
|
||||
REQUIRED_BACKUP_ARTIFACTS = {
|
||||
"postgres-contract.dump",
|
||||
"postgres-ai_platform.dump",
|
||||
"postgres-operations.dump",
|
||||
"contract-files.tar.gz",
|
||||
}
|
||||
|
||||
|
||||
def parse_env_file(path: Path) -> dict[str, str]:
|
||||
@@ -132,6 +145,7 @@ def create_backup(
|
||||
ops_profile=True,
|
||||
)
|
||||
)
|
||||
_verify_contract_archive(partial_dir / "contract-files.tar.gz")
|
||||
minio_env_path = _write_minio_alias_env(values)
|
||||
try:
|
||||
_run(
|
||||
@@ -145,6 +159,11 @@ def create_backup(
|
||||
minio_env_path.unlink(missing_ok=True)
|
||||
|
||||
artifacts = _artifact_metadata(partial_dir)
|
||||
minio_mirror = _minio_mirror_metadata(
|
||||
partial_dir,
|
||||
artifacts=artifacts,
|
||||
bucket=values.get("AI_PLATFORM_OBJECT_STORAGE_BUCKET", "ai-platform-uploads"),
|
||||
)
|
||||
manifest = {
|
||||
"schema_version": "yuqei-production-backup.v1",
|
||||
"backup_id": run_id,
|
||||
@@ -158,6 +177,7 @@ def create_backup(
|
||||
"ai_platform_api": values.get("AI_PLATFORM_API_IMAGE"),
|
||||
"ai_worker": values.get("AI_WORKER_IMAGE"),
|
||||
},
|
||||
"minio_mirror": minio_mirror,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
(partial_dir / "manifest.json").write_text(
|
||||
@@ -183,10 +203,15 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]:
|
||||
artifacts = payload.get("artifacts")
|
||||
if not isinstance(artifacts, list) or not artifacts:
|
||||
raise RuntimeError("Backup manifest has no artifacts.")
|
||||
artifact_paths: set[str] = set()
|
||||
minio_records: list[dict[str, object]] = []
|
||||
for item in artifacts:
|
||||
if not isinstance(item, dict):
|
||||
raise RuntimeError("Backup manifest artifact is invalid.")
|
||||
relative_path = str(item.get("path", ""))
|
||||
if relative_path in artifact_paths:
|
||||
raise RuntimeError(f"Duplicate backup artifact path: {relative_path}")
|
||||
artifact_paths.add(relative_path)
|
||||
target = (backup_dir / relative_path).resolve()
|
||||
target.relative_to(backup_dir.resolve())
|
||||
if not target.is_file():
|
||||
@@ -195,6 +220,19 @@ def verify_manifest(backup_dir: Path) -> dict[str, object]:
|
||||
raise RuntimeError(f"Backup artifact size mismatch: {relative_path}")
|
||||
if _sha256(target) != item.get("sha256"):
|
||||
raise RuntimeError(f"Backup artifact checksum mismatch: {relative_path}")
|
||||
if relative_path.startswith("minio/"):
|
||||
minio_records.append(
|
||||
{
|
||||
"path": relative_path,
|
||||
"size_bytes": target.stat().st_size,
|
||||
"sha256": item.get("sha256"),
|
||||
}
|
||||
)
|
||||
missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths)
|
||||
if missing:
|
||||
raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}")
|
||||
_verify_minio_mirror(backup_dir, payload, minio_records)
|
||||
_verify_contract_archive(backup_dir / "contract-files.tar.gz")
|
||||
return payload
|
||||
|
||||
|
||||
@@ -224,6 +262,117 @@ def _artifact_metadata(root: Path) -> list[dict[str, object]]:
|
||||
]
|
||||
|
||||
|
||||
def _minio_content_index(records: list[dict[str, object]]) -> str:
|
||||
canonical = json.dumps(
|
||||
sorted(records, key=lambda item: str(item["path"]).encode("utf-8")),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
def _verify_contract_archive(path: Path) -> None:
|
||||
with tarfile.open(path, mode="r:gz") as archive:
|
||||
seen: set[str] = set()
|
||||
seen_folded: set[str] = set()
|
||||
for member in archive.getmembers():
|
||||
raw_name = member.name
|
||||
if "\\" in raw_name or "\x00" in raw_name:
|
||||
raise RuntimeError("Contract file archive contains an unsafe path.")
|
||||
member_path = PurePosixPath(raw_name)
|
||||
if member_path.is_absolute() or ".." in member_path.parts:
|
||||
raise RuntimeError("Contract file archive contains an unsafe path.")
|
||||
if not (member.isfile() or member.isdir()):
|
||||
raise RuntimeError("Contract file archive contains a link or special entry.")
|
||||
normalized = member_path.as_posix().removeprefix("./")
|
||||
if normalized not in {"", "."}:
|
||||
folded = normalized.casefold()
|
||||
if normalized in seen or folded in seen_folded:
|
||||
raise RuntimeError("Contract file archive contains a duplicate path.")
|
||||
seen.add(normalized)
|
||||
seen_folded.add(folded)
|
||||
|
||||
|
||||
def _minio_mirror_metadata(
|
||||
root: Path,
|
||||
*,
|
||||
artifacts: list[dict[str, object]],
|
||||
bucket: str,
|
||||
) -> dict[str, object]:
|
||||
minio_root = root / "minio"
|
||||
if minio_root.is_symlink() or not minio_root.is_dir():
|
||||
raise RuntimeError("MinIO mirror directory is missing or unsafe.")
|
||||
records = [
|
||||
{
|
||||
"path": str(item["path"]),
|
||||
"size_bytes": int(item["size_bytes"]),
|
||||
"sha256": str(item["sha256"]),
|
||||
}
|
||||
for item in artifacts
|
||||
if str(item.get("path", "")).startswith("minio/")
|
||||
]
|
||||
return {
|
||||
"bucket": bucket,
|
||||
"path": "minio",
|
||||
"object_count": len(records),
|
||||
"total_size_bytes": sum(int(item["size_bytes"]) for item in records),
|
||||
"content_index_sha256": _minio_content_index(records),
|
||||
}
|
||||
|
||||
|
||||
def _verify_minio_mirror(
|
||||
backup_dir: Path,
|
||||
manifest: dict[str, object],
|
||||
records: list[dict[str, object]],
|
||||
) -> None:
|
||||
minio_root = backup_dir / "minio"
|
||||
if minio_root.is_symlink() or not minio_root.is_dir():
|
||||
raise RuntimeError("MinIO mirror directory is missing or unsafe.")
|
||||
actual_files: set[str] = set()
|
||||
for path in minio_root.rglob("*"):
|
||||
relative = path.relative_to(backup_dir).as_posix()
|
||||
if path.is_symlink():
|
||||
raise RuntimeError(f"MinIO mirror contains a symbolic link: {relative}")
|
||||
if path.is_file():
|
||||
actual_files.add(relative)
|
||||
elif not path.is_dir():
|
||||
raise RuntimeError(f"MinIO mirror contains a special entry: {relative}")
|
||||
recorded_files = {str(item["path"]) for item in records}
|
||||
if actual_files != recorded_files:
|
||||
raise RuntimeError("MinIO mirror content differs from the backup manifest.")
|
||||
|
||||
metadata = manifest.get("minio_mirror")
|
||||
if metadata is None:
|
||||
if not records:
|
||||
raise RuntimeError(
|
||||
"Legacy backup has no MinIO mirror metadata or recorded MinIO content."
|
||||
)
|
||||
return
|
||||
if not isinstance(metadata, dict):
|
||||
raise RuntimeError("MinIO mirror metadata is invalid.")
|
||||
expected = {
|
||||
"bucket",
|
||||
"path",
|
||||
"object_count",
|
||||
"total_size_bytes",
|
||||
"content_index_sha256",
|
||||
}
|
||||
if set(metadata) != expected or metadata.get("path") != "minio":
|
||||
raise RuntimeError("MinIO mirror metadata fields are invalid.")
|
||||
if (
|
||||
not isinstance(metadata.get("bucket"), str)
|
||||
or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"])
|
||||
):
|
||||
raise RuntimeError("MinIO mirror bucket metadata is invalid.")
|
||||
if metadata.get("object_count") != len(records):
|
||||
raise RuntimeError("MinIO mirror object count mismatch.")
|
||||
if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records):
|
||||
raise RuntimeError("MinIO mirror total size mismatch.")
|
||||
if metadata.get("content_index_sha256") != _minio_content_index(records):
|
||||
raise RuntimeError("MinIO mirror content index mismatch.")
|
||||
|
||||
|
||||
def minio_mirror_command(
|
||||
*,
|
||||
values: dict[str, str],
|
||||
|
||||
@@ -13,7 +13,7 @@ import subprocess # nosec B404
|
||||
import sys
|
||||
import tarfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
@@ -21,6 +21,13 @@ BASE_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.yml"
|
||||
PRODUCTION_COMPOSE = REPO_ROOT / "yuqei-infra" / "compose" / "docker-compose.production.yml"
|
||||
DEFAULT_ARTIFACT_ROOT = REPO_ROOT / "project-docs" / "qa-artifacts" / "v2-production-restore-check"
|
||||
SAFE_DATABASE_PATTERN = re.compile(r"^yuqei_restore_check_(contract|ai_platform|operations)_[0-9]{14}$")
|
||||
S3_BUCKET_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,62}$")
|
||||
REQUIRED_BACKUP_ARTIFACTS = {
|
||||
"postgres-contract.dump",
|
||||
"postgres-ai_platform.dump",
|
||||
"postgres-operations.dump",
|
||||
"contract-files.tar.gz",
|
||||
}
|
||||
|
||||
|
||||
def read_env(path: Path) -> dict[str, str]:
|
||||
@@ -46,10 +53,15 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
|
||||
raise RuntimeError("Backup manifest has no artifacts.")
|
||||
verified: list[dict[str, object]] = []
|
||||
root = backup_dir.resolve()
|
||||
artifact_paths: set[str] = set()
|
||||
minio_records: list[dict[str, object]] = []
|
||||
for item in artifacts:
|
||||
if not isinstance(item, dict):
|
||||
raise RuntimeError("Invalid artifact entry.")
|
||||
relative_path = str(item.get("path", ""))
|
||||
if relative_path in artifact_paths:
|
||||
raise RuntimeError(f"Duplicate backup artifact path: {relative_path}")
|
||||
artifact_paths.add(relative_path)
|
||||
path = (root / relative_path).resolve()
|
||||
path.relative_to(root)
|
||||
if not path.is_file():
|
||||
@@ -59,17 +71,101 @@ def validate_backup(backup_dir: Path) -> tuple[dict[str, object], list[dict[str,
|
||||
if actual_size != int(item.get("size_bytes", -1)) or actual_sha256 != item.get("sha256"):
|
||||
raise RuntimeError(f"Backup artifact integrity mismatch: {relative_path}")
|
||||
verified.append({"path": relative_path, "size_bytes": actual_size, "sha256_match": True})
|
||||
if relative_path.startswith("minio/"):
|
||||
minio_records.append(
|
||||
{"path": relative_path, "size_bytes": actual_size, "sha256": actual_sha256}
|
||||
)
|
||||
|
||||
missing = sorted(REQUIRED_BACKUP_ARTIFACTS - artifact_paths)
|
||||
if missing:
|
||||
raise RuntimeError(f"Backup manifest is missing required artifacts: {missing}")
|
||||
_verify_minio_mirror(backup_dir, manifest, minio_records)
|
||||
|
||||
contract_archive = backup_dir / "contract-files.tar.gz"
|
||||
if contract_archive.is_file():
|
||||
with tarfile.open(contract_archive, mode="r:gz") as archive:
|
||||
for member in archive.getmembers():
|
||||
member_path = Path(member.name)
|
||||
if member_path.is_absolute() or ".." in member_path.parts:
|
||||
raise RuntimeError("Contract file archive contains an unsafe path.")
|
||||
with tarfile.open(contract_archive, mode="r:gz") as archive:
|
||||
seen: set[str] = set()
|
||||
seen_folded: set[str] = set()
|
||||
for member in archive.getmembers():
|
||||
raw_name = member.name
|
||||
if "\\" in raw_name or "\x00" in raw_name:
|
||||
raise RuntimeError("Contract file archive contains an unsafe path.")
|
||||
member_path = PurePosixPath(raw_name)
|
||||
if member_path.is_absolute() or ".." in member_path.parts:
|
||||
raise RuntimeError("Contract file archive contains an unsafe path.")
|
||||
if not (member.isfile() or member.isdir()):
|
||||
raise RuntimeError("Contract file archive contains a link or special entry.")
|
||||
normalized = member_path.as_posix().removeprefix("./")
|
||||
if normalized not in {"", "."}:
|
||||
folded = normalized.casefold()
|
||||
if normalized in seen or folded in seen_folded:
|
||||
raise RuntimeError("Contract file archive contains a duplicate path.")
|
||||
seen.add(normalized)
|
||||
seen_folded.add(folded)
|
||||
return manifest, verified
|
||||
|
||||
|
||||
def _minio_content_index(records: list[dict[str, object]]) -> str:
|
||||
canonical = json.dumps(
|
||||
sorted(records, key=lambda item: str(item["path"]).encode("utf-8")),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
def _verify_minio_mirror(
|
||||
backup_dir: Path,
|
||||
manifest: dict[str, object],
|
||||
records: list[dict[str, object]],
|
||||
) -> None:
|
||||
minio_root = backup_dir / "minio"
|
||||
if minio_root.is_symlink() or not minio_root.is_dir():
|
||||
raise RuntimeError("MinIO mirror directory is missing or unsafe.")
|
||||
actual_files: set[str] = set()
|
||||
for path in minio_root.rglob("*"):
|
||||
relative = path.relative_to(backup_dir).as_posix()
|
||||
if path.is_symlink():
|
||||
raise RuntimeError(f"MinIO mirror contains a symbolic link: {relative}")
|
||||
if path.is_file():
|
||||
actual_files.add(relative)
|
||||
elif not path.is_dir():
|
||||
raise RuntimeError(f"MinIO mirror contains a special entry: {relative}")
|
||||
recorded_files = {str(item["path"]) for item in records}
|
||||
if actual_files != recorded_files:
|
||||
raise RuntimeError("MinIO mirror content differs from the backup manifest.")
|
||||
|
||||
metadata = manifest.get("minio_mirror")
|
||||
if metadata is None:
|
||||
if not records:
|
||||
raise RuntimeError(
|
||||
"Legacy backup has no MinIO mirror metadata or recorded MinIO content."
|
||||
)
|
||||
return
|
||||
if not isinstance(metadata, dict):
|
||||
raise RuntimeError("MinIO mirror metadata is invalid.")
|
||||
expected = {
|
||||
"bucket",
|
||||
"path",
|
||||
"object_count",
|
||||
"total_size_bytes",
|
||||
"content_index_sha256",
|
||||
}
|
||||
if set(metadata) != expected or metadata.get("path") != "minio":
|
||||
raise RuntimeError("MinIO mirror metadata fields are invalid.")
|
||||
if (
|
||||
not isinstance(metadata.get("bucket"), str)
|
||||
or not S3_BUCKET_PATTERN.fullmatch(metadata["bucket"])
|
||||
):
|
||||
raise RuntimeError("MinIO mirror bucket metadata is invalid.")
|
||||
if metadata.get("object_count") != len(records):
|
||||
raise RuntimeError("MinIO mirror object count mismatch.")
|
||||
if metadata.get("total_size_bytes") != sum(int(item["size_bytes"]) for item in records):
|
||||
raise RuntimeError("MinIO mirror total size mismatch.")
|
||||
if metadata.get("content_index_sha256") != _minio_content_index(records):
|
||||
raise RuntimeError("MinIO mirror content index mismatch.")
|
||||
|
||||
|
||||
def restore_database_checks(
|
||||
*,
|
||||
backup_dir: Path,
|
||||
|
||||
@@ -123,6 +123,105 @@ function safeFilename(value) {
|
||||
return String(value).replace(/[^\w-]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 90) || "artifact";
|
||||
}
|
||||
|
||||
function safeRequestPath(value) {
|
||||
try {
|
||||
return new URL(String(value)).pathname || "/";
|
||||
} catch {
|
||||
return "<unparseable>";
|
||||
}
|
||||
}
|
||||
|
||||
function sanitizeFailure(value) {
|
||||
const normalized = String(value || "Unknown failure")
|
||||
.replace(/[\u0000-\u001f\u007f]+/g, " ")
|
||||
.replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, (match) => `${match.split(/\s+/, 1)[0]} [REDACTED]`)
|
||||
.replace(/https?:\/\/[^\s"'<>]+/gi, (url) => safeRequestPath(url))
|
||||
.replace(
|
||||
/\b(password|passwd|client[-_]?secret|secret|access[-_]?token|refresh[-_]?token|id[-_]?token|session[-_]?token|token|api[-_]?key|proxy[-_]?key|private[-_]?key|credential|authorization|cookie|session(?:[-_]?id)?)\b(\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;}]*)/gi,
|
||||
(_match, label, separator) => `${label}${separator}[REDACTED]`,
|
||||
)
|
||||
.trim();
|
||||
return (normalized || "Unknown failure").slice(0, 500);
|
||||
}
|
||||
|
||||
function requestMetadata(request) {
|
||||
return {
|
||||
method: String(request.method() || "UNKNOWN").toUpperCase(),
|
||||
path: safeRequestPath(request.url()),
|
||||
resource: String(request.resourceType() || "unknown"),
|
||||
};
|
||||
}
|
||||
|
||||
function isExpectedHttpError(entry) {
|
||||
return entry.method === "GET" && entry.path === "/favicon.ico" && entry.status === 404;
|
||||
}
|
||||
|
||||
function isExpectedConsoleError(message) {
|
||||
const location = typeof message.location === "function" ? message.location() : null;
|
||||
return safeRequestPath(location?.url || "") === "/favicon.ico";
|
||||
}
|
||||
|
||||
function observeBrowserErrors(page) {
|
||||
const diagnostics = {
|
||||
consoleErrors: [],
|
||||
pageErrors: [],
|
||||
requestFailures: [],
|
||||
unexpectedHttpResponses: [],
|
||||
};
|
||||
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error" && !isExpectedConsoleError(message)) {
|
||||
diagnostics.consoleErrors.push({
|
||||
resource: "console",
|
||||
failure: sanitizeFailure(message.text()),
|
||||
});
|
||||
}
|
||||
});
|
||||
page.on("pageerror", (error) => {
|
||||
diagnostics.pageErrors.push({
|
||||
resource: "page",
|
||||
failure: sanitizeFailure(error?.message || error),
|
||||
});
|
||||
});
|
||||
page.on("requestfailed", (request) => {
|
||||
diagnostics.requestFailures.push({
|
||||
...requestMetadata(request),
|
||||
failure: sanitizeFailure(request.failure()?.errorText),
|
||||
});
|
||||
});
|
||||
page.on("response", (response) => {
|
||||
const status = response.status();
|
||||
if (status < 400) {
|
||||
return;
|
||||
}
|
||||
const entry = {
|
||||
...requestMetadata(response.request()),
|
||||
status,
|
||||
failure: `HTTP ${status}`,
|
||||
};
|
||||
if (!isExpectedHttpError(entry)) {
|
||||
diagnostics.unexpectedHttpResponses.push(entry);
|
||||
}
|
||||
});
|
||||
|
||||
return diagnostics;
|
||||
}
|
||||
|
||||
function assertBrowserErrorsAbsent(diagnostics) {
|
||||
const counts = {
|
||||
console: diagnostics.consoleErrors.length,
|
||||
page: diagnostics.pageErrors.length,
|
||||
request: diagnostics.requestFailures.length,
|
||||
http: diagnostics.unexpectedHttpResponses.length,
|
||||
};
|
||||
const total = Object.values(counts).reduce((sum, count) => sum + count, 0);
|
||||
if (total > 0) {
|
||||
throw new Error(
|
||||
`Unexpected browser errors detected: console=${counts.console}, page=${counts.page}, request=${counts.request}, http=${counts.http}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForContract(apiUrl, predicate, timeoutMs) {
|
||||
const deadline = Date.now() + Math.min(timeoutMs, 10000);
|
||||
let contract = null;
|
||||
@@ -156,6 +255,37 @@ async function login(page, args) {
|
||||
await page.waitForURL(/\/workspace(?:\?.*)?$/, { timeout: args.timeoutMs });
|
||||
}
|
||||
|
||||
async function verifyAuthenticatedSession(page, args) {
|
||||
const endpoint = `${args.baseUrl}/api/auth/session`;
|
||||
const result = await page.evaluate(async (sessionEndpoint) => {
|
||||
const response = await fetch(sessionEndpoint, {
|
||||
cache: "no-store",
|
||||
credentials: "same-origin",
|
||||
});
|
||||
const payload = await response.json().catch(() => null);
|
||||
const session = payload && typeof payload.session === "object" ? payload.session : null;
|
||||
return {
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
tenantId: typeof session?.tenant_id === "string" ? session.tenant_id : null,
|
||||
userId: typeof session?.user_id === "string" ? session.user_id : null,
|
||||
};
|
||||
}, endpoint);
|
||||
|
||||
if (!result.ok) {
|
||||
throw new Error(`Authenticated session verification failed: HTTP ${result.status}.`);
|
||||
}
|
||||
if (!result.tenantId || !result.userId) {
|
||||
throw new Error("Authenticated session verification failed: the session endpoint returned no tenant/user identity.");
|
||||
}
|
||||
if (result.tenantId !== args.tenantId || result.userId !== args.userId) {
|
||||
throw new Error(
|
||||
`Authenticated session identity mismatch: expected tenant=${args.tenantId}, user=${args.userId}; got tenant=${result.tenantId}, user=${result.userId}.`,
|
||||
);
|
||||
}
|
||||
return { tenantId: result.tenantId, userId: result.userId };
|
||||
}
|
||||
|
||||
async function createContractFromTemplate(page, args) {
|
||||
await page.goto(`${args.baseUrl}/legal-documents?template=TPL-COOPERATION-FRAMEWORK-001`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
@@ -580,12 +710,7 @@ async function main() {
|
||||
const browser = await chromium.launch({ headless: !args.headed, channel: "chrome" });
|
||||
const page = await browser.newPage({ viewport: { width: 1440, height: 1050 } });
|
||||
page.setDefaultTimeout(args.timeoutMs);
|
||||
const consoleErrors = [];
|
||||
page.on("console", (message) => {
|
||||
if (message.type() === "error" && !message.text().includes("favicon.ico")) {
|
||||
consoleErrors.push(message.text());
|
||||
}
|
||||
});
|
||||
const browserErrors = observeBrowserErrors(page);
|
||||
|
||||
const summary = {
|
||||
status: "fail",
|
||||
@@ -595,16 +720,18 @@ async function main() {
|
||||
outputDir: args.outputDir,
|
||||
contractId: args.contractId || null,
|
||||
login: false,
|
||||
authenticatedSession: null,
|
||||
templateDraft: false,
|
||||
wordDownload: null,
|
||||
review: null,
|
||||
fileRepository: null,
|
||||
lifecycle: null,
|
||||
consoleErrors,
|
||||
...browserErrors,
|
||||
};
|
||||
|
||||
try {
|
||||
await login(page, args);
|
||||
summary.authenticatedSession = await verifyAuthenticatedSession(page, args);
|
||||
summary.login = true;
|
||||
|
||||
if (!summary.contractId) {
|
||||
@@ -625,13 +752,13 @@ async function main() {
|
||||
summary.review = await startAndRunReview(page, args, summary.contractId);
|
||||
summary.fileRepository = await uploadAndVerifyFile(page, args, summary.contractId);
|
||||
summary.lifecycle = await runContractLifecycle(page, args, summary.contractId);
|
||||
summary.consoleErrors = consoleErrors;
|
||||
await page.waitForTimeout(250);
|
||||
assertBrowserErrorsAbsent(browserErrors);
|
||||
summary.status = "pass";
|
||||
} catch (error) {
|
||||
summary.error = error instanceof Error ? error.message : String(error);
|
||||
await page.screenshot({ path: path.join(args.outputDir, "core-regression-failure.png"), fullPage: true }).catch(() => null);
|
||||
} finally {
|
||||
summary.consoleErrors = consoleErrors;
|
||||
fs.writeFileSync(path.join(args.outputDir, "core-user-flow-summary.json"), JSON.stringify(summary, null, 2), "utf8");
|
||||
await browser.close();
|
||||
}
|
||||
@@ -643,7 +770,18 @@ async function main() {
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
assertBrowserErrorsAbsent,
|
||||
isExpectedHttpError,
|
||||
observeBrowserErrors,
|
||||
safeRequestPath,
|
||||
sanitizeFailure,
|
||||
verifyAuthenticatedSession,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -12,6 +13,8 @@ CONTRACT_MIGRATIONS = (
|
||||
AI_MIGRATIONS = (
|
||||
REPO_ROOT / "yuqei-ai-platform" / "services" / "ai-platform-api" / "migrations" / "versions"
|
||||
)
|
||||
POSTGRES_IDENTIFIER_MAX_BYTES = 63
|
||||
MIGRATION_IDENTIFIER_PATTERN = re.compile(r"(?<![A-Za-z0-9_])(?:ix|ux|ck|fk|pk)_[A-Za-z0-9_]+")
|
||||
|
||||
|
||||
def _literal_assignment(module: ast.Module, name: str) -> object:
|
||||
@@ -43,6 +46,18 @@ def _migration_heads(directory: Path) -> set[str]:
|
||||
return revisions - parents
|
||||
|
||||
|
||||
def _explicit_identifier_literals(directory: Path) -> list[tuple[Path, int, str]]:
|
||||
identifiers: list[tuple[Path, int, str]] = []
|
||||
for path in directory.glob("*.py"):
|
||||
module = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(module):
|
||||
if not isinstance(node, ast.Constant) or not isinstance(node.value, str):
|
||||
continue
|
||||
for match in MIGRATION_IDENTIFIER_PATTERN.finditer(node.value):
|
||||
identifiers.append((path, node.lineno, match.group(0)))
|
||||
return identifiers
|
||||
|
||||
|
||||
def _workflow_step(workflow: str, name: str) -> str:
|
||||
marker = f" - name: {name}"
|
||||
start = workflow.index(marker)
|
||||
@@ -83,3 +98,17 @@ def test_ci_migration_steps_track_real_unique_heads_and_use_empty_sqlite() -> No
|
||||
]
|
||||
|
||||
assert '20260826_0073 (head)' not in workflow
|
||||
|
||||
|
||||
def test_explicit_migration_identifiers_fit_postgresql_limit() -> None:
|
||||
offenders = [
|
||||
f"{path.relative_to(REPO_ROOT)}:{line}: {identifier!r} "
|
||||
f"({len(identifier.encode('utf-8'))} bytes)"
|
||||
for directory in (CONTRACT_MIGRATIONS, AI_MIGRATIONS)
|
||||
for path, line, identifier in _explicit_identifier_literals(directory)
|
||||
if len(identifier.encode("utf-8")) > POSTGRES_IDENTIFIER_MAX_BYTES
|
||||
]
|
||||
assert offenders == [], (
|
||||
"Explicit migration identifiers exceed PostgreSQL's 63-byte limit:\n"
|
||||
+ "\n".join(offenders)
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@ import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
from datetime import datetime, timezone
|
||||
@@ -132,6 +133,16 @@ def test_minio_backup_invokes_pinned_mc_directly_without_a_shell(tmp_path: Path)
|
||||
assert all("secret" not in token for token in command)
|
||||
|
||||
|
||||
def test_backup_quiesces_every_long_running_application_writer() -> None:
|
||||
assert backup_run.WRITER_SERVICES == (
|
||||
"contract-api",
|
||||
"contract-review-worker",
|
||||
"contract-playbook-governance-worker",
|
||||
"ai-platform-api",
|
||||
"worker",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("label", "validation_name", "table_name"),
|
||||
[
|
||||
@@ -674,42 +685,150 @@ def test_production_preflight_rejects_smoke_database_public_bind_and_fallback()
|
||||
assert {"production_modes", "internal_port_binding", "database_boundaries"} <= failed
|
||||
|
||||
|
||||
def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None:
|
||||
def _complete_backup_fixture(tmp_path: Path) -> tuple[Path, dict[str, object]]:
|
||||
backup_dir = tmp_path / "backup-20260716T120000Z"
|
||||
backup_dir.mkdir()
|
||||
backup_dir.mkdir(parents=True)
|
||||
archive_path = backup_dir / "contract-files.tar.gz"
|
||||
with tarfile.open(archive_path, mode="w:gz") as archive:
|
||||
content = b"contract"
|
||||
info = tarfile.TarInfo("contract-1/document.txt")
|
||||
info.size = len(content)
|
||||
archive.addfile(info, io.BytesIO(content))
|
||||
dump_path = backup_dir / "postgres-contract.dump"
|
||||
dump_path.write_bytes(b"postgres-dump")
|
||||
for label in ("contract", "ai_platform", "operations"):
|
||||
(backup_dir / f"postgres-{label}.dump").write_bytes(
|
||||
f"postgres-{label}-dump".encode("utf-8")
|
||||
)
|
||||
minio_object = backup_dir / "minio" / "tenant-a" / "object.bin"
|
||||
minio_object.parent.mkdir(parents=True)
|
||||
minio_object.write_bytes(b"minio-object")
|
||||
artifacts = []
|
||||
for path in (archive_path, dump_path):
|
||||
for path in sorted(backup_dir.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
artifacts.append(
|
||||
{
|
||||
"path": path.name,
|
||||
"path": path.relative_to(backup_dir).as_posix(),
|
||||
"size_bytes": path.stat().st_size,
|
||||
"sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
}
|
||||
)
|
||||
minio_records = [
|
||||
item for item in artifacts if str(item["path"]).startswith("minio/")
|
||||
]
|
||||
manifest: dict[str, object] = {
|
||||
"schema_version": "yuqei-production-backup.v1",
|
||||
"backup_id": backup_dir.name,
|
||||
"created_at": "2026-07-16T12:00:00+00:00",
|
||||
"minio_mirror": {
|
||||
"bucket": "ai-platform-uploads",
|
||||
"path": "minio",
|
||||
"object_count": len(minio_records),
|
||||
"total_size_bytes": sum(int(item["size_bytes"]) for item in minio_records),
|
||||
"content_index_sha256": backup_run._minio_content_index(minio_records),
|
||||
},
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
(backup_dir / "manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "yuqei-production-backup.v1",
|
||||
"backup_id": backup_dir.name,
|
||||
"created_at": "2026-07-16T12:00:00+00:00",
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
),
|
||||
json.dumps(manifest),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return backup_dir, manifest
|
||||
|
||||
|
||||
def test_backup_manifest_and_contract_archive_are_verified(tmp_path: Path) -> None:
|
||||
backup_dir, _ = _complete_backup_fixture(tmp_path)
|
||||
|
||||
generated_manifest = backup_run.verify_manifest(backup_dir)
|
||||
manifest, verified = backup_verify.validate_backup(backup_dir)
|
||||
|
||||
assert generated_manifest["backup_id"] == backup_dir.name
|
||||
assert manifest["backup_id"] == backup_dir.name
|
||||
assert len(verified) == 2
|
||||
assert len(verified) == 5
|
||||
|
||||
|
||||
def test_backup_manifest_requires_all_databases_contract_files_and_minio(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
backup_dir, manifest = _complete_backup_fixture(tmp_path)
|
||||
manifest["artifacts"] = [
|
||||
item
|
||||
for item in manifest["artifacts"]
|
||||
if item["path"] != "postgres-operations.dump"
|
||||
]
|
||||
(backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
with pytest.raises(RuntimeError, match="missing required artifacts"):
|
||||
backup_verify.validate_backup(backup_dir)
|
||||
|
||||
legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy")
|
||||
legacy_manifest.pop("minio_mirror")
|
||||
legacy_manifest["artifacts"] = [
|
||||
item
|
||||
for item in legacy_manifest["artifacts"]
|
||||
if not str(item["path"]).startswith("minio/")
|
||||
]
|
||||
shutil.rmtree(legacy_dir / "minio")
|
||||
(legacy_dir / "minio").mkdir()
|
||||
(legacy_dir / "manifest.json").write_text(
|
||||
json.dumps(legacy_manifest), encoding="utf-8"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="no MinIO mirror metadata or recorded MinIO content"):
|
||||
backup_verify.validate_backup(legacy_dir)
|
||||
|
||||
|
||||
def test_backup_manifest_supports_explicit_empty_minio_and_legacy_content(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
empty_dir, empty_manifest = _complete_backup_fixture(tmp_path / "empty")
|
||||
minio_path = empty_dir / "minio" / "tenant-a" / "object.bin"
|
||||
minio_path.unlink()
|
||||
empty_manifest["artifacts"] = [
|
||||
item
|
||||
for item in empty_manifest["artifacts"]
|
||||
if not str(item["path"]).startswith("minio/")
|
||||
]
|
||||
empty_manifest["minio_mirror"] = {
|
||||
"bucket": "ai-platform-uploads",
|
||||
"path": "minio",
|
||||
"object_count": 0,
|
||||
"total_size_bytes": 0,
|
||||
"content_index_sha256": backup_run._minio_content_index([]),
|
||||
}
|
||||
(empty_dir / "manifest.json").write_text(
|
||||
json.dumps(empty_manifest), encoding="utf-8"
|
||||
)
|
||||
backup_verify.validate_backup(empty_dir)
|
||||
|
||||
legacy_dir, legacy_manifest = _complete_backup_fixture(tmp_path / "legacy-content")
|
||||
legacy_manifest.pop("minio_mirror")
|
||||
(legacy_dir / "manifest.json").write_text(
|
||||
json.dumps(legacy_manifest), encoding="utf-8"
|
||||
)
|
||||
backup_verify.validate_backup(legacy_dir)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("member_type", (tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.CHRTYPE))
|
||||
def test_contract_archive_rejects_links_and_special_entries(
|
||||
tmp_path: Path,
|
||||
member_type: bytes,
|
||||
) -> None:
|
||||
backup_dir, manifest = _complete_backup_fixture(tmp_path)
|
||||
archive_path = backup_dir / "contract-files.tar.gz"
|
||||
with tarfile.open(archive_path, mode="w:gz") as archive:
|
||||
info = tarfile.TarInfo("unsafe-entry")
|
||||
info.type = member_type
|
||||
if member_type in {tarfile.SYMTYPE, tarfile.LNKTYPE}:
|
||||
info.linkname = "outside"
|
||||
archive.addfile(info)
|
||||
for item in manifest["artifacts"]:
|
||||
if item["path"] == "contract-files.tar.gz":
|
||||
item["size_bytes"] = archive_path.stat().st_size
|
||||
item["sha256"] = hashlib.sha256(archive_path.read_bytes()).hexdigest()
|
||||
(backup_dir / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with pytest.raises(RuntimeError, match="link or special entry"):
|
||||
backup_run.verify_manifest(backup_dir)
|
||||
with pytest.raises(RuntimeError, match="link or special entry"):
|
||||
backup_verify.validate_backup(backup_dir)
|
||||
|
||||
|
||||
def test_provider_acceptance_separates_technical_success_from_legal_approval() -> None:
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
const assert = require("node:assert/strict");
|
||||
const { EventEmitter } = require("node:events");
|
||||
const test = require("node:test");
|
||||
|
||||
const {
|
||||
assertBrowserErrorsAbsent,
|
||||
observeBrowserErrors,
|
||||
safeRequestPath,
|
||||
sanitizeFailure,
|
||||
verifyAuthenticatedSession,
|
||||
} = require("../scripts/verify-core-user-flows.cjs");
|
||||
|
||||
test("safeRequestPath removes origins, queries, and fragments", () => {
|
||||
assert.equal(safeRequestPath("https://example.test/contracts/123?token=secret#fragment"), "/contracts/123");
|
||||
assert.equal(safeRequestPath("not a URL"), "<unparseable>");
|
||||
});
|
||||
|
||||
test("sanitizeFailure redacts credentials and URL queries", () => {
|
||||
const sanitized = sanitizeFailure(
|
||||
"Bearer top-secret-token failed at https://example.test/api/jobs?access_token=secret password=hidden access_token=also-hidden",
|
||||
);
|
||||
|
||||
assert.equal(sanitized.includes("top-secret-token"), false);
|
||||
assert.equal(sanitized.includes("access_token=secret"), false);
|
||||
assert.equal(sanitized.includes("access_token=also-hidden"), false);
|
||||
assert.equal(sanitized.includes("password=hidden"), false);
|
||||
assert.match(sanitized, /Bearer \[REDACTED\]/);
|
||||
assert.match(sanitized, /\/api\/jobs/);
|
||||
assert.match(sanitized, /password=\[REDACTED\]/);
|
||||
});
|
||||
|
||||
test("browser diagnostics retain only safe metadata and gate every unexpected class", () => {
|
||||
const page = new EventEmitter();
|
||||
const diagnostics = observeBrowserErrors(page);
|
||||
const request = {
|
||||
method: () => "get",
|
||||
url: () => "https://example.test/api/contracts?token=secret",
|
||||
resourceType: () => "fetch",
|
||||
failure: () => ({ errorText: "net::ERR_FAILED token=secret" }),
|
||||
};
|
||||
|
||||
page.emit("console", {
|
||||
type: () => "error",
|
||||
text: () => "Bearer secret-value was rejected",
|
||||
location: () => ({ url: "https://example.test/app" }),
|
||||
});
|
||||
page.emit("pageerror", new Error("password=secret"));
|
||||
page.emit("requestfailed", request);
|
||||
page.emit("response", {
|
||||
status: () => 503,
|
||||
request: () => request,
|
||||
});
|
||||
page.emit("response", {
|
||||
status: () => 404,
|
||||
request: () => ({
|
||||
method: () => "GET",
|
||||
url: () => "https://example.test/favicon.ico?cache=1",
|
||||
resourceType: () => "image",
|
||||
}),
|
||||
});
|
||||
|
||||
assert.deepEqual(diagnostics.consoleErrors, [{ resource: "console", failure: "Bearer [REDACTED] was rejected" }]);
|
||||
assert.deepEqual(diagnostics.pageErrors, [{ resource: "page", failure: "password=[REDACTED]" }]);
|
||||
assert.deepEqual(diagnostics.requestFailures, [{
|
||||
method: "GET",
|
||||
path: "/api/contracts",
|
||||
resource: "fetch",
|
||||
failure: "net::ERR_FAILED token=[REDACTED]",
|
||||
}]);
|
||||
assert.deepEqual(diagnostics.unexpectedHttpResponses, [{
|
||||
method: "GET",
|
||||
path: "/api/contracts",
|
||||
resource: "fetch",
|
||||
status: 503,
|
||||
failure: "HTTP 503",
|
||||
}]);
|
||||
assert.throws(() => assertBrowserErrorsAbsent(diagnostics), /console=1, page=1, request=1, http=1/);
|
||||
});
|
||||
|
||||
test("browser diagnostics accept an entirely clean run", () => {
|
||||
assert.doesNotThrow(() => assertBrowserErrorsAbsent({
|
||||
consoleErrors: [],
|
||||
pageErrors: [],
|
||||
requestFailures: [],
|
||||
unexpectedHttpResponses: [],
|
||||
}));
|
||||
});
|
||||
|
||||
test("authenticated session must match the supplied tenant and user", async () => {
|
||||
const page = {
|
||||
evaluate: async (_callback, endpoint) => {
|
||||
assert.equal(endpoint, "https://example.test/api/auth/session");
|
||||
return { ok: true, status: 200, tenantId: "tenant-a", userId: "user-a" };
|
||||
},
|
||||
};
|
||||
const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" };
|
||||
|
||||
assert.deepEqual(await verifyAuthenticatedSession(page, args), { tenantId: "tenant-a", userId: "user-a" });
|
||||
});
|
||||
|
||||
test("authenticated session mismatch fails before the journey proceeds", async () => {
|
||||
const page = {
|
||||
evaluate: async () => ({ ok: true, status: 200, tenantId: "tenant-a", userId: "other-user" }),
|
||||
};
|
||||
const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" };
|
||||
|
||||
await assert.rejects(
|
||||
verifyAuthenticatedSession(page, args),
|
||||
/Authenticated session identity mismatch/,
|
||||
);
|
||||
});
|
||||
|
||||
test("unavailable or malformed session endpoint fails clearly", async () => {
|
||||
const args = { baseUrl: "https://example.test", tenantId: "tenant-a", userId: "user-a" };
|
||||
|
||||
await assert.rejects(
|
||||
verifyAuthenticatedSession({
|
||||
evaluate: async () => ({ ok: false, status: 404, tenantId: null, userId: null }),
|
||||
}, args),
|
||||
/Authenticated session verification failed: HTTP 404/,
|
||||
);
|
||||
await assert.rejects(
|
||||
verifyAuthenticatedSession({
|
||||
evaluate: async () => ({ ok: true, status: 200, tenantId: null, userId: null }),
|
||||
}, args),
|
||||
/session endpoint returned no tenant\/user identity/,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user