diff --git a/project-docs/运维手册-2026-05-01.md b/project-docs/运维手册-2026-05-01.md index bc3784a4..65611b41 100644 --- a/project-docs/运维手册-2026-05-01.md +++ b/project-docs/运维手册-2026-05-01.md @@ -70,7 +70,33 @@ powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\rotate-admin-pas ## 四、备份恢复口径 -阶段 5 下一步将新增 `scripts/run-backup-drill.ps1`。当前原则先固定： +执行备份与恢复演练： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-backup-drill.ps1 +``` + +默认远端备份目录： + +```text +/opt/ai-contract-platform/backups/phase5// +``` + +默认本地产物目录： + +```text +project-docs/qa-artifacts/phase5-backup-drill// +``` + +当前备份范围： + +1. PostgreSQL custom-format dump：`postgres.dump` +2. PostgreSQL 隔离临时库恢复验证 +3. 部署配置、迁移、脚本和文档压缩包：`project-config-docs-scripts.tgz` +4. MinIO `contracts` bucket 对象清单和镜像 +5. `manifest.tsv` 与 `sha256sums.txt` + +执行原则： 1. 生产备份可以读取生产数据。 2. 恢复验证必须在隔离目录或临时数据库执行。 diff --git a/project-docs/闭环交付与深度开发方案-2026-05-01.md b/project-docs/闭环交付与深度开发方案-2026-05-01.md index 126bbed6..14961cd8 100644 --- a/project-docs/闭环交付与深度开发方案-2026-05-01.md +++ b/project-docs/闭环交付与深度开发方案-2026-05-01.md @@ -18,7 +18,7 @@ 阶段 1 已完成源码闭环与生产入口闭环，生产无人 E2E `18/18` 通过。阶段 2 已完成 Review 重试、Knowledge Base 文档重试、Review -> Chat -> Contract 上下文联动、首页交付工作台和阶段 2 自动化无人闭环，本地与生产 E2E 均 `7/7` 通过。阶段 3 已完成系统 readiness、试点交付准备页、默认账号说明、演示样本数据、依赖健康门禁和阶段 3 自动化无人闭环，本地与生产 E2E 均 `5/5` 通过。 -当前后续重点为“生产运行、安全、监控、备份与依赖治理”：默认密码轮换已完成，阶段 5 Step 1 生产健康巡检自动化已完成并 PASS，下一步进入备份恢复演练。 +当前后续重点为“生产运行、安全、监控、备份与依赖治理”：默认密码轮换已完成，阶段 5 Step 1 生产健康巡检自动化和 Step 2 备份恢复演练均已 PASS，下一步进入依赖治理专项。 --- @@ -210,4 +210,4 @@ AI合同平台当前已经不是从零建设阶段。登录、合同审查、Rev ## 六、当前一句话结论 -AI合同平台阶段 1、阶段 2 和阶段 3 均已完成闭环交付验证，阶段 4 已完成默认管理员密码轮换闭环，阶段 5 已启动并完成生产健康巡检自动化，下一步进入备份恢复演练。 +AI合同平台阶段 1、阶段 2 和阶段 3 均已完成闭环交付验证，阶段 4 已完成默认管理员密码轮换闭环，阶段 5 已启动并完成生产健康巡检与备份恢复演练，下一步进入依赖治理专项。 diff --git a/project-docs/阶段5-生产运行安全监控备份与依赖治理执行记录-2026-05-01.md b/project-docs/阶段5-生产运行安全监控备份与依赖治理执行记录-2026-05-01.md index a9de7221..b306567c 100644 --- a/project-docs/阶段5-生产运行安全监控备份与依赖治理执行记录-2026-05-01.md +++ b/project-docs/阶段5-生产运行安全监控备份与依赖治理执行记录-2026-05-01.md @@ -11,12 +11,12 @@ | gstack 安装校验 | PASS | 自动化执行前已返回 `GSTACK_OK` | | 代码提交与推送 | PASS | 已推送到 `origin/main`，远端为 `192.168.3.90` 仓库 | | Step 1：生产健康巡检自动化 | PASS | `scripts/run-production-health-check.ps1` 已实现并完成生产巡检 | -| Step 2：备份与恢复演练 | PENDING | 下一步进入 | -| Step 3：依赖治理专项 | PENDING | 待备份基线后处理 | +| Step 2：备份与恢复演练 | PASS | `scripts/run-backup-drill.ps1` 已实现并完成生产备份/隔离恢复验证 | +| Step 3：依赖治理专项 | PENDING | 下一步进入 | | Step 4：安全基线检查 | PENDING | 将复用 Step 1 readiness 与默认密码验证 | | Step 5：生产回归闭环 | PENDING | 待运维加固动作后复跑阶段 1/2/3 | -阶段 5 Step 1 当前判定：PASS。 +阶段 5 当前判定：Step 1、Step 2 均已 PASS。 --- @@ -86,12 +86,60 @@ default-password-login.json ## 五、下一步 -阶段 5 下一步进入 Step 2：备份与恢复演练。 +阶段 5 下一步进入 Step 3：依赖治理专项。 优先交付： -1. `scripts/run-backup-drill.ps1` -2. PostgreSQL 备份文件生成和校验 -3. 部署配置、脚本、文档备份 -4. 隔离恢复可用性验证 -5. 备份恢复演练记录 +1. `npm audit --json` 风险记录 +2. high/critical 漏洞清零验证 +3. moderate 漏洞修复或延期说明 +4. 后端 `pip check` 依赖一致性验证 +5. 依赖治理执行记录 + +--- + +## 六、Step 2 备份与恢复演练 + +新增脚本： + +```powershell +scripts\run-backup-drill.ps1 +``` + +本次演练产物： + +| 项目 | 路径 | +|---|---| +| 本地产物目录 | `project-docs/qa-artifacts/phase5-backup-drill/2026-05-01T08-40-59/` | +| 远端备份目录 | `/opt/ai-contract-platform/backups/phase5/2026-05-01T08-40-59` | +| 数据库备份 | `/opt/ai-contract-platform/backups/phase5/2026-05-01T08-40-59/postgres.dump` | +| 配置/文档/脚本备份 | `/opt/ai-contract-platform/backups/phase5/2026-05-01T08-40-59/project-config-docs-scripts.tgz` | +| 对象存储镜像 | `/opt/ai-contract-platform/backups/phase5/2026-05-01T08-40-59/minio-contracts` | + +本次验证结果： + +| 检查项 | 结果 | 说明 | +|---|---|---| +| 远端备份工具预检 | PASS | `docker compose`、`pg_dump`、`pg_restore`、`tar`、`mc` 可用 | +| PostgreSQL 备份 | PASS | 生成 custom-format dump | +| PostgreSQL 隔离恢复 | PASS | 恢复到临时库 `restore_check_2026_05_01T08_40_59` 并验证 schema 后删除 | +| 配置/文档/脚本备份 | PASS | 生成 `project-config-docs-scripts.tgz` | +| MinIO 对象存储备份 | PASS | 生成对象清单并镜像 `contracts` bucket，当前对象数 `1` | +| 备份清单与校验和 | PASS | 生成 `manifest.tsv` 与 `sha256sums.txt` | + +远端备份清单： + +```text +./manifest.tsv 0 +./minio-objects.txt 114 +./postgres.dump 127859 +./project-config-docs-scripts.tgz 159067 +./restore-schema.sql 18284 +``` + +核心校验和： + +```text +f6598a0d3b98fc1b5e6520955b1f54b20617bad61bf2628795b6a7b2ae6c161e postgres.dump +8c831b371aac6b3c4c98074eb6df03e78ef4131c14ef8dd1cd7c3fd8189d033f project-config-docs-scripts.tgz +``` diff --git a/project-docs/阶段5-生产运行安全监控备份与依赖治理方案-2026-05-01.md b/project-docs/阶段5-生产运行安全监控备份与依赖治理方案-2026-05-01.md index 29598b5d..efc40a8c 100644 --- a/project-docs/阶段5-生产运行安全监控备份与依赖治理方案-2026-05-01.md +++ b/project-docs/阶段5-生产运行安全监控备份与依赖治理方案-2026-05-01.md @@ -177,7 +177,10 @@ $env:AI_CONTRACT_ADMIN_PASSWORD = '<管理员实际密码>' - Step 1 已完成：`scripts/run-production-health-check.ps1` - 生产巡检结果：PASS - 巡检产物：`project-docs/qa-artifacts/phase5-production-health/2026-05-01T08-32-50/` -- 下一步：Step 2 备份与恢复演练 +- Step 2 已完成：`scripts/run-backup-drill.ps1` +- 备份恢复演练结果：PASS +- 备份演练产物：`project-docs/qa-artifacts/phase5-backup-drill/2026-05-01T08-40-59/` +- 下一步：Step 3 依赖治理专项 --- diff --git a/scripts/run-backup-drill.ps1 b/scripts/run-backup-drill.ps1 new file mode 100644 index 00000000..c37be4e4 --- /dev/null +++ b/scripts/run-backup-drill.ps1 @@ -0,0 +1,303 @@ +param( + [string]$ServerHost = "192.168.3.90", + [string]$SshUser = "root", + [string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519", + [string]$RemoteProjectPath = "/opt/ai-contract-platform", + [string]$ComposeFile = "docker-compose.server.yml", + [string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase5", + [string]$DatabaseName = "ai_contract_platform", + [string]$PostgresUser = "postgres", + [string]$MinioEndpoint = "http://192.168.3.90:9000", + [string]$MinioAccessKey = "minioadmin", + [string]$MinioSecretKey = "minioadmin", + [string]$MinioBucket = "contracts", + [string]$ArtifactRoot = "" +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss" +$safeStamp = $timestamp -replace '[^0-9A-Za-z]', '_' +$restoreDbName = "restore_check_$safeStamp" +if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase5-backup-drill" +} +$artifactDir = Join-Path $ArtifactRoot $timestamp +$remoteBackupDir = "$RemoteBackupRoot/$timestamp" +$summaryJson = Join-Path $artifactDir "backup-drill-summary.json" +$summaryMd = Join-Path $artifactDir "backup-drill-summary.md" + +New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null + +if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { + throw "gstack is required for all AI-assisted work in this repo." +} +if (-not (Test-Path $SshKeyPath)) { + throw "SSH key not found: $SshKeyPath" +} + +$checks = New-Object System.Collections.Generic.List[object] + +function Add-Check { + param( + [string]$Key, + [string]$Label, + [ValidateSet("pass", "fail", "warn")] + [string]$Status, + [string]$Detail, + [bool]$Required = $true, + [hashtable]$Meta = @{} + ) + + $checks.Add([ordered]@{ + key = $Key + label = $Label + status = $Status + required = $Required + detail = $Detail + meta = $Meta + }) | Out-Null +} + +function Save-Json { + param( + [string]$Path, + [object]$Value + ) + $Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path +} + +function Invoke-Remote { + param( + [string]$Name, + [string]$Command + ) + + $target = "${SshUser}@${ServerHost}" + $logPath = Join-Path $artifactDir "$Name.log" + $args = @( + "-i", $SshKeyPath, + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + $target, + $Command + ) + $previousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $output = & ssh @args 2>&1 + $exitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + $outputText = ($output -join [Environment]::NewLine) + $outputText | Set-Content -Encoding UTF8 -Path $logPath + return [ordered]@{ + exit_code = $exitCode + output = $outputText + log = $logPath + } +} + +function Invoke-ScpFromRemote { + param( + [string]$RemotePath, + [string]$LocalPath + ) + + $target = "${SshUser}@${ServerHost}:$RemotePath" + $args = @( + "-i", $SshKeyPath, + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + $target, + $LocalPath + ) + $previousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $output = & scp @args 2>&1 + $exitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + return [ordered]@{ + exit_code = $exitCode + output = ($output -join [Environment]::NewLine) + } +} + +$preflightCommand = "set -e; mkdir -p '$remoteBackupDir'; cd '$RemoteProjectPath'; docker compose -f '$ComposeFile' ps; docker compose -f '$ComposeFile' exec -T postgres pg_dump --version; docker compose -f '$ComposeFile' exec -T postgres pg_restore --version; command -v tar; command -v mc" +$preflight = Invoke-Remote -Name "01-preflight" -Command $preflightCommand +if ($preflight.exit_code -eq 0) { + Add-Check -Key "remote_preflight" -Label "Remote backup prerequisites" -Status "pass" -Detail "Docker compose, pg_dump, pg_restore, tar and mc are available" -Meta @{ + log = $preflight.log + } +} else { + Add-Check -Key "remote_preflight" -Label "Remote backup prerequisites" -Status "fail" -Detail "Remote prerequisites failed" -Meta @{ + log = $preflight.log + } +} + +$dumpPath = "$remoteBackupDir/postgres.dump" +$dumpCommand = "set -e; cd '$RemoteProjectPath'; docker compose -f '$ComposeFile' exec -T postgres pg_dump -U '$PostgresUser' -d '$DatabaseName' -Fc > '$dumpPath'; test -s '$dumpPath'; ls -lh '$dumpPath'" +$dump = Invoke-Remote -Name "02-postgres-dump" -Command $dumpCommand +if ($dump.exit_code -eq 0) { + Add-Check -Key "postgres_dump" -Label "PostgreSQL backup" -Status "pass" -Detail "Custom-format PostgreSQL dump created on remote host" -Meta @{ + remote_path = $dumpPath + log = $dump.log + } +} else { + Add-Check -Key "postgres_dump" -Label "PostgreSQL backup" -Status "fail" -Detail "PostgreSQL dump failed" -Meta @{ + remote_path = $dumpPath + log = $dump.log + } +} + +$restoreCommand = @" +set -e +cd '$RemoteProjectPath' +restore_db='$restoreDbName' +dump_path='$dumpPath' +trap "docker compose -f '$ComposeFile' exec -T postgres dropdb -U '$PostgresUser' --if-exists \"`$restore_db\" >/dev/null 2>&1 || true" EXIT +docker compose -f '$ComposeFile' exec -T postgres dropdb -U '$PostgresUser' --if-exists "`$restore_db" +docker compose -f '$ComposeFile' exec -T postgres createdb -U '$PostgresUser' "`$restore_db" +cat "`$dump_path" | docker compose -f '$ComposeFile' exec -T postgres pg_restore -U '$PostgresUser' -d "`$restore_db" --no-owner +docker compose -f '$ComposeFile' exec -T postgres pg_dump -U '$PostgresUser' -d "`$restore_db" -s > '$remoteBackupDir/restore-schema.sql' +grep -q "CREATE TABLE" '$remoteBackupDir/restore-schema.sql' +docker compose -f '$ComposeFile' exec -T postgres dropdb -U '$PostgresUser' "`$restore_db" +trap - EXIT +echo "restore_schema_verified=true" +"@ -replace "`r?`n", "; " +$restore = Invoke-Remote -Name "03-postgres-restore-check" -Command $restoreCommand +if ($restore.exit_code -eq 0) { + Add-Check -Key "postgres_restore_check" -Label "PostgreSQL restore drill" -Status "pass" -Detail "Dump restored into isolated temporary database and schema was verified" -Meta @{ + restore_db = $restoreDbName + log = $restore.log + } +} else { + Add-Check -Key "postgres_restore_check" -Label "PostgreSQL restore drill" -Status "fail" -Detail "Restore check failed; temporary database name is recorded for cleanup review" -Meta @{ + restore_db = $restoreDbName + log = $restore.log + } +} + +$configArchivePath = "$remoteBackupDir/project-config-docs-scripts.tgz" +$configCommand = "set -e; cd '$RemoteProjectPath'; tar --exclude='project-docs/qa-artifacts' --exclude='backups' -czf '$configArchivePath' docker-compose.server.yml backend/Dockerfile backend/requirements.txt backend/alembic.ini backend/migrations frontend/Dockerfile frontend/package.json frontend/package-lock.json scripts project-docs; test -s '$configArchivePath'; ls -lh '$configArchivePath'" +$configBackup = Invoke-Remote -Name "04-config-docs-scripts-backup" -Command $configCommand +if ($configBackup.exit_code -eq 0) { + Add-Check -Key "config_docs_scripts_backup" -Label "Config docs scripts backup" -Status "pass" -Detail "Deployment config, docs and scripts archive created" -Meta @{ + remote_path = $configArchivePath + log = $configBackup.log + } +} else { + Add-Check -Key "config_docs_scripts_backup" -Label "Config docs scripts backup" -Status "fail" -Detail "Config/docs/scripts backup failed" -Meta @{ + remote_path = $configArchivePath + log = $configBackup.log + } +} + +$minioDir = "$remoteBackupDir/minio-$MinioBucket" +$minioCommand = @" +set -e +mkdir -p '$minioDir' +mc alias set phase5 '$MinioEndpoint' '$MinioAccessKey' '$MinioSecretKey' >/dev/null +mc ls --recursive "phase5/$MinioBucket" > '$remoteBackupDir/minio-objects.txt' +mc mirror --overwrite "phase5/$MinioBucket" '$minioDir' +object_count=`$(find '$minioDir' -type f | wc -l) +echo "object_count=`$object_count" +du -sh '$minioDir' || true +"@ -replace "`r?`n", "; " +$minioBackup = Invoke-Remote -Name "05-minio-backup" -Command $minioCommand +if ($minioBackup.exit_code -eq 0) { + Add-Check -Key "object_storage_backup" -Label "Object storage backup" -Status "pass" -Detail "MinIO object list and mirror completed" -Meta @{ + remote_path = $minioDir + log = $minioBackup.log + } +} else { + Add-Check -Key "object_storage_backup" -Label "Object storage backup" -Status "fail" -Detail "MinIO object list or mirror failed" -Meta @{ + remote_path = $minioDir + log = $minioBackup.log + } +} + +$manifestCommand = "set -e; cd '$remoteBackupDir'; find . -type f -maxdepth 4 -printf '%p`t%s`n' | sort > manifest.tsv; sha256sum postgres.dump project-config-docs-scripts.tgz > sha256sums.txt; cat manifest.tsv; cat sha256sums.txt" +$manifest = Invoke-Remote -Name "06-manifest" -Command $manifestCommand +if ($manifest.exit_code -eq 0) { + Add-Check -Key "backup_manifest" -Label "Backup manifest" -Status "pass" -Detail "Manifest and sha256 sums generated" -Meta @{ + remote_path = "$remoteBackupDir/manifest.tsv" + log = $manifest.log + } +} else { + Add-Check -Key "backup_manifest" -Label "Backup manifest" -Status "fail" -Detail "Manifest generation failed" -Meta @{ + remote_path = "$remoteBackupDir/manifest.tsv" + log = $manifest.log + } +} + +foreach ($remoteFile in @("manifest.tsv", "sha256sums.txt", "minio-objects.txt")) { + $localFile = Join-Path $artifactDir $remoteFile + $scpResult = Invoke-ScpFromRemote -RemotePath "$remoteBackupDir/$remoteFile" -LocalPath $localFile + $scpLogPath = Join-Path $artifactDir "scp-$remoteFile.log" + $scpResult.output | Set-Content -Encoding UTF8 -Path $scpLogPath +} + +$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) +$warningsList = @($checks | Where-Object { $_.status -eq "warn" }) +$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" } + +$summary = [ordered]@{ + name = "phase5-backup-drill" + status = $overallStatus + generated_at = (Get-Date).ToUniversalTime().ToString("o") + artifact_dir = $artifactDir + remote_backup_dir = $remoteBackupDir + restore_database = $restoreDbName + targets = [ordered]@{ + server_host = $ServerHost + remote_project_path = $RemoteProjectPath + compose_file = $ComposeFile + database = $DatabaseName + minio_endpoint = $MinioEndpoint + minio_bucket = $MinioBucket + } + checks = $checks + totals = [ordered]@{ + total = $checks.Count + failed_required = $failedRequired.Count + warnings = $warningsList.Count + } +} +Save-Json -Path $summaryJson -Value $summary + +$markdown = New-Object System.Collections.Generic.List[string] +$markdown.Add("# Phase 5 Backup Drill") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("- Status: $overallStatus") | Out-Null +$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null +$markdown.Add("- Remote backup dir: $remoteBackupDir") | Out-Null +$markdown.Add("- Local artifact dir: $artifactDir") | Out-Null +$markdown.Add("- Restore database: $restoreDbName") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("| Check | Required | Status | Detail |") | Out-Null +$markdown.Add("|---|---:|---|---|") | Out-Null +foreach ($check in $checks) { + $markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null +} +$markdown.Add("") | Out-Null +$markdown.Add("Local artifacts:") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add('- `backup-drill-summary.json`') | Out-Null +$markdown.Add('- `manifest.tsv`') | Out-Null +$markdown.Add('- `sha256sums.txt`') | Out-Null +$markdown.Add('- `minio-objects.txt`') | Out-Null +$markdown.Add('- `*.log`') | Out-Null +$markdown | Set-Content -Encoding UTF8 -Path $summaryMd + +Write-Host "PHASE5_BACKUP_DRILL_$overallStatus $artifactDir" +if ($overallStatus -eq "FAIL") { + throw "Phase 5 backup drill failed. See $summaryMd" +}