|
|
|
@@ -0,0 +1,303 @@
|
|
|
|
|
param(
|
|
|
|
|
[string]$ServerHost = "192.168.3.90",
|
|
|
|
|
[string]$SshUser = "root",
|
|
|
|
|
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
|
|
|
|
|
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
|
|
|
|
|
[string]$ComposeFile = "docker-compose.server.yml",
|
|
|
|
|
[string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase5",
|
|
|
|
|
[string]$DatabaseName = "ai_contract_platform",
|
|
|
|
|
[string]$PostgresUser = "postgres",
|
|
|
|
|
[string]$MinioEndpoint = "http://192.168.3.90:9000",
|
|
|
|
|
[string]$MinioAccessKey = "minioadmin",
|
|
|
|
|
[string]$MinioSecretKey = "minioadmin",
|
|
|
|
|
[string]$MinioBucket = "contracts",
|
|
|
|
|
[string]$ArtifactRoot = ""
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
|
|
|
|
|
|
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
|
|
|
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss"
|
|
|
|
|
$safeStamp = $timestamp -replace '[^0-9A-Za-z]', '_'
|
|
|
|
|
$restoreDbName = "restore_check_$safeStamp"
|
|
|
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
|
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase5-backup-drill"
|
|
|
|
|
}
|
|
|
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
|
|
|
$remoteBackupDir = "$RemoteBackupRoot/$timestamp"
|
|
|
|
|
$summaryJson = Join-Path $artifactDir "backup-drill-summary.json"
|
|
|
|
|
$summaryMd = Join-Path $artifactDir "backup-drill-summary.md"
|
|
|
|
|
|
|
|
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
|
|
|
|
|
|
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
|
|
|
|
throw "gstack is required for all AI-assisted work in this repo."
|
|
|
|
|
}
|
|
|
|
|
if (-not (Test-Path $SshKeyPath)) {
|
|
|
|
|
throw "SSH key not found: $SshKeyPath"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
|
|
|
|
|
|
|
|
function Add-Check {
|
|
|
|
|
param(
|
|
|
|
|
[string]$Key,
|
|
|
|
|
[string]$Label,
|
|
|
|
|
[ValidateSet("pass", "fail", "warn")]
|
|
|
|
|
[string]$Status,
|
|
|
|
|
[string]$Detail,
|
|
|
|
|
[bool]$Required = $true,
|
|
|
|
|
[hashtable]$Meta = @{}
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$checks.Add([ordered]@{
|
|
|
|
|
key = $Key
|
|
|
|
|
label = $Label
|
|
|
|
|
status = $Status
|
|
|
|
|
required = $Required
|
|
|
|
|
detail = $Detail
|
|
|
|
|
meta = $Meta
|
|
|
|
|
}) | Out-Null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Save-Json {
|
|
|
|
|
param(
|
|
|
|
|
[string]$Path,
|
|
|
|
|
[object]$Value
|
|
|
|
|
)
|
|
|
|
|
$Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Invoke-Remote {
|
|
|
|
|
param(
|
|
|
|
|
[string]$Name,
|
|
|
|
|
[string]$Command
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$target = "${SshUser}@${ServerHost}"
|
|
|
|
|
$logPath = Join-Path $artifactDir "$Name.log"
|
|
|
|
|
$args = @(
|
|
|
|
|
"-i", $SshKeyPath,
|
|
|
|
|
"-o", "BatchMode=yes",
|
|
|
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
|
|
|
"-o", "ConnectTimeout=10",
|
|
|
|
|
$target,
|
|
|
|
|
$Command
|
|
|
|
|
)
|
|
|
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
|
|
|
$ErrorActionPreference = "Continue"
|
|
|
|
|
try {
|
|
|
|
|
$output = & ssh @args 2>&1
|
|
|
|
|
$exitCode = $LASTEXITCODE
|
|
|
|
|
} finally {
|
|
|
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
|
|
|
}
|
|
|
|
|
$outputText = ($output -join [Environment]::NewLine)
|
|
|
|
|
$outputText | Set-Content -Encoding UTF8 -Path $logPath
|
|
|
|
|
return [ordered]@{
|
|
|
|
|
exit_code = $exitCode
|
|
|
|
|
output = $outputText
|
|
|
|
|
log = $logPath
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function Invoke-ScpFromRemote {
|
|
|
|
|
param(
|
|
|
|
|
[string]$RemotePath,
|
|
|
|
|
[string]$LocalPath
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
$target = "${SshUser}@${ServerHost}:$RemotePath"
|
|
|
|
|
$args = @(
|
|
|
|
|
"-i", $SshKeyPath,
|
|
|
|
|
"-o", "BatchMode=yes",
|
|
|
|
|
"-o", "StrictHostKeyChecking=accept-new",
|
|
|
|
|
"-o", "ConnectTimeout=10",
|
|
|
|
|
$target,
|
|
|
|
|
$LocalPath
|
|
|
|
|
)
|
|
|
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
|
|
|
$ErrorActionPreference = "Continue"
|
|
|
|
|
try {
|
|
|
|
|
$output = & scp @args 2>&1
|
|
|
|
|
$exitCode = $LASTEXITCODE
|
|
|
|
|
} finally {
|
|
|
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
|
|
|
}
|
|
|
|
|
return [ordered]@{
|
|
|
|
|
exit_code = $exitCode
|
|
|
|
|
output = ($output -join [Environment]::NewLine)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$preflightCommand = "set -e; mkdir -p '$remoteBackupDir'; cd '$RemoteProjectPath'; docker compose -f '$ComposeFile' ps; docker compose -f '$ComposeFile' exec -T postgres pg_dump --version; docker compose -f '$ComposeFile' exec -T postgres pg_restore --version; command -v tar; command -v mc"
|
|
|
|
|
$preflight = Invoke-Remote -Name "01-preflight" -Command $preflightCommand
|
|
|
|
|
if ($preflight.exit_code -eq 0) {
|
|
|
|
|
Add-Check -Key "remote_preflight" -Label "Remote backup prerequisites" -Status "pass" -Detail "Docker compose, pg_dump, pg_restore, tar and mc are available" -Meta @{
|
|
|
|
|
log = $preflight.log
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "remote_preflight" -Label "Remote backup prerequisites" -Status "fail" -Detail "Remote prerequisites failed" -Meta @{
|
|
|
|
|
log = $preflight.log
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$dumpPath = "$remoteBackupDir/postgres.dump"
|
|
|
|
|
$dumpCommand = "set -e; cd '$RemoteProjectPath'; docker compose -f '$ComposeFile' exec -T postgres pg_dump -U '$PostgresUser' -d '$DatabaseName' -Fc > '$dumpPath'; test -s '$dumpPath'; ls -lh '$dumpPath'"
|
|
|
|
|
$dump = Invoke-Remote -Name "02-postgres-dump" -Command $dumpCommand
|
|
|
|
|
if ($dump.exit_code -eq 0) {
|
|
|
|
|
Add-Check -Key "postgres_dump" -Label "PostgreSQL backup" -Status "pass" -Detail "Custom-format PostgreSQL dump created on remote host" -Meta @{
|
|
|
|
|
remote_path = $dumpPath
|
|
|
|
|
log = $dump.log
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "postgres_dump" -Label "PostgreSQL backup" -Status "fail" -Detail "PostgreSQL dump failed" -Meta @{
|
|
|
|
|
remote_path = $dumpPath
|
|
|
|
|
log = $dump.log
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$restoreCommand = @"
|
|
|
|
|
set -e
|
|
|
|
|
cd '$RemoteProjectPath'
|
|
|
|
|
restore_db='$restoreDbName'
|
|
|
|
|
dump_path='$dumpPath'
|
|
|
|
|
trap "docker compose -f '$ComposeFile' exec -T postgres dropdb -U '$PostgresUser' --if-exists \"`$restore_db\" >/dev/null 2>&1 || true" EXIT
|
|
|
|
|
docker compose -f '$ComposeFile' exec -T postgres dropdb -U '$PostgresUser' --if-exists "`$restore_db"
|
|
|
|
|
docker compose -f '$ComposeFile' exec -T postgres createdb -U '$PostgresUser' "`$restore_db"
|
|
|
|
|
cat "`$dump_path" | docker compose -f '$ComposeFile' exec -T postgres pg_restore -U '$PostgresUser' -d "`$restore_db" --no-owner
|
|
|
|
|
docker compose -f '$ComposeFile' exec -T postgres pg_dump -U '$PostgresUser' -d "`$restore_db" -s > '$remoteBackupDir/restore-schema.sql'
|
|
|
|
|
grep -q "CREATE TABLE" '$remoteBackupDir/restore-schema.sql'
|
|
|
|
|
docker compose -f '$ComposeFile' exec -T postgres dropdb -U '$PostgresUser' "`$restore_db"
|
|
|
|
|
trap - EXIT
|
|
|
|
|
echo "restore_schema_verified=true"
|
|
|
|
|
"@ -replace "`r?`n", "; "
|
|
|
|
|
$restore = Invoke-Remote -Name "03-postgres-restore-check" -Command $restoreCommand
|
|
|
|
|
if ($restore.exit_code -eq 0) {
|
|
|
|
|
Add-Check -Key "postgres_restore_check" -Label "PostgreSQL restore drill" -Status "pass" -Detail "Dump restored into isolated temporary database and schema was verified" -Meta @{
|
|
|
|
|
restore_db = $restoreDbName
|
|
|
|
|
log = $restore.log
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "postgres_restore_check" -Label "PostgreSQL restore drill" -Status "fail" -Detail "Restore check failed; temporary database name is recorded for cleanup review" -Meta @{
|
|
|
|
|
restore_db = $restoreDbName
|
|
|
|
|
log = $restore.log
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$configArchivePath = "$remoteBackupDir/project-config-docs-scripts.tgz"
|
|
|
|
|
$configCommand = "set -e; cd '$RemoteProjectPath'; tar --exclude='project-docs/qa-artifacts' --exclude='backups' -czf '$configArchivePath' docker-compose.server.yml backend/Dockerfile backend/requirements.txt backend/alembic.ini backend/migrations frontend/Dockerfile frontend/package.json frontend/package-lock.json scripts project-docs; test -s '$configArchivePath'; ls -lh '$configArchivePath'"
|
|
|
|
|
$configBackup = Invoke-Remote -Name "04-config-docs-scripts-backup" -Command $configCommand
|
|
|
|
|
if ($configBackup.exit_code -eq 0) {
|
|
|
|
|
Add-Check -Key "config_docs_scripts_backup" -Label "Config docs scripts backup" -Status "pass" -Detail "Deployment config, docs and scripts archive created" -Meta @{
|
|
|
|
|
remote_path = $configArchivePath
|
|
|
|
|
log = $configBackup.log
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "config_docs_scripts_backup" -Label "Config docs scripts backup" -Status "fail" -Detail "Config/docs/scripts backup failed" -Meta @{
|
|
|
|
|
remote_path = $configArchivePath
|
|
|
|
|
log = $configBackup.log
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$minioDir = "$remoteBackupDir/minio-$MinioBucket"
|
|
|
|
|
$minioCommand = @"
|
|
|
|
|
set -e
|
|
|
|
|
mkdir -p '$minioDir'
|
|
|
|
|
mc alias set phase5 '$MinioEndpoint' '$MinioAccessKey' '$MinioSecretKey' >/dev/null
|
|
|
|
|
mc ls --recursive "phase5/$MinioBucket" > '$remoteBackupDir/minio-objects.txt'
|
|
|
|
|
mc mirror --overwrite "phase5/$MinioBucket" '$minioDir'
|
|
|
|
|
object_count=`$(find '$minioDir' -type f | wc -l)
|
|
|
|
|
echo "object_count=`$object_count"
|
|
|
|
|
du -sh '$minioDir' || true
|
|
|
|
|
"@ -replace "`r?`n", "; "
|
|
|
|
|
$minioBackup = Invoke-Remote -Name "05-minio-backup" -Command $minioCommand
|
|
|
|
|
if ($minioBackup.exit_code -eq 0) {
|
|
|
|
|
Add-Check -Key "object_storage_backup" -Label "Object storage backup" -Status "pass" -Detail "MinIO object list and mirror completed" -Meta @{
|
|
|
|
|
remote_path = $minioDir
|
|
|
|
|
log = $minioBackup.log
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "object_storage_backup" -Label "Object storage backup" -Status "fail" -Detail "MinIO object list or mirror failed" -Meta @{
|
|
|
|
|
remote_path = $minioDir
|
|
|
|
|
log = $minioBackup.log
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$manifestCommand = "set -e; cd '$remoteBackupDir'; find . -type f -maxdepth 4 -printf '%p`t%s`n' | sort > manifest.tsv; sha256sum postgres.dump project-config-docs-scripts.tgz > sha256sums.txt; cat manifest.tsv; cat sha256sums.txt"
|
|
|
|
|
$manifest = Invoke-Remote -Name "06-manifest" -Command $manifestCommand
|
|
|
|
|
if ($manifest.exit_code -eq 0) {
|
|
|
|
|
Add-Check -Key "backup_manifest" -Label "Backup manifest" -Status "pass" -Detail "Manifest and sha256 sums generated" -Meta @{
|
|
|
|
|
remote_path = "$remoteBackupDir/manifest.tsv"
|
|
|
|
|
log = $manifest.log
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
Add-Check -Key "backup_manifest" -Label "Backup manifest" -Status "fail" -Detail "Manifest generation failed" -Meta @{
|
|
|
|
|
remote_path = "$remoteBackupDir/manifest.tsv"
|
|
|
|
|
log = $manifest.log
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
foreach ($remoteFile in @("manifest.tsv", "sha256sums.txt", "minio-objects.txt")) {
|
|
|
|
|
$localFile = Join-Path $artifactDir $remoteFile
|
|
|
|
|
$scpResult = Invoke-ScpFromRemote -RemotePath "$remoteBackupDir/$remoteFile" -LocalPath $localFile
|
|
|
|
|
$scpLogPath = Join-Path $artifactDir "scp-$remoteFile.log"
|
|
|
|
|
$scpResult.output | Set-Content -Encoding UTF8 -Path $scpLogPath
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
|
|
|
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
|
|
|
|
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }
|
|
|
|
|
|
|
|
|
|
$summary = [ordered]@{
|
|
|
|
|
name = "phase5-backup-drill"
|
|
|
|
|
status = $overallStatus
|
|
|
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
|
|
|
artifact_dir = $artifactDir
|
|
|
|
|
remote_backup_dir = $remoteBackupDir
|
|
|
|
|
restore_database = $restoreDbName
|
|
|
|
|
targets = [ordered]@{
|
|
|
|
|
server_host = $ServerHost
|
|
|
|
|
remote_project_path = $RemoteProjectPath
|
|
|
|
|
compose_file = $ComposeFile
|
|
|
|
|
database = $DatabaseName
|
|
|
|
|
minio_endpoint = $MinioEndpoint
|
|
|
|
|
minio_bucket = $MinioBucket
|
|
|
|
|
}
|
|
|
|
|
checks = $checks
|
|
|
|
|
totals = [ordered]@{
|
|
|
|
|
total = $checks.Count
|
|
|
|
|
failed_required = $failedRequired.Count
|
|
|
|
|
warnings = $warningsList.Count
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
|
|
|
|
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
|
|
|
$markdown.Add("# Phase 5 Backup Drill") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("- Status: $overallStatus") | Out-Null
|
|
|
|
|
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
|
|
|
|
$markdown.Add("- Remote backup dir: $remoteBackupDir") | Out-Null
|
|
|
|
|
$markdown.Add("- Local artifact dir: $artifactDir") | Out-Null
|
|
|
|
|
$markdown.Add("- Restore database: $restoreDbName") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
|
|
|
|
$markdown.Add("|---|---:|---|---|") | Out-Null
|
|
|
|
|
foreach ($check in $checks) {
|
|
|
|
|
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
|
|
|
|
}
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add("Local artifacts:") | Out-Null
|
|
|
|
|
$markdown.Add("") | Out-Null
|
|
|
|
|
$markdown.Add('- `backup-drill-summary.json`') | Out-Null
|
|
|
|
|
$markdown.Add('- `manifest.tsv`') | Out-Null
|
|
|
|
|
$markdown.Add('- `sha256sums.txt`') | Out-Null
|
|
|
|
|
$markdown.Add('- `minio-objects.txt`') | Out-Null
|
|
|
|
|
$markdown.Add('- `*.log`') | Out-Null
|
|
|
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
|
|
|
|
|
|
|
|
Write-Host "PHASE5_BACKUP_DRILL_$overallStatus $artifactDir"
|
|
|
|
|
if ($overallStatus -eq "FAIL") {
|
|
|
|
|
throw "Phase 5 backup drill failed. See $summaryMd"
|
|
|
|
|
}
|
Codex