param( [string]$ServerHost = "192.168.3.90", [string]$SshUser = "root", [string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519", [string]$RemoteProjectPath = "/opt/ai-contract-platform", [string]$ComposeFile = "docker-compose.server.yml", [string]$SharedAiBackupRoot = "/opt/ai-contract-platform/backups/shared-ai-platform", [string]$BackendBackupRoot = "/opt/ai-contract-platform/backups/backend-shared-ai-rollout", [string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/shared-ai-platform-rollback", [string]$SharedAiBackupPath = "", [string]$BackendBackupPath = "", [string]$ArtifactRoot = "", [switch]$ListBackups, [switch]$DryRun, [switch]$AllowDirtyWorktree, [switch]$SkipOpsCheck ) $ErrorActionPreference = "Stop" $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff" if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\shared-ai-platform-docker-rollback" } $artifactDir = Join-Path $ArtifactRoot $timestamp $summaryJson = Join-Path $artifactDir "shared-ai-platform-docker-rollback-summary.json" $summaryMd = Join-Path $artifactDir "shared-ai-platform-docker-rollback-summary.md" $gitStatusPath = Join-Path $artifactDir "git-status.txt" New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null $checks = New-Object System.Collections.Generic.List[object] function Add-Check { param([string]$Key,[string]$Label,[ValidateSet("pass","fail","warn","skip")][string]$Status,[string]$Detail,[bool]$Required=$true,[hashtable]$Meta=@{}) $checks.Add([ordered]@{ key=$Key; label=$Label; status=$Status; required=$Required; detail=$Detail; meta=$Meta }) | Out-Null } function Save-Json { param([string]$Path,[object]$Value) $Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path } function Invoke-Remote { param([string]$Name,[string]$Command) $target = "${SshUser}@${ServerHost}" $logPath = Join-Path $artifactDir "$Name.log" $args = @("-i",$SshKeyPath,"-o","BatchMode=yes","-o","StrictHostKeyChecking=accept-new","-o","ConnectTimeout=10",$target,$Command) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh @args 2>&1 $exitCode = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } ($output -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $logPath return [ordered]@{ exit_code=$exitCode; output=($output -join [Environment]::NewLine); log=$logPath } } if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing" } else { Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK" } if (-not (Test-Path $SshKeyPath)) { Add-Check -Key "ssh_key" -Label "SSH key" -Status "fail" -Detail "SSH key not found: $SshKeyPath" } else { Add-Check -Key "ssh_key" -Label "SSH key" -Status "pass" -Detail "SSH key exists" } $gitStatus = git -C $repoRoot status --short ($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath if ($gitStatus) { if ($AllowDirtyWorktree) { Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; rollback allowed by -AllowDirtyWorktree" -Meta @{ artifact = $gitStatusPath } } else { Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes; pass -AllowDirtyWorktree for controlled rollback validation" -Meta @{ artifact = $gitStatusPath } } } else { Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath } } function Select-LatestBackup { param([string]$Label,[string]$Root,[string]$Pattern,[string]$ExplicitPath,[string]$CheckKey,[string]$LogName) $listScript = @" set -euo pipefail root='$Root' if ls "`$root"/$Pattern >/dev/null 2>&1; then ls -1t "`$root"/$Pattern | head -20 fi "@ $result = Invoke-Remote -Name $LogName -Command $listScript $items = @($result.output -split "`r?`n" | Where-Object { $_.Trim() }) $selected = if ($ExplicitPath) { $ExplicitPath } elseif ($items.Count -gt 0) { $items[0] } else { "" } if ($selected) { Add-Check -Key $CheckKey -Label $Label -Status "pass" -Detail "Selected backup found" -Meta @{ selected_backup = $selected; log = $result.log } } else { Add-Check -Key $CheckKey -Label $Label -Status "fail" -Detail "No backup available" -Meta @{ log = $result.log } } return $selected } $selectedSharedBackup = Select-LatestBackup -Label "shared-ai backup" -Root $SharedAiBackupRoot -Pattern "shared-ai-platform-before-*.tgz" -ExplicitPath $SharedAiBackupPath -CheckKey "shared_backup" -LogName "01-list-shared-ai-backups" $selectedBackendBackup = Select-LatestBackup -Label "backend backup" -Root $BackendBackupRoot -Pattern "backend-before-*.tgz" -ExplicitPath $BackendBackupPath -CheckKey "backend_backup" -LogName "02-list-backend-backups" if ($ListBackups) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "skip" -Required $false -Detail "Listed backups only; no remote mutation" } else { $failedBeforeRollback = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeRollback.Count -eq 0) { $dryRunFlag = if ($DryRun) { "true" } else { "false" } $rollbackScript = @" set -euo pipefail project='$RemoteProjectPath' compose='$ComposeFile' shared_backup='$selectedSharedBackup' backend_backup='$selectedBackendBackup' rollback_root='$RemoteRollbackRoot' stamp='$timestamp' dry_run='$dryRunFlag' cd "`$project" test -f "`$compose" test -s "`$shared_backup" test -s "`$backend_backup" if [ "`$dry_run" = "true" ]; then echo "ROLLBACK_DRY_RUN=1" exit 0 fi mkdir -p "`$rollback_root" tar -czf "`$rollback_root/shared-ai-current-before-rollback-`$stamp.tgz" -C "`$project" shared-ai-platform || true tar -czf "`$rollback_root/backend-current-before-rollback-`$stamp.tgz" -C "`$project" backend || true rm -rf "`$project/shared-ai-platform" mkdir -p "`$project/shared-ai-platform" tar -xzf "`$shared_backup" --strip-components=1 -C "`$project/shared-ai-platform" rm -rf "`$project/backend" mkdir -p "`$project/backend" tar -xzf "`$backend_backup" --strip-components=1 -C "`$project/backend" docker compose -f "`$compose" build shared-ai-platform backend docker compose -f "`$compose" up -d shared-ai-platform backend docker compose -f "`$compose" ps shared-ai-platform backend frontend "@ $rollback = Invoke-Remote -Name "03-remote-shared-ai-rollback" -Command $rollbackScript if ($rollback.exit_code -eq 0 -and $DryRun) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "pass" -Detail "Rollback dry run validated backups" -Meta @{ log = $rollback.log } } elseif ($rollback.exit_code -eq 0) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "pass" -Detail "Remote shared-ai-platform rollback completed" -Meta @{ log = $rollback.log } } else { Add-Check -Key "rollback_action" -Label "rollback action" -Status "fail" -Detail "Remote rollback failed" -Meta @{ log = $rollback.log } } } } if ($ListBackups -or $DryRun -or $SkipOpsCheck) { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "skip" -Required $false -Detail "Skipped" } else { $failedBeforeOps = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeOps.Count -eq 0) { $opsScript = Join-Path $repoRoot "scripts\run-shared-ai-platform-ops-check.ps1" $opsLog = Join-Path $artifactDir "04-post-rollback-ops-check.log" $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $opsOutput = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $opsScript -ArtifactRoot (Join-Path $artifactDir "ops-check") 2>&1 $opsExit = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } ($opsOutput -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $opsLog if ($opsExit -eq 0) { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "pass" -Detail "Shared-ai ops check passed" -Meta @{ log = $opsLog } } else { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "fail" -Detail "Shared-ai ops check failed" -Meta @{ log = $opsLog } } } } $failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) $overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } else { "PASS" } $summary = [ordered]@{ name="shared-ai-platform-docker-rollback"; status=$overallStatus; generated_at=(Get-Date).ToUniversalTime().ToString("o"); artifact_dir=$artifactDir; checks=$checks } Save-Json -Path $summaryJson -Value $summary $markdown = @("# Shared AI Platform Docker Rollback","","- Status: $overallStatus","- Generated at: $($summary.generated_at)","","## Checks") foreach ($check in $checks) { $markdown += "- [$($check.status)] $($check.label): $($check.detail)" } $markdown | Set-Content -Encoding UTF8 -Path $summaryMd Write-Output $artifactDir