param( [string]$ServerHost = "192.168.3.90", [string]$SshUser = "root", [string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519", [string]$RemoteProjectPath = "/opt/ai-contract-platform", [string]$ComposeFile = "docker-compose.server.yml", [string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy", [string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/phase7-rollback", [string]$BackupPath = "", [string]$FrontendBaseUrl = "http://192.168.3.90:3003", [string]$BackendBaseUrl = "http://192.168.3.90:8001", [string]$ApiUrl = "http://192.168.3.90:8001/api/v1", [string]$HealthUrl = "http://192.168.3.90:8001/health", [string]$ReadinessUrl = "http://192.168.3.90:8001/api/v1/system/readiness", [string]$ArtifactRoot = "", [switch]$ListBackups, [switch]$DryRun, [switch]$AllowDirtyWorktree, [switch]$SkipOpsCheck, [switch]$SkipReleaseGate ) $ErrorActionPreference = "Stop" $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff" if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7-frontend-docker-rollback" } if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) { $ArtifactRoot = Join-Path $repoRoot $ArtifactRoot } $artifactDir = Join-Path $ArtifactRoot $timestamp $summaryJson = Join-Path $artifactDir "frontend-docker-rollback-summary.json" $summaryMd = Join-Path $artifactDir "frontend-docker-rollback-summary.md" $gitStatusPath = Join-Path $artifactDir "git-status.txt" New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null $checks = New-Object System.Collections.Generic.List[object] function Add-Check { param( [string]$Key, [string]$Label, [ValidateSet("pass", "fail", "warn", "skip")] [string]$Status, [string]$Detail, [bool]$Required = $true, [hashtable]$Meta = @{} ) $checks.Add([ordered]@{ key = $Key label = $Label status = $Status required = $Required detail = $Detail meta = $Meta }) | Out-Null } function Save-Json { param( [string]$Path, [object]$Value ) $Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path } function Invoke-Remote { param( [string]$Name, [string]$Command ) $target = "${SshUser}@${ServerHost}" $logPath = Join-Path $artifactDir "$Name.log" $args = @( "-i", $SshKeyPath, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", $target, $Command ) $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & ssh @args 2>&1 $exitCode = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } $outputText = ($output -join [Environment]::NewLine) $outputText | Set-Content -Encoding UTF8 -Path $logPath return [ordered]@{ exit_code = $exitCode output = $outputText log = $logPath } } function Convert-KeyValueOutput { param([string]$Text) $data = @{} foreach ($line in ($Text -split "`r?`n")) { if ($line -match '^([A-Z0-9_]+)=(.*)$') { $data[$matches[1]] = $matches[2] } } return $data } if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing" } else { Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK" } if (-not (Test-Path $SshKeyPath)) { Add-Check -Key "ssh_key" -Label "SSH key" -Status "fail" -Detail "SSH key not found: $SshKeyPath" } else { Add-Check -Key "ssh_key" -Label "SSH key" -Status "pass" -Detail "SSH key exists" } $branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim() $commit = (git -C $repoRoot rev-parse HEAD).Trim() $remoteUrl = (git -C $repoRoot remote get-url origin).Trim() $gitStatus = git -C $repoRoot status --short ($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath if ($gitStatus) { if ($AllowDirtyWorktree) { Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; rollback allowed by -AllowDirtyWorktree" -Meta @{ artifact = $gitStatusPath } } else { Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes; pass -AllowDirtyWorktree for development validation" -Meta @{ artifact = $gitStatusPath } } } else { Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath } } if ($remoteUrl -match "192\.168\.3\.90") { Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{ branch = $branch commit = $commit remote = $remoteUrl } } else { Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{ branch = $branch commit = $commit remote = $remoteUrl } } $backupListScript = @" set -euo pipefail backup_root='$RemoteBackupRoot' if ls "`$backup_root"/frontend-before-*.tgz >/dev/null 2>&1; then i=0 for f in `$(ls -1t "`$backup_root"/frontend-before-*.tgz | head -20); do i=`$((i+1)) size="`$(stat -c %s "`$f")" epoch="`$(stat -c %Y "`$f")" echo "BACKUP_ITEM=`$i,`$f,`$size,`$epoch" done echo "BACKUP_COUNT=`$i" else echo "BACKUP_COUNT=0" fi "@ $backupList = Invoke-Remote -Name "01-list-frontend-backups" -Command $backupListScript $backupData = Convert-KeyValueOutput -Text $backupList.output $backupCount = if ($backupData.BACKUP_COUNT) { [int]$backupData.BACKUP_COUNT } else { 0 } $availableBackups = @() foreach ($line in ($backupList.output -split "`r?`n")) { if ($line -match '^BACKUP_ITEM=(\d+),([^,]+),(\d+),(\d+)$') { $availableBackups += [ordered]@{ index = [int]$matches[1] path = $matches[2] size = [int64]$matches[3] epoch = [int64]$matches[4] } } } $autoSelectedBackup = $null if ($availableBackups.Count -gt 0) { $autoSelectedBackup = ($availableBackups | Sort-Object index | Select-Object -First 1).path } if ($backupList.exit_code -eq 0 -and $backupCount -gt 0) { Add-Check -Key "backup_list" -Label "frontend backup list" -Status "pass" -Detail "Found $backupCount frontend backup(s)" -Meta @{ selected_backup = $autoSelectedBackup log = $backupList.log } } else { Add-Check -Key "backup_list" -Label "frontend backup list" -Status "fail" -Detail "No frontend backup is available for rollback" -Meta @{ log = $backupList.log } } if ($ListBackups) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "skip" -Required $false -Detail "Listed backups only; no remote mutation" } else { $failedBeforeRollback = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeRollback.Count -eq 0) { $dryRunFlag = if ($DryRun) { "true" } else { "false" } $requestedBackup = if ([string]::IsNullOrWhiteSpace($BackupPath)) { $autoSelectedBackup } else { $BackupPath } $rollbackScript = @" set -euo pipefail project='$RemoteProjectPath' compose='$ComposeFile' backup_root='$RemoteBackupRoot' rollback_root='$RemoteRollbackRoot' requested='$requestedBackup' stamp='$timestamp' dry_run='$dryRunFlag' cd "`$project" test "`$PWD" = "`$project" test -f "`$compose" test -d "frontend" selected="`$requested" case "`$selected" in "`$backup_root"/frontend-before-*.tgz) ;; *) echo "ROLLBACK_SELECTED_BACKUP=`$selected"; echo "ROLLBACK_ERROR=selected backup is outside backup root"; exit 2 ;; esac test -s "`$selected" tar -tzf "`$selected" frontend/package.json >/dev/null selected_size="`$(stat -c %s "`$selected")" echo "ROLLBACK_SELECTED_BACKUP=`$selected" echo "ROLLBACK_SELECTED_BACKUP_SIZE=`$selected_size" if [ "`$dry_run" = "true" ]; then echo "ROLLBACK_DRY_RUN=1" exit 0 fi mkdir -p "`$rollback_root" current_backup="`$rollback_root/frontend-current-before-rollback-`$stamp.tgz" tar -czf "`$current_backup" frontend test -s "`$current_backup" rm -rf frontend tar -xzf "`$selected" test -f frontend/package.json docker compose -f "`$compose" build --no-cache frontend docker compose -f "`$compose" up -d frontend docker compose -f "`$compose" ps backend frontend current_backup_size="`$(stat -c %s "`$current_backup")" echo "ROLLBACK_CURRENT_BACKUP=`$current_backup" echo "ROLLBACK_CURRENT_BACKUP_SIZE=`$current_backup_size" "@ $rollback = Invoke-Remote -Name "02-remote-frontend-rollback" -Command $rollbackScript $rollbackData = Convert-KeyValueOutput -Text $rollback.output if ($rollback.exit_code -eq 0 -and $DryRun) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "pass" -Detail "Rollback dry run validated selected backup without remote mutation" -Meta @{ selected_backup = $rollbackData.ROLLBACK_SELECTED_BACKUP log = $rollback.log } } elseif ($rollback.exit_code -eq 0) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "pass" -Detail "Remote frontend rollback, rebuild and restart completed" -Meta @{ selected_backup = $rollbackData.ROLLBACK_SELECTED_BACKUP current_backup = $rollbackData.ROLLBACK_CURRENT_BACKUP log = $rollback.log } } else { Add-Check -Key "rollback_action" -Label "rollback action" -Status "fail" -Detail "Remote frontend rollback failed" -Meta @{ exit_code = $rollback.exit_code log = $rollback.log } } } } $failedBeforeChecks = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($ListBackups -or $DryRun) { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "skip" -Required $false -Detail "Skipped because no rollback mutation was performed" Add-Check -Key "release_gate" -Label "post-rollback release gate" -Status "skip" -Required $false -Detail "Skipped because no rollback mutation was performed" } elseif ($failedBeforeChecks.Count -eq 0) { if ($SkipOpsCheck) { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "skip" -Required $false -Detail "Skipped by -SkipOpsCheck" } else { $opsScript = Join-Path $repoRoot "scripts\run-production-ops-check.ps1" $opsLog = Join-Path $artifactDir "03-post-rollback-ops-check.log" $opsArgs = @( "-ServerHost", $ServerHost, "-SshUser", $SshUser, "-SshKeyPath", $SshKeyPath, "-RemoteProjectPath", $RemoteProjectPath, "-ComposeFile", $ComposeFile, "-RemoteBackupRoot", $RemoteBackupRoot, "-RemoteRollbackRoot", $RemoteRollbackRoot, "-FrontendBaseUrl", $FrontendBaseUrl, "-BackendBaseUrl", $BackendBaseUrl, "-HealthUrl", $HealthUrl, "-ReadinessUrl", $ReadinessUrl, "-ArtifactRoot", (Join-Path $artifactDir "ops-check") ) if (-not $AllowDirtyWorktree) { $opsArgs += "-RequireCleanWorktree" } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $opsOutput = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $opsScript @opsArgs 2>&1 $opsExit = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } $opsText = ($opsOutput -join [Environment]::NewLine) $opsText | Set-Content -Encoding UTF8 -Path $opsLog $opsArtifact = $null $opsMatch = [regex]::Match($opsText, '([A-Z]:\\[^\r\n]+)$', [System.Text.RegularExpressions.RegexOptions]::Multiline) if ($opsMatch.Success) { $opsArtifact = $opsMatch.Groups[1].Value.Trim() } if ($opsExit -eq 0 -and $opsText -match "_WARN") { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "warn" -Required $false -Detail "Ops check completed with warnings" -Meta @{ artifact_dir = $opsArtifact log = $opsLog } } elseif ($opsExit -eq 0) { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "pass" -Detail "Ops check passed" -Meta @{ artifact_dir = $opsArtifact log = $opsLog } } else { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "fail" -Detail "Ops check failed" -Meta @{ artifact_dir = $opsArtifact log = $opsLog } } } if ($SkipReleaseGate) { Add-Check -Key "release_gate" -Label "post-rollback release gate" -Status "skip" -Required $false -Detail "Skipped by -SkipReleaseGate" } elseif ([string]::IsNullOrWhiteSpace($env:AI_CONTRACT_ADMIN_PASSWORD)) { Add-Check -Key "release_gate" -Label "post-rollback release gate" -Status "fail" -Detail "AI_CONTRACT_ADMIN_PASSWORD must be set in the current process" } else { $gateScript = Join-Path $repoRoot "scripts\run-production-release-gate.ps1" $gateLog = Join-Path $artifactDir "04-post-rollback-release-gate.log" $gateArgs = @( "-ServerHost", $ServerHost, "-FrontendBaseUrl", $FrontendBaseUrl, "-BackendBaseUrl", $BackendBaseUrl, "-ApiUrl", $ApiUrl, "-HealthUrl", $HealthUrl, "-ReadinessUrl", $ReadinessUrl, "-ArtifactRoot", (Join-Path $artifactDir "release-gate") ) if (-not $AllowDirtyWorktree) { $gateArgs += "-RequireCleanWorktree" } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $gateOutput = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $gateScript @gateArgs 2>&1 $gateExit = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } $gateText = ($gateOutput -join [Environment]::NewLine) $gateText | Set-Content -Encoding UTF8 -Path $gateLog $gateArtifact = $null $gateMatch = [regex]::Match($gateText, '([A-Z]:\\[^\r\n]+)$', [System.Text.RegularExpressions.RegexOptions]::Multiline) if ($gateMatch.Success) { $gateArtifact = $gateMatch.Groups[1].Value.Trim() } if ($gateExit -eq 0 -and $gateText -match "_WARN") { Add-Check -Key "release_gate" -Label "post-rollback release gate" -Status "warn" -Required $false -Detail "Release gate completed with warnings" -Meta @{ artifact_dir = $gateArtifact log = $gateLog } } elseif ($gateExit -eq 0) { Add-Check -Key "release_gate" -Label "post-rollback release gate" -Status "pass" -Detail "Release gate passed" -Meta @{ artifact_dir = $gateArtifact log = $gateLog } } else { Add-Check -Key "release_gate" -Label "post-rollback release gate" -Status "fail" -Detail "Release gate failed" -Meta @{ artifact_dir = $gateArtifact log = $gateLog } } } } $failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) $warningsList = @($checks | Where-Object { $_.status -eq "warn" }) $skippedList = @($checks | Where-Object { $_.status -eq "skip" }) $overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" } $summary = [ordered]@{ name = "phase7-frontend-docker-rollback" status = $overallStatus generated_at = (Get-Date).ToUniversalTime().ToString("o") artifact_dir = $artifactDir branch = $branch commit = $commit remote = $remoteUrl targets = [ordered]@{ server_host = $ServerHost remote_project_path = $RemoteProjectPath compose_file = $ComposeFile remote_backup_root = $RemoteBackupRoot remote_rollback_root = $RemoteRollbackRoot frontend_base_url = $FrontendBaseUrl backend_base_url = $BackendBaseUrl } options = [ordered]@{ backup_path = $BackupPath list_backups = [bool]$ListBackups dry_run = [bool]$DryRun allow_dirty_worktree = [bool]$AllowDirtyWorktree skip_ops_check = [bool]$SkipOpsCheck skip_release_gate = [bool]$SkipReleaseGate } checks = $checks totals = [ordered]@{ total = $checks.Count failed_required = $failedRequired.Count warnings = $warningsList.Count skipped = $skippedList.Count } } Save-Json -Path $summaryJson -Value $summary $markdown = New-Object System.Collections.Generic.List[string] $markdown.Add("# Phase 7 Frontend Docker Rollback") | Out-Null $markdown.Add("") | Out-Null $markdown.Add("- Status: $overallStatus") | Out-Null $markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null $markdown.Add("- Branch: $branch") | Out-Null $markdown.Add("- Commit: $commit") | Out-Null $markdown.Add("- Artifact dir: $artifactDir") | Out-Null $markdown.Add("") | Out-Null $markdown.Add("| Check | Required | Status | Detail |") | Out-Null $markdown.Add("|---|---:|---|---|") | Out-Null foreach ($check in $checks) { $markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null } $markdown.Add("") | Out-Null $markdown.Add("Local artifacts:") | Out-Null $markdown.Add("") | Out-Null $markdown.Add('- `frontend-docker-rollback-summary.json`') | Out-Null $markdown.Add('- `frontend-docker-rollback-summary.md`') | Out-Null $markdown.Add('- `git-status.txt`') | Out-Null $markdown.Add('- `*.log`') | Out-Null foreach ($check in $checks) { if ($check.meta.artifact_dir) { $markdown.Add(('- {0}: `{1}`' -f $check.label, $check.meta.artifact_dir)) | Out-Null } } $markdown | Set-Content -Encoding UTF8 -Path $summaryMd Write-Host "PHASE7_FRONTEND_DOCKER_ROLLBACK_$overallStatus $artifactDir" if ($overallStatus -eq "FAIL") { throw "Phase 7 frontend Docker rollback failed. See $summaryMd" }