165 lines
9.0 KiB
PowerShell
165 lines
9.0 KiB
PowerShell
param(
|
|
[string]$ServerHost = "192.168.3.90",
|
|
[string]$SshUser = "root",
|
|
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
|
|
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
|
|
[string]$ComposeFile = "docker-compose.server.yml",
|
|
[string]$SharedAiBackupRoot = "/opt/ai-contract-platform/backups/shared-ai-platform",
|
|
[string]$BackendBackupRoot = "/opt/ai-contract-platform/backups/backend-shared-ai-rollout",
|
|
[string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/shared-ai-platform-rollback",
|
|
[string]$SharedAiBackupPath = "",
|
|
[string]$BackendBackupPath = "",
|
|
[string]$ArtifactRoot = "",
|
|
[switch]$ListBackups,
|
|
[switch]$DryRun,
|
|
[switch]$AllowDirtyWorktree,
|
|
[switch]$SkipOpsCheck
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\shared-ai-platform-docker-rollback"
|
|
}
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
$summaryJson = Join-Path $artifactDir "shared-ai-platform-docker-rollback-summary.json"
|
|
$summaryMd = Join-Path $artifactDir "shared-ai-platform-docker-rollback-summary.md"
|
|
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
|
|
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
|
|
function Add-Check {
|
|
param([string]$Key,[string]$Label,[ValidateSet("pass","fail","warn","skip")][string]$Status,[string]$Detail,[bool]$Required=$true,[hashtable]$Meta=@{})
|
|
$checks.Add([ordered]@{ key=$Key; label=$Label; status=$Status; required=$Required; detail=$Detail; meta=$Meta }) | Out-Null
|
|
}
|
|
|
|
function Save-Json {
|
|
param([string]$Path,[object]$Value)
|
|
$Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
|
|
}
|
|
|
|
function Invoke-Remote {
|
|
param([string]$Name,[string]$Command)
|
|
$target = "${SshUser}@${ServerHost}"
|
|
$logPath = Join-Path $artifactDir "$Name.log"
|
|
$args = @("-i",$SshKeyPath,"-o","BatchMode=yes","-o","StrictHostKeyChecking=accept-new","-o","ConnectTimeout=10",$target,$Command)
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & ssh @args 2>&1
|
|
$exitCode = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
($output -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $logPath
|
|
return [ordered]@{ exit_code=$exitCode; output=($output -join [Environment]::NewLine); log=$logPath }
|
|
}
|
|
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing" } else { Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK" }
|
|
if (-not (Test-Path $SshKeyPath)) { Add-Check -Key "ssh_key" -Label "SSH key" -Status "fail" -Detail "SSH key not found: $SshKeyPath" } else { Add-Check -Key "ssh_key" -Label "SSH key" -Status "pass" -Detail "SSH key exists" }
|
|
|
|
$gitStatus = git -C $repoRoot status --short
|
|
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
|
|
if ($gitStatus) {
|
|
if ($AllowDirtyWorktree) { Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; rollback allowed by -AllowDirtyWorktree" -Meta @{ artifact = $gitStatusPath } }
|
|
else { Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes; pass -AllowDirtyWorktree for controlled rollback validation" -Meta @{ artifact = $gitStatusPath } }
|
|
} else {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
|
|
}
|
|
|
|
function Select-LatestBackup {
|
|
param([string]$Label,[string]$Root,[string]$Pattern,[string]$ExplicitPath,[string]$CheckKey,[string]$LogName)
|
|
$listScript = @"
|
|
set -euo pipefail
|
|
root='$Root'
|
|
if ls "`$root"/$Pattern >/dev/null 2>&1; then
|
|
ls -1t "`$root"/$Pattern | head -20
|
|
fi
|
|
"@
|
|
$result = Invoke-Remote -Name $LogName -Command $listScript
|
|
$items = @($result.output -split "`r?`n" | Where-Object { $_.Trim() })
|
|
$selected = if ($ExplicitPath) { $ExplicitPath } elseif ($items.Count -gt 0) { $items[0] } else { "" }
|
|
if ($selected) { Add-Check -Key $CheckKey -Label $Label -Status "pass" -Detail "Selected backup found" -Meta @{ selected_backup = $selected; log = $result.log } }
|
|
else { Add-Check -Key $CheckKey -Label $Label -Status "fail" -Detail "No backup available" -Meta @{ log = $result.log } }
|
|
return $selected
|
|
}
|
|
|
|
$selectedSharedBackup = Select-LatestBackup -Label "shared-ai backup" -Root $SharedAiBackupRoot -Pattern "shared-ai-platform-before-*.tgz" -ExplicitPath $SharedAiBackupPath -CheckKey "shared_backup" -LogName "01-list-shared-ai-backups"
|
|
$selectedBackendBackup = Select-LatestBackup -Label "backend backup" -Root $BackendBackupRoot -Pattern "backend-before-*.tgz" -ExplicitPath $BackendBackupPath -CheckKey "backend_backup" -LogName "02-list-backend-backups"
|
|
|
|
if ($ListBackups) {
|
|
Add-Check -Key "rollback_action" -Label "rollback action" -Status "skip" -Required $false -Detail "Listed backups only; no remote mutation"
|
|
} else {
|
|
$failedBeforeRollback = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
if ($failedBeforeRollback.Count -eq 0) {
|
|
$dryRunFlag = if ($DryRun) { "true" } else { "false" }
|
|
$rollbackScript = @"
|
|
set -euo pipefail
|
|
project='$RemoteProjectPath'
|
|
compose='$ComposeFile'
|
|
shared_backup='$selectedSharedBackup'
|
|
backend_backup='$selectedBackendBackup'
|
|
rollback_root='$RemoteRollbackRoot'
|
|
stamp='$timestamp'
|
|
dry_run='$dryRunFlag'
|
|
cd "`$project"
|
|
test -f "`$compose"
|
|
test -s "`$shared_backup"
|
|
test -s "`$backend_backup"
|
|
if [ "`$dry_run" = "true" ]; then
|
|
echo "ROLLBACK_DRY_RUN=1"
|
|
exit 0
|
|
fi
|
|
mkdir -p "`$rollback_root"
|
|
tar -czf "`$rollback_root/shared-ai-current-before-rollback-`$stamp.tgz" -C "`$project" shared-ai-platform || true
|
|
tar -czf "`$rollback_root/backend-current-before-rollback-`$stamp.tgz" -C "`$project" backend || true
|
|
rm -rf "`$project/shared-ai-platform"
|
|
mkdir -p "`$project/shared-ai-platform"
|
|
tar -xzf "`$shared_backup" --strip-components=1 -C "`$project/shared-ai-platform"
|
|
rm -rf "`$project/backend"
|
|
mkdir -p "`$project/backend"
|
|
tar -xzf "`$backend_backup" --strip-components=1 -C "`$project/backend"
|
|
docker compose -f "`$compose" build shared-ai-platform backend
|
|
docker compose -f "`$compose" up -d shared-ai-platform backend
|
|
docker compose -f "`$compose" ps shared-ai-platform backend frontend
|
|
"@
|
|
$rollback = Invoke-Remote -Name "03-remote-shared-ai-rollback" -Command $rollbackScript
|
|
if ($rollback.exit_code -eq 0 -and $DryRun) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "pass" -Detail "Rollback dry run validated backups" -Meta @{ log = $rollback.log } }
|
|
elseif ($rollback.exit_code -eq 0) { Add-Check -Key "rollback_action" -Label "rollback action" -Status "pass" -Detail "Remote shared-ai-platform rollback completed" -Meta @{ log = $rollback.log } }
|
|
else { Add-Check -Key "rollback_action" -Label "rollback action" -Status "fail" -Detail "Remote rollback failed" -Meta @{ log = $rollback.log } }
|
|
}
|
|
}
|
|
|
|
if ($ListBackups -or $DryRun -or $SkipOpsCheck) {
|
|
Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "skip" -Required $false -Detail "Skipped"
|
|
} else {
|
|
$failedBeforeOps = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
if ($failedBeforeOps.Count -eq 0) {
|
|
$opsScript = Join-Path $repoRoot "scripts\run-shared-ai-platform-ops-check.ps1"
|
|
$opsLog = Join-Path $artifactDir "04-post-rollback-ops-check.log"
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$opsOutput = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $opsScript -ArtifactRoot (Join-Path $artifactDir "ops-check") 2>&1
|
|
$opsExit = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
($opsOutput -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $opsLog
|
|
if ($opsExit -eq 0) { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "pass" -Detail "Shared-ai ops check passed" -Meta @{ log = $opsLog } }
|
|
else { Add-Check -Key "ops_check" -Label "post-rollback ops check" -Status "fail" -Detail "Shared-ai ops check failed" -Meta @{ log = $opsLog } }
|
|
}
|
|
}
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } else { "PASS" }
|
|
$summary = [ordered]@{ name="shared-ai-platform-docker-rollback"; status=$overallStatus; generated_at=(Get-Date).ToUniversalTime().ToString("o"); artifact_dir=$artifactDir; checks=$checks }
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
$markdown = @("# Shared AI Platform Docker Rollback","","- Status: $overallStatus","- Generated at: $($summary.generated_at)","","## Checks")
|
|
foreach ($check in $checks) { $markdown += "- [$($check.status)] $($check.label): $($check.detail)" }
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
Write-Output $artifactDir
|