param( [string]$ServerHost = "192.168.3.90", [string]$SshUser = "root", [string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519", [string]$RemoteProjectPath = "/tmp/ai-contract-platform-121bf33bf415", [string]$RemoteBackupRoot = "", [string]$BackendContainerName = "ai-contract-backend-restore", [string]$FrontendContainerName = "ai-contract-frontend", [string]$WorkerContainerName = "ai-contract-legal-change-worker", [string]$BackendImage = "ai-contract-backend-restore:121bf33bf415", [string]$FrontendImage = "node:22-bookworm-slim", [int]$BackendPort = 8001, [int]$FrontendHostPort = 3003, [int]$FrontendContainerPort = 3000, [int]$WorkerLimit = 10, [int]$WorkerIntervalSeconds = 300, [string]$BackendBaseUrl = "http://192.168.3.90:8001", [string]$FrontendBaseUrl = "http://192.168.3.90:3003", [string]$HealthUrl = "http://192.168.3.90:8001/health", [string]$SmokeFrontendPath = "/operation-kpis", [string]$ArtifactRoot = "", [switch]$AllowDirtyWorktree, [switch]$UseSshpass, [switch]$PlanOnly, [switch]$SkipSmoke, [switch]$SkipLegalChangeWorker, [switch]$KeepRemoteArchive ) $ErrorActionPreference = "Stop" $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss" if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\restore-docker-deploy" } if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) { $ArtifactRoot = Join-Path $repoRoot $ArtifactRoot } if ([string]::IsNullOrWhiteSpace($RemoteBackupRoot)) { $RemoteBackupRoot = "$RemoteProjectPath/backups/restore-deploy" } $artifactDir = Join-Path $ArtifactRoot $timestamp $summaryJson = Join-Path $artifactDir "restore-docker-deploy-summary.json" $summaryMd = Join-Path $artifactDir "restore-docker-deploy-summary.md" $gitStatusPath = Join-Path $artifactDir "git-status.txt" $backendArchiveName = "ai-contract-backend-restore-$timestamp.tgz" $frontendArchiveName = "ai-contract-frontend-restore-$timestamp.tgz" $remoteScriptName = "ai-contract-restore-deploy-$timestamp.sh" $localBackendArchive = Join-Path $artifactDir $backendArchiveName $localFrontendArchive = Join-Path $artifactDir $frontendArchiveName $localRemoteScript = Join-Path $artifactDir $remoteScriptName $remoteBackendArchive = "/tmp/$backendArchiveName" $remoteFrontendArchive = "/tmp/$frontendArchiveName" $remoteScriptPath = "/tmp/$remoteScriptName" New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null $checks = New-Object System.Collections.Generic.List[object] $script:AuthMode = $null $script:SshpassPath = $null function Add-Check { param( [string]$Key, [string]$Label, [ValidateSet("pass", "fail", "warn", "skip")] [string]$Status, [string]$Detail, [bool]$Required = $true, [hashtable]$Meta = @{} ) $checks.Add([ordered]@{ key = $Key label = $Label status = $Status required = $Required detail = $Detail meta = $Meta }) | Out-Null } function Save-Json { param( [string]$Path, [object]$Value ) $Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path } function Invoke-LocalStep { param( [string]$Name, [scriptblock]$Script ) $logPath = Join-Path $artifactDir "$Name.log" $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & $Script 2>&1 $exitCode = if ($LASTEXITCODE -is [int]) { $LASTEXITCODE } else { 0 } } catch { $output = @($_.Exception.Message) $exitCode = 1 } finally { $ErrorActionPreference = $previousErrorActionPreference } $outputText = ($output -join [Environment]::NewLine) $outputText | Set-Content -Encoding UTF8 -Path $logPath return [ordered]@{ exit_code = $exitCode output = $outputText log = $logPath } } function Resolve-SshpassPath { $command = Get-Command sshpass.exe -ErrorAction SilentlyContinue if ($null -ne $command) { return $command.Source } $candidates = @( (Join-Path $env:LOCALAPPDATA "Microsoft\WinGet\Packages\xhcoding.sshpass-win32_Microsoft.Winget.Source_8wekyb3d8bbwe\sshpass.exe") ) foreach ($candidate in $candidates) { if (Test-Path $candidate) { return $candidate } } return $null } function ConvertTo-ShellSingleQuote { param([string]$Value) return "'" + ($Value -replace "'", "'`"`"`'") + "'" } function Invoke-Remote { param( [string]$Name, [string]$Command ) $target = "${SshUser}@${ServerHost}" $logPath = Join-Path $artifactDir "$Name.log" if ($script:AuthMode -eq "sshpass") { $filePath = $script:SshpassPath $args = @( "-e", "ssh", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", $target, $Command ) } else { $filePath = "ssh" $args = @( "-i", $SshKeyPath, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", $target, $Command ) } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & $filePath @args 2>&1 $exitCode = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } $outputText = ($output -join [Environment]::NewLine) $outputText | Set-Content -Encoding UTF8 -Path $logPath return [ordered]@{ exit_code = $exitCode output = $outputText log = $logPath } } function Invoke-ScpToRemote { param( [string]$Name, [string]$LocalPath, [string]$RemotePath ) $target = "${SshUser}@${ServerHost}:$RemotePath" $logPath = Join-Path $artifactDir "$Name.log" if ($script:AuthMode -eq "sshpass") { $filePath = $script:SshpassPath $args = @( "-e", "scp", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", $LocalPath, $target ) } else { $filePath = "scp" $args = @( "-i", $SshKeyPath, "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", $LocalPath, $target ) } $previousErrorActionPreference = $ErrorActionPreference $ErrorActionPreference = "Continue" try { $output = & $filePath @args 2>&1 $exitCode = $LASTEXITCODE } finally { $ErrorActionPreference = $previousErrorActionPreference } $outputText = ($output -join [Environment]::NewLine) $outputText | Set-Content -Encoding UTF8 -Path $logPath return [ordered]@{ exit_code = $exitCode output = $outputText log = $logPath } } function New-RemoteDeployScript { $remoteShell = @' #!/usr/bin/env bash set -euo pipefail project=__REMOTE_PROJECT_PATH__ backend_archive=__REMOTE_BACKEND_ARCHIVE__ frontend_archive=__REMOTE_FRONTEND_ARCHIVE__ backup_root=__REMOTE_BACKUP_ROOT__ stamp=__STAMP__ backend_container=__BACKEND_CONTAINER__ frontend_container=__FRONTEND_CONTAINER__ worker_container=__WORKER_CONTAINER__ backend_image=__BACKEND_IMAGE__ frontend_image=__FRONTEND_IMAGE__ backend_port=__BACKEND_PORT__ frontend_host_port=__FRONTEND_HOST_PORT__ frontend_container_port=__FRONTEND_CONTAINER_PORT__ worker_limit=__WORKER_LIMIT__ worker_interval_seconds=__WORKER_INTERVAL_SECONDS__ smoke_frontend_path=__SMOKE_FRONTEND_PATH__ skip_legal_change_worker=__SKIP_LEGAL_CHANGE_WORKER__ keep_remote_archive=__KEEP_REMOTE_ARCHIVE__ case "$project" in /tmp/ai-contract-platform-*|/opt/ai-contract-platform) ;; *) echo "REFUSE_UNEXPECTED_PROJECT_PATH=$project" exit 20 ;; esac cd "$project" test "$PWD" = "$project" test -d backend test -d frontend test -f backend/app/main.py test -f frontend/package.json test -f backend/.env test -s "$backend_archive" test -s "$frontend_archive" docker image inspect "$backend_image" >/dev/null docker image inspect "$frontend_image" >/dev/null backup_dir="$backup_root/$stamp" mkdir -p "$backup_dir" tar -czf "$backup_dir/backend-before-$stamp.tgz" backend tar -czf "$backup_dir/frontend-before-$stamp.tgz" frontend test -s "$backup_dir/backend-before-$stamp.tgz" test -s "$backup_dir/frontend-before-$stamp.tgz" cp backend/.env "$backup_dir/backend.env" if docker ps -a --format '{{.Names}}' | grep -Fx "$backend_container" >/dev/null; then docker inspect "$backend_container" > "$backup_dir/backend-container-inspect.json" fi if docker ps -a --format '{{.Names}}' | grep -Fx "$frontend_container" >/dev/null; then docker inspect "$frontend_container" > "$backup_dir/frontend-container-inspect.json" fi if docker ps -a --format '{{.Names}}' | grep -Fx "$worker_container" >/dev/null; then docker inspect "$worker_container" > "$backup_dir/legal-change-worker-container-inspect.json" fi temp_root="/tmp/ai-contract-restore-$stamp" rm -rf "$temp_root" mkdir -p "$temp_root" cp backend/.env "$temp_root/backend.env" if [ -f frontend/.env.local ]; then cp frontend/.env.local "$temp_root/frontend.env.local"; fi if [ -d frontend/node_modules ]; then mv frontend/node_modules "$temp_root/frontend-node_modules"; fi rm -rf backend frontend mkdir -p backend frontend tar -xzf "$backend_archive" --strip-components=1 -C backend tar -xzf "$frontend_archive" --strip-components=1 -C frontend cp "$temp_root/backend.env" backend/.env if [ -f "$temp_root/frontend.env.local" ]; then cp "$temp_root/frontend.env.local" frontend/.env.local; fi if [ -d "$temp_root/frontend-node_modules" ]; then mv "$temp_root/frontend-node_modules" frontend/node_modules; fi test -f backend/app/main.py test -f backend/migrations/env.py test -f frontend/package.json docker run --rm -v "$project/frontend:/app" -w /app "$frontend_image" sh -lc 'if [ ! -d node_modules ]; then npm ci; fi; npm run build' docker rm -f "$backend_container" >/dev/null 2>&1 || true docker rm -f "$frontend_container" >/dev/null 2>&1 || true if [ "$skip_legal_change_worker" != "true" ]; then docker rm -f "$worker_container" >/dev/null 2>&1 || true fi mkdir -p "$project/storage" docker run -d \ --name "$backend_container" \ --restart unless-stopped \ --network host \ --env-file "$project/backend/.env" \ -v "$project/backend:/app" \ -v "$project/storage:/data/ai-contract-storage" \ -w /app \ "$backend_image" \ sh -lc "alembic upgrade head && python -m uvicorn app.main:app --host 0.0.0.0 --port $backend_port" docker run -d \ --name "$frontend_container" \ --restart unless-stopped \ -p "$frontend_host_port:$frontend_container_port" \ -v "$project/frontend:/app" \ -w /app \ "$frontend_image" \ sh -lc "npm run start -- --hostname 0.0.0.0 --port $frontend_container_port" if [ "$skip_legal_change_worker" != "true" ]; then docker run -d \ --name "$worker_container" \ --restart unless-stopped \ --network host \ --env-file "$project/backend/.env" \ -e LEGAL_CHANGE_IMPACT_WORKER_ENABLED=true \ -e LEGAL_CHANGE_IMPACT_WORKER_BATCH_SIZE="$worker_limit" \ -e LEGAL_CHANGE_IMPACT_WORKER_INTERVAL_SECONDS="$worker_interval_seconds" \ -e LEGAL_CHANGE_IMPACT_SEMANTIC_ENABLED=true \ -v "$project/backend:/app" \ -v "$project/storage:/data/ai-contract-storage" \ -w /app \ "$backend_image" \ python -m app.tasks.legal_change_impact_worker --limit "$worker_limit" --interval-seconds "$worker_interval_seconds" sleep 2 if ! docker ps --filter "name=$worker_container" --filter "status=running" --format '{{.Names}}' | grep -Fx "$worker_container" >/dev/null; then docker logs --tail 120 "$worker_container" || true exit 32 fi fi for i in $(seq 1 40); do if curl -fsS "http://127.0.0.1:$backend_port/health" >/dev/null; then break fi if [ "$i" = "40" ]; then docker logs --tail 120 "$backend_container" || true exit 30 fi sleep 2 done for i in $(seq 1 40); do if curl -fsSI "http://127.0.0.1:$frontend_host_port$smoke_frontend_path" >/dev/null; then break fi if [ "$i" = "40" ]; then docker logs --tail 120 "$frontend_container" || true exit 31 fi sleep 2 done docker exec "$backend_container" sh -lc 'cd /app && alembic current' > "$backup_dir/alembic-current-after-deploy.txt" docker ps -a --filter "name=$backend_container" --filter "name=$frontend_container" --filter "name=$worker_container" > "$backup_dir/docker-ps-after-deploy.txt" if [ "$keep_remote_archive" != "true" ]; then rm -f "$backend_archive" "$frontend_archive" fi rm -rf "$temp_root" echo "REMOTE_RESTORE_DEPLOY_BACKUP=$backup_dir" '@ $replacements = @{ "__REMOTE_PROJECT_PATH__" = ConvertTo-ShellSingleQuote $RemoteProjectPath "__REMOTE_BACKEND_ARCHIVE__" = ConvertTo-ShellSingleQuote $remoteBackendArchive "__REMOTE_FRONTEND_ARCHIVE__" = ConvertTo-ShellSingleQuote $remoteFrontendArchive "__REMOTE_BACKUP_ROOT__" = ConvertTo-ShellSingleQuote $RemoteBackupRoot "__STAMP__" = ConvertTo-ShellSingleQuote $timestamp "__BACKEND_CONTAINER__" = ConvertTo-ShellSingleQuote $BackendContainerName "__FRONTEND_CONTAINER__" = ConvertTo-ShellSingleQuote $FrontendContainerName "__WORKER_CONTAINER__" = ConvertTo-ShellSingleQuote $WorkerContainerName "__BACKEND_IMAGE__" = ConvertTo-ShellSingleQuote $BackendImage "__FRONTEND_IMAGE__" = ConvertTo-ShellSingleQuote $FrontendImage "__BACKEND_PORT__" = $BackendPort.ToString() "__FRONTEND_HOST_PORT__" = $FrontendHostPort.ToString() "__FRONTEND_CONTAINER_PORT__" = $FrontendContainerPort.ToString() "__WORKER_LIMIT__" = $WorkerLimit.ToString() "__WORKER_INTERVAL_SECONDS__" = $WorkerIntervalSeconds.ToString() "__SMOKE_FRONTEND_PATH__" = ConvertTo-ShellSingleQuote $SmokeFrontendPath "__SKIP_LEGAL_CHANGE_WORKER__" = ConvertTo-ShellSingleQuote ([bool]$SkipLegalChangeWorker).ToString().ToLowerInvariant() "__KEEP_REMOTE_ARCHIVE__" = ConvertTo-ShellSingleQuote ([bool]$KeepRemoteArchive).ToString().ToLowerInvariant() } foreach ($key in $replacements.Keys) { $remoteShell = $remoteShell.Replace($key, [string]$replacements[$key]) } return ($remoteShell -replace "`r?`n", "`n") } if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing" } else { Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK" } if ($UseSshpass -or -not [string]::IsNullOrWhiteSpace($env:SSHPASS)) { $script:SshpassPath = Resolve-SshpassPath if ([string]::IsNullOrWhiteSpace($script:SshpassPath)) { Add-Check -Key "ssh_auth" -Label "SSH authentication" -Status "fail" -Detail "sshpass.exe not found; install sshpass or use SSH key auth" } elseif ([string]::IsNullOrWhiteSpace($env:SSHPASS)) { Add-Check -Key "ssh_auth" -Label "SSH authentication" -Status "fail" -Detail "SSHPASS environment variable is empty" } else { $script:AuthMode = "sshpass" Add-Check -Key "ssh_auth" -Label "SSH authentication" -Status "pass" -Detail "using sshpass from environment variable" -Meta @{ auth_mode = "sshpass" } } } elseif (Test-Path $SshKeyPath) { $script:AuthMode = "key" Add-Check -Key "ssh_auth" -Label "SSH authentication" -Status "pass" -Detail "using SSH key" -Meta @{ auth_mode = "key"; key = $SshKeyPath } } else { Add-Check -Key "ssh_auth" -Label "SSH authentication" -Status "fail" -Detail "SSH key not found and SSHPASS is not configured" } $branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim() $commit = (git -C $repoRoot rev-parse HEAD).Trim() $remoteUrl = (git -C $repoRoot remote get-url origin).Trim() $gitStatus = git -C $repoRoot status --short ($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath if ($gitStatus) { if ($AllowDirtyWorktree) { Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "working tree has uncommitted changes; deploy allowed by -AllowDirtyWorktree" -Meta @{ artifact = $gitStatusPath } } else { Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "working tree has uncommitted changes; pass -AllowDirtyWorktree for development server deploy" -Meta @{ artifact = $gitStatusPath } } } else { Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "working tree is clean" -Meta @{ artifact = $gitStatusPath } } if ($remoteUrl -match "192\.168\.3\.90") { Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Required $false -Detail "origin points to 192.168.3.90" -Meta @{ remote = $remoteUrl } } else { Add-Check -Key "origin_remote" -Label "origin remote" -Status "warn" -Required $false -Detail "origin does not point to 192.168.3.90; restore deploy can still package the local worktree" -Meta @{ remote = $remoteUrl } } $failedPreflight = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedPreflight.Count -eq 0) { $remoteProjectLiteral = ConvertTo-ShellSingleQuote $RemoteProjectPath $backendContainerLiteral = ConvertTo-ShellSingleQuote $BackendContainerName $frontendContainerLiteral = ConvertTo-ShellSingleQuote $FrontendContainerName $backendImageLiteral = ConvertTo-ShellSingleQuote $BackendImage $frontendImageLiteral = ConvertTo-ShellSingleQuote $FrontendImage $preflightCommand = @" set -euo pipefail project=$remoteProjectLiteral backend_container=$backendContainerLiteral frontend_container=$frontendContainerLiteral worker_container=$(ConvertTo-ShellSingleQuote $WorkerContainerName) backend_image=$backendImageLiteral frontend_image=$frontendImageLiteral test -d "`$project" test -d "`$project/backend" test -d "`$project/frontend" test -f "`$project/backend/app/main.py" test -f "`$project/frontend/package.json" test -f "`$project/backend/.env" docker image inspect "`$backend_image" >/dev/null docker image inspect "`$frontend_image" >/dev/null docker ps -a --filter "name=`$backend_container" --format '{{.Names}} {{.Status}}' docker ps -a --filter "name=`$frontend_container" --format '{{.Names}} {{.Status}} {{.Ports}}' docker ps -a --filter "name=`$worker_container" --format '{{.Names}} {{.Status}}' "@ -replace "`r?`n", "; " $remotePreflight = Invoke-Remote -Name "01-remote-restore-preflight" -Command $preflightCommand if ($remotePreflight.exit_code -eq 0) { Add-Check -Key "remote_preflight" -Label "remote restore preflight" -Status "pass" -Detail "remote project path, images and containers are compatible" -Meta @{ log = $remotePreflight.log } } else { Add-Check -Key "remote_preflight" -Label "remote restore preflight" -Status "fail" -Detail "remote restore preflight failed" -Meta @{ log = $remotePreflight.log } } } if ($PlanOnly) { Add-Check -Key "deployment" -Label "restore deployment" -Status "skip" -Required $false -Detail "skipped by -PlanOnly" } else { $failedBeforeArchive = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeArchive.Count -eq 0) { $backendArchive = Invoke-LocalStep -Name "02-create-backend-archive" -Script { Push-Location $repoRoot try { & tar -czf $localBackendArchive ` --exclude=backend/.venv ` --exclude=backend/.pytest_cache ` --exclude=backend/__pycache__ ` --exclude=backend/app/__pycache__ ` --exclude=backend/tests/__pycache__ ` backend } finally { Pop-Location } } if ($backendArchive.exit_code -eq 0 -and (Test-Path $localBackendArchive)) { Add-Check -Key "backend_archive" -Label "backend archive" -Status "pass" -Detail "backend archive created" -Meta @{ archive = $localBackendArchive; bytes = (Get-Item $localBackendArchive).Length; log = $backendArchive.log } } else { Add-Check -Key "backend_archive" -Label "backend archive" -Status "fail" -Detail "backend archive creation failed" -Meta @{ log = $backendArchive.log } } $frontendArchive = Invoke-LocalStep -Name "03-create-frontend-archive" -Script { Push-Location $repoRoot try { & tar -czf $localFrontendArchive ` --exclude=frontend/node_modules ` --exclude=frontend/.next ` --exclude=frontend/.next-dev-log ` frontend } finally { Pop-Location } } if ($frontendArchive.exit_code -eq 0 -and (Test-Path $localFrontendArchive)) { Add-Check -Key "frontend_archive" -Label "frontend archive" -Status "pass" -Detail "frontend archive created" -Meta @{ archive = $localFrontendArchive; bytes = (Get-Item $localFrontendArchive).Length; log = $frontendArchive.log } } else { Add-Check -Key "frontend_archive" -Label "frontend archive" -Status "fail" -Detail "frontend archive creation failed" -Meta @{ log = $frontendArchive.log } } } $failedBeforeUpload = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeUpload.Count -eq 0) { $remoteScriptText = New-RemoteDeployScript $utf8NoBom = New-Object System.Text.UTF8Encoding($false) [System.IO.File]::WriteAllText($localRemoteScript, $remoteScriptText, $utf8NoBom) $uploadBackend = Invoke-ScpToRemote -Name "04-upload-backend-archive" -LocalPath $localBackendArchive -RemotePath $remoteBackendArchive if ($uploadBackend.exit_code -eq 0) { Add-Check -Key "upload_backend" -Label "upload backend archive" -Status "pass" -Detail "backend archive uploaded" -Meta @{ remote_archive = $remoteBackendArchive; log = $uploadBackend.log } } else { Add-Check -Key "upload_backend" -Label "upload backend archive" -Status "fail" -Detail "backend archive upload failed" -Meta @{ log = $uploadBackend.log } } $uploadFrontend = Invoke-ScpToRemote -Name "05-upload-frontend-archive" -LocalPath $localFrontendArchive -RemotePath $remoteFrontendArchive if ($uploadFrontend.exit_code -eq 0) { Add-Check -Key "upload_frontend" -Label "upload frontend archive" -Status "pass" -Detail "frontend archive uploaded" -Meta @{ remote_archive = $remoteFrontendArchive; log = $uploadFrontend.log } } else { Add-Check -Key "upload_frontend" -Label "upload frontend archive" -Status "fail" -Detail "frontend archive upload failed" -Meta @{ log = $uploadFrontend.log } } $uploadScript = Invoke-ScpToRemote -Name "06-upload-remote-script" -LocalPath $localRemoteScript -RemotePath $remoteScriptPath if ($uploadScript.exit_code -eq 0) { Add-Check -Key "upload_remote_script" -Label "upload remote script" -Status "pass" -Detail "remote restore script uploaded" -Meta @{ remote_script = $remoteScriptPath; log = $uploadScript.log } } else { Add-Check -Key "upload_remote_script" -Label "upload remote script" -Status "fail" -Detail "remote restore script upload failed" -Meta @{ log = $uploadScript.log } } } $failedBeforeDeploy = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeDeploy.Count -eq 0) { $deployCommand = "bash $(ConvertTo-ShellSingleQuote $remoteScriptPath)" $remoteDeploy = Invoke-Remote -Name "07-remote-restore-deploy" -Command $deployCommand $backupMatch = [regex]::Match($remoteDeploy.output, 'REMOTE_RESTORE_DEPLOY_BACKUP=([^\r\n]+)') $remoteBackupDir = if ($backupMatch.Success) { $backupMatch.Groups[1].Value.Trim() } else { "$RemoteBackupRoot/$timestamp" } if ($remoteDeploy.exit_code -eq 0) { Add-Check -Key "remote_deploy" -Label "remote restore deploy" -Status "pass" -Detail "remote restore deploy completed" -Meta @{ remote_backup = $remoteBackupDir; log = $remoteDeploy.log } } else { Add-Check -Key "remote_deploy" -Label "remote restore deploy" -Status "fail" -Detail "remote restore deploy failed" -Meta @{ remote_backup = $remoteBackupDir; log = $remoteDeploy.log } } } } if ($SkipSmoke -or $PlanOnly) { Add-Check -Key "local_smoke" -Label "local HTTP smoke" -Status "skip" -Required $false -Detail "skipped by option" } else { $failedBeforeSmoke = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) if ($failedBeforeSmoke.Count -eq 0) { $healthSmoke = Invoke-LocalStep -Name "08-local-health-smoke" -Script { Invoke-WebRequest -UseBasicParsing -Uri $HealthUrl -TimeoutSec 15 | Select-Object StatusCode, Content } $frontendSmokeUrl = "$FrontendBaseUrl$SmokeFrontendPath" $frontendSmoke = Invoke-LocalStep -Name "09-local-frontend-smoke" -Script { Invoke-WebRequest -UseBasicParsing -Uri $frontendSmokeUrl -Method Head -TimeoutSec 15 | Select-Object StatusCode } if ($healthSmoke.exit_code -eq 0 -and $frontendSmoke.exit_code -eq 0) { Add-Check -Key "local_smoke" -Label "local HTTP smoke" -Status "pass" -Detail "health and frontend route returned successfully" -Meta @{ health_url = $HealthUrl; frontend_url = $frontendSmokeUrl; health_log = $healthSmoke.log; frontend_log = $frontendSmoke.log } } else { Add-Check -Key "local_smoke" -Label "local HTTP smoke" -Status "fail" -Detail "health or frontend route smoke failed" -Meta @{ health_url = $HealthUrl; frontend_url = $frontendSmokeUrl; health_log = $healthSmoke.log; frontend_log = $frontendSmoke.log } } } } $failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) $warningsList = @($checks | Where-Object { $_.status -eq "warn" }) $skippedList = @($checks | Where-Object { $_.status -eq "skip" }) $overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" } $summary = [ordered]@{ name = "restore-docker-deploy" status = $overallStatus generated_at = (Get-Date).ToUniversalTime().ToString("o") artifact_dir = $artifactDir branch = $branch commit = $commit remote = $remoteUrl auth_mode = $script:AuthMode local_backend_archive = $localBackendArchive local_frontend_archive = $localFrontendArchive remote_backend_archive = $remoteBackendArchive remote_frontend_archive = $remoteFrontendArchive remote_script = $remoteScriptPath targets = [ordered]@{ server_host = $ServerHost remote_project_path = $RemoteProjectPath remote_backup_root = $RemoteBackupRoot backend_container = $BackendContainerName frontend_container = $FrontendContainerName worker_container = $WorkerContainerName backend_image = $BackendImage frontend_image = $FrontendImage backend_base_url = $BackendBaseUrl frontend_base_url = $FrontendBaseUrl smoke_frontend_path = $SmokeFrontendPath } options = [ordered]@{ allow_dirty_worktree = [bool]$AllowDirtyWorktree use_sshpass = [bool]$UseSshpass plan_only = [bool]$PlanOnly skip_smoke = [bool]$SkipSmoke skip_legal_change_worker = [bool]$SkipLegalChangeWorker worker_limit = $WorkerLimit worker_interval_seconds = $WorkerIntervalSeconds keep_remote_archive = [bool]$KeepRemoteArchive } checks = $checks totals = [ordered]@{ total = $checks.Count failed_required = $failedRequired.Count warnings = $warningsList.Count skipped = $skippedList.Count } } Save-Json -Path $summaryJson -Value $summary $markdown = New-Object System.Collections.Generic.List[string] $markdown.Add("# Restore Docker Deploy") | Out-Null $markdown.Add("") | Out-Null $markdown.Add("- Status: $overallStatus") | Out-Null $markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null $markdown.Add("- Branch: $branch") | Out-Null $markdown.Add("- Commit: $commit") | Out-Null $markdown.Add("- Artifact dir: $artifactDir") | Out-Null $markdown.Add("- Remote project path: $RemoteProjectPath") | Out-Null $markdown.Add("- Backend container: $BackendContainerName") | Out-Null $markdown.Add("- Frontend container: $FrontendContainerName") | Out-Null $markdown.Add("- Legal change worker container: $WorkerContainerName") | Out-Null $markdown.Add("") | Out-Null $markdown.Add("| Check | Required | Status | Detail |") | Out-Null $markdown.Add("|---|---:|---|---|") | Out-Null foreach ($check in $checks) { $markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null } $markdown.Add("") | Out-Null $markdown.Add("## Artifacts") | Out-Null $markdown.Add("") | Out-Null $markdown.Add('- Summary JSON: `restore-docker-deploy-summary.json`') | Out-Null $markdown.Add('- Summary Markdown: `restore-docker-deploy-summary.md`') | Out-Null $markdown.Add('- Git status: `git-status.txt`') | Out-Null foreach ($check in $checks) { if ($check.meta.log) { $markdown.Add(('- {0} log: `{1}`' -f $check.label, $check.meta.log)) | Out-Null } if ($check.meta.remote_backup) { $markdown.Add(('- {0} backup: `{1}`' -f $check.label, $check.meta.remote_backup)) | Out-Null } } $markdown | Set-Content -Encoding UTF8 -Path $summaryMd Write-Host "RESTORE_DOCKER_DEPLOY_$overallStatus $artifactDir" if ($overallStatus -eq "FAIL") { throw "Restore Docker deploy failed. See $summaryMd" }