chore: add production health check automation
This commit is contained in:
Codex
@@ -0,0 +1,333 @@
|
||||
param(
|
||||
[string]$ServerHost = "192.168.3.90",
|
||||
[string]$BackendBaseUrl = "http://192.168.3.90:8001",
|
||||
[string]$FrontendBaseUrl = "http://192.168.3.90:3003",
|
||||
[string]$SshUser = "root",
|
||||
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
|
||||
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
|
||||
[string]$ComposeFile = "docker-compose.server.yml",
|
||||
[string]$ArtifactRoot = "",
|
||||
[string]$AdminAccount = "admin@example.com",
|
||||
[string]$DefaultPassword = "123456",
|
||||
[switch]$SkipSsh
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
|
||||
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss"
|
||||
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
||||
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase5-production-health"
|
||||
}
|
||||
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
||||
$summaryJson = Join-Path $artifactDir "production-health-summary.json"
|
||||
$summaryMd = Join-Path $artifactDir "production-health-summary.md"
|
||||
$containerStatusPath = Join-Path $artifactDir "container-status.txt"
|
||||
$readinessPath = Join-Path $artifactDir "readiness.json"
|
||||
$healthPath = Join-Path $artifactDir "health.json"
|
||||
$defaultLoginPath = Join-Path $artifactDir "default-password-login.json"
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
||||
|
||||
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
||||
throw "gstack is required for all AI-assisted work in this repo."
|
||||
}
|
||||
|
||||
Add-Type -AssemblyName System.Net.Http
|
||||
|
||||
$checks = New-Object System.Collections.Generic.List[object]
|
||||
|
||||
function Add-Check {
|
||||
param(
|
||||
[string]$Key,
|
||||
[string]$Label,
|
||||
[ValidateSet("pass", "fail", "warn")]
|
||||
[string]$Status,
|
||||
[string]$Detail,
|
||||
[bool]$Required = $true,
|
||||
[hashtable]$Meta = @{}
|
||||
)
|
||||
|
||||
$checks.Add([ordered]@{
|
||||
key = $Key
|
||||
label = $Label
|
||||
status = $Status
|
||||
required = $Required
|
||||
detail = $Detail
|
||||
meta = $Meta
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
function ConvertTo-SafeFileName {
|
||||
param([string]$Value)
|
||||
return ($Value -replace '[^a-zA-Z0-9._-]', '-').Trim("-")
|
||||
}
|
||||
|
||||
function Save-Json {
|
||||
param(
|
||||
[string]$Path,
|
||||
[object]$Value
|
||||
)
|
||||
$Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
|
||||
}
|
||||
|
||||
function Invoke-HttpProbe {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Url,
|
||||
[object]$Body = $null
|
||||
)
|
||||
|
||||
$handler = New-Object System.Net.Http.HttpClientHandler
|
||||
$client = New-Object System.Net.Http.HttpClient($handler)
|
||||
$client.Timeout = [TimeSpan]::FromSeconds(15)
|
||||
try {
|
||||
$httpMethod = New-Object System.Net.Http.HttpMethod($Method)
|
||||
$request = New-Object System.Net.Http.HttpRequestMessage($httpMethod, $Url)
|
||||
if ($null -ne $Body) {
|
||||
$json = $Body | ConvertTo-Json -Depth 8 -Compress
|
||||
$request.Content = New-Object System.Net.Http.StringContent($json, [System.Text.Encoding]::UTF8, "application/json")
|
||||
}
|
||||
|
||||
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
||||
$text = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
||||
$parsed = $null
|
||||
if (-not [string]::IsNullOrWhiteSpace($text)) {
|
||||
try {
|
||||
$parsed = $text | ConvertFrom-Json -ErrorAction Stop
|
||||
} catch {
|
||||
$parsed = $null
|
||||
}
|
||||
}
|
||||
|
||||
return [ordered]@{
|
||||
ok = $true
|
||||
url = $Url
|
||||
method = $Method
|
||||
status_code = [int]$response.StatusCode
|
||||
body = $parsed
|
||||
body_text = if ($parsed) { $null } else { $text }
|
||||
}
|
||||
} catch {
|
||||
return [ordered]@{
|
||||
ok = $false
|
||||
url = $Url
|
||||
method = $Method
|
||||
status_code = $null
|
||||
error = $_.Exception.Message
|
||||
}
|
||||
} finally {
|
||||
$client.Dispose()
|
||||
$handler.Dispose()
|
||||
}
|
||||
}
|
||||
|
||||
function Test-TcpPort {
|
||||
param(
|
||||
[string]$HostName,
|
||||
[int]$Port,
|
||||
[int]$TimeoutMs = 3000
|
||||
)
|
||||
|
||||
$client = New-Object System.Net.Sockets.TcpClient
|
||||
try {
|
||||
$async = $client.BeginConnect($HostName, $Port, $null, $null)
|
||||
$connected = $async.AsyncWaitHandle.WaitOne($TimeoutMs, $false)
|
||||
if (-not $connected) {
|
||||
return $false
|
||||
}
|
||||
$client.EndConnect($async)
|
||||
return $client.Connected
|
||||
} catch {
|
||||
return $false
|
||||
} finally {
|
||||
$client.Close()
|
||||
}
|
||||
}
|
||||
|
||||
function Get-UriHostAndPort {
|
||||
param(
|
||||
[string]$Url,
|
||||
[int]$DefaultPort
|
||||
)
|
||||
$uri = [Uri]$Url
|
||||
$port = if ($uri.IsDefaultPort) { $DefaultPort } else { $uri.Port }
|
||||
return [ordered]@{
|
||||
host = $uri.Host
|
||||
port = $port
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-SshCommand {
|
||||
param([string]$Command)
|
||||
|
||||
$target = "${SshUser}@${ServerHost}"
|
||||
$args = @(
|
||||
"-i", $SshKeyPath,
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "ConnectTimeout=10",
|
||||
$target,
|
||||
$Command
|
||||
)
|
||||
$output = & ssh @args 2>&1
|
||||
return [ordered]@{
|
||||
exit_code = $LASTEXITCODE
|
||||
output = ($output -join [Environment]::NewLine)
|
||||
}
|
||||
}
|
||||
|
||||
$backendTarget = Get-UriHostAndPort -Url $BackendBaseUrl -DefaultPort 80
|
||||
$frontendTarget = Get-UriHostAndPort -Url $FrontendBaseUrl -DefaultPort 80
|
||||
|
||||
if ($SkipSsh) {
|
||||
Add-Check -Key "container_status" -Label "Docker containers" -Status "warn" -Required $false -Detail "Skipped by -SkipSsh"
|
||||
} elseif (-not (Test-Path $SshKeyPath)) {
|
||||
Add-Check -Key "container_status" -Label "Docker containers" -Status "fail" -Detail "SSH key not found: $SshKeyPath"
|
||||
} else {
|
||||
$remoteCommand = "cd $RemoteProjectPath; docker compose -f $ComposeFile ps"
|
||||
$sshResult = Invoke-SshCommand -Command $remoteCommand
|
||||
$sshResult.output | Set-Content -Encoding UTF8 -Path $containerStatusPath
|
||||
$hasBackend = $sshResult.output -match "backend"
|
||||
$hasFrontend = $sshResult.output -match "frontend"
|
||||
$looksRunning = $sshResult.output -match "running|Up|healthy"
|
||||
if ($sshResult.exit_code -eq 0 -and $hasBackend -and $hasFrontend -and $looksRunning) {
|
||||
Add-Check -Key "container_status" -Label "Docker containers" -Status "pass" -Detail "backend/frontend containers are visible and running" -Meta @{
|
||||
artifact = $containerStatusPath
|
||||
}
|
||||
} else {
|
||||
Add-Check -Key "container_status" -Label "Docker containers" -Status "fail" -Detail "Could not confirm backend/frontend running state, SSH exit=$($sshResult.exit_code)" -Meta @{
|
||||
artifact = $containerStatusPath
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($target in @(
|
||||
@{ key = "backend_port"; label = "Backend port"; host = $backendTarget.host; port = $backendTarget.port },
|
||||
@{ key = "frontend_port"; label = "Frontend port"; host = $frontendTarget.host; port = $frontendTarget.port }
|
||||
)) {
|
||||
$portOpen = Test-TcpPort -HostName $target.host -Port $target.port
|
||||
if ($portOpen) {
|
||||
Add-Check -Key $target.key -Label $target.label -Status "pass" -Detail "$($target.host):$($target.port) accepts TCP connections"
|
||||
} else {
|
||||
Add-Check -Key $target.key -Label $target.label -Status "fail" -Detail "$($target.host):$($target.port) does not accept TCP connections"
|
||||
}
|
||||
}
|
||||
|
||||
$health = Invoke-HttpProbe -Method "GET" -Url "$BackendBaseUrl/health"
|
||||
Save-Json -Path $healthPath -Value $health
|
||||
if ($health.ok -and $health.status_code -eq 200) {
|
||||
Add-Check -Key "backend_health" -Label "Backend /health" -Status "pass" -Detail "/health returned 200" -Meta @{ artifact = $healthPath }
|
||||
} else {
|
||||
Add-Check -Key "backend_health" -Label "Backend /health" -Status "fail" -Detail "/health did not return 200" -Meta @{
|
||||
status_code = $health.status_code
|
||||
artifact = $healthPath
|
||||
}
|
||||
}
|
||||
|
||||
$readiness = Invoke-HttpProbe -Method "GET" -Url "$BackendBaseUrl/api/v1/system/readiness"
|
||||
Save-Json -Path $readinessPath -Value $readiness
|
||||
$readinessData = $readiness.body.data
|
||||
$passwordRotated = $false
|
||||
if ($readinessData -and $readinessData.pilot_assets -and $readinessData.pilot_assets.default_admin) {
|
||||
$passwordRotated = [bool]$readinessData.pilot_assets.default_admin.password_rotated
|
||||
}
|
||||
$warnings = if ($readinessData -and $readinessData.summary) { [int]$readinessData.summary.warnings } else { -1 }
|
||||
if ($readiness.ok -and $readiness.status_code -eq 200 -and $readiness.body.code -eq 0 -and $readinessData.status -eq "ready" -and $warnings -eq 0 -and $passwordRotated) {
|
||||
Add-Check -Key "system_readiness" -Label "System readiness" -Status "pass" -Detail "readiness=ready, warnings=0, password_rotated=true" -Meta @{ artifact = $readinessPath }
|
||||
} else {
|
||||
Add-Check -Key "system_readiness" -Label "System readiness" -Status "fail" -Detail "readiness did not satisfy ready/warnings=0/password_rotated=true" -Meta @{
|
||||
status_code = $readiness.status_code
|
||||
readiness_status = $readinessData.status
|
||||
warnings = $warnings
|
||||
password_rotated = $passwordRotated
|
||||
artifact = $readinessPath
|
||||
}
|
||||
}
|
||||
|
||||
foreach ($page in @("/login", "/pilot-readiness")) {
|
||||
$url = "$FrontendBaseUrl$page"
|
||||
$probe = Invoke-HttpProbe -Method "GET" -Url $url
|
||||
$fileName = "page-$((ConvertTo-SafeFileName -Value $page)).json"
|
||||
$pagePath = Join-Path $artifactDir $fileName
|
||||
Save-Json -Path $pagePath -Value $probe
|
||||
$key = "frontend_page_$((ConvertTo-SafeFileName -Value $page).ToLowerInvariant())"
|
||||
if ($probe.ok -and $probe.status_code -eq 200) {
|
||||
Add-Check -Key $key -Label "Frontend$page" -Status "pass" -Detail "$page returned 200" -Meta @{ artifact = $pagePath }
|
||||
} else {
|
||||
Add-Check -Key $key -Label "Frontend$page" -Status "fail" -Detail "$page did not return 200" -Meta @{
|
||||
status_code = $probe.status_code
|
||||
artifact = $pagePath
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$defaultLogin = Invoke-HttpProbe -Method "POST" -Url "$BackendBaseUrl/api/v1/auth/login" -Body @{
|
||||
account = $AdminAccount
|
||||
password = $DefaultPassword
|
||||
}
|
||||
Save-Json -Path $defaultLoginPath -Value $defaultLogin
|
||||
if ($defaultLogin.ok -and $defaultLogin.status_code -eq 401) {
|
||||
Add-Check -Key "default_password_rejected" -Label "Default password rejection" -Status "pass" -Detail "Default admin password login returned 401" -Meta @{ artifact = $defaultLoginPath }
|
||||
} else {
|
||||
Add-Check -Key "default_password_rejected" -Label "Default password rejection" -Status "fail" -Detail "Default admin password was not clearly rejected" -Meta @{
|
||||
status_code = $defaultLogin.status_code
|
||||
artifact = $defaultLoginPath
|
||||
}
|
||||
}
|
||||
|
||||
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
||||
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
||||
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }
|
||||
|
||||
$summary = [ordered]@{
|
||||
name = "phase5-production-health"
|
||||
status = $overallStatus
|
||||
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
artifact_dir = $artifactDir
|
||||
targets = [ordered]@{
|
||||
server_host = $ServerHost
|
||||
backend_base_url = $BackendBaseUrl
|
||||
frontend_base_url = $FrontendBaseUrl
|
||||
remote_project_path = $RemoteProjectPath
|
||||
compose_file = $ComposeFile
|
||||
}
|
||||
checks = $checks
|
||||
totals = [ordered]@{
|
||||
total = $checks.Count
|
||||
failed_required = $failedRequired.Count
|
||||
warnings = $warningsList.Count
|
||||
}
|
||||
}
|
||||
Save-Json -Path $summaryJson -Value $summary
|
||||
|
||||
$markdown = New-Object System.Collections.Generic.List[string]
|
||||
$markdown.Add("# Phase 5 Production Health Check") | Out-Null
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add("- Status: $overallStatus") | Out-Null
|
||||
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
||||
$markdown.Add("- Backend: $BackendBaseUrl") | Out-Null
|
||||
$markdown.Add("- Frontend: $FrontendBaseUrl") | Out-Null
|
||||
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
||||
$markdown.Add("|---|---:|---|---|") | Out-Null
|
||||
foreach ($check in $checks) {
|
||||
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
||||
}
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add("Raw artifacts:") | Out-Null
|
||||
$markdown.Add("") | Out-Null
|
||||
$markdown.Add('- `production-health-summary.json`') | Out-Null
|
||||
$markdown.Add('- `readiness.json`') | Out-Null
|
||||
$markdown.Add('- `health.json`') | Out-Null
|
||||
$markdown.Add('- `default-password-login.json`') | Out-Null
|
||||
if (Test-Path $containerStatusPath) {
|
||||
$markdown.Add('- `container-status.txt`') | Out-Null
|
||||
}
|
||||
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
||||
|
||||
Write-Host "PHASE5_PRODUCTION_HEALTH_$overallStatus $artifactDir"
|
||||
if ($overallStatus -eq "FAIL") {
|
||||
throw "Phase 5 production health check failed. See $summaryMd"
|
||||
}
|
||||
Reference in New Issue
Block a user