ai-contract-platform/scripts/run-production-ops-check.ps1

param(
  [string]$ServerHost = "192.168.3.90",
  [string]$SshUser = "root",
  [string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
  [string]$RemoteProjectPath = "/opt/ai-contract-platform",
  [string]$ComposeFile = "docker-compose.server.yml",
  [string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy",
  [string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/phase7-rollback",
  [string]$FrontendBaseUrl = "http://192.168.3.90:3003",
  [string]$BackendBaseUrl = "http://192.168.3.90:8001",
  [string]$HealthUrl = "http://192.168.3.90:8001/health",
  [string]$ReadinessUrl = "http://192.168.3.90:8001/api/v1/system/readiness",
  [string]$ArtifactRoot = "",
  [int]$DiskWarnPercent = 80,
  [int]$DiskFailPercent = 90,
  [int]$MinFrontendBackupCount = 1,
  [int]$MaxNewestBackupAgeHours = 168,
  [switch]$RequireCleanWorktree
)

$ErrorActionPreference = "Stop"

$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
  $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7-production-ops-check"
}
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
  $ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
}
$artifactDir = Join-Path $ArtifactRoot $timestamp
$summaryJson = Join-Path $artifactDir "production-ops-check-summary.json"
$summaryMd = Join-Path $artifactDir "production-ops-check-summary.md"
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
$remoteInfoPath = Join-Path $artifactDir "remote-ops-info.txt"
$healthPath = Join-Path $artifactDir "health.json"
$readinessPath = Join-Path $artifactDir "readiness.json"
$frontendPath = Join-Path $artifactDir "frontend-home.json"

New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null

$checks = New-Object System.Collections.Generic.List[object]

function Add-Check {
  param(
    [string]$Key,
    [string]$Label,
    [ValidateSet("pass", "fail", "warn", "skip")]
    [string]$Status,
    [string]$Detail,
    [bool]$Required = $true,
    [hashtable]$Meta = @{}
  )

  $checks.Add([ordered]@{
    key = $Key
    label = $Label
    status = $Status
    required = $Required
    detail = $Detail
    meta = $Meta
  }) | Out-Null
}

function Save-Json {
  param(
    [string]$Path,
    [object]$Value
  )
  $Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
}

function Invoke-HttpProbe {
  param(
    [string]$Method,
    [string]$Url
  )

  Add-Type -AssemblyName System.Net.Http
  $handler = New-Object System.Net.Http.HttpClientHandler
  $client = New-Object System.Net.Http.HttpClient($handler)
  $client.Timeout = [TimeSpan]::FromSeconds(15)
  try {
    $request = New-Object System.Net.Http.HttpRequestMessage((New-Object System.Net.Http.HttpMethod($Method)), $Url)
    $response = $client.SendAsync($request).GetAwaiter().GetResult()
    $text = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
    $parsed = $null
    if (-not [string]::IsNullOrWhiteSpace($text)) {
      try {
        $parsed = $text | ConvertFrom-Json -ErrorAction Stop
      } catch {
        $parsed = $null
      }
    }

    return [ordered]@{
      ok = $true
      url = $Url
      method = $Method
      status_code = [int]$response.StatusCode
      body = $parsed
      body_text = if ($parsed) { $null } else { $text.Substring(0, [Math]::Min(500, $text.Length)) }
    }
  } catch {
    return [ordered]@{
      ok = $false
      url = $Url
      method = $Method
      status_code = $null
      error = $_.Exception.Message
    }
  } finally {
    $client.Dispose()
    $handler.Dispose()
  }
}

function Invoke-Remote {
  param(
    [string]$Name,
    [string]$Command
  )

  $target = "${SshUser}@${ServerHost}"
  $logPath = Join-Path $artifactDir "$Name.log"
  $args = @(
    "-i", $SshKeyPath,
    "-o", "BatchMode=yes",
    "-o", "StrictHostKeyChecking=accept-new",
    "-o", "ConnectTimeout=10",
    $target,
    $Command
  )
  $previousErrorActionPreference = $ErrorActionPreference
  $ErrorActionPreference = "Continue"
  try {
    $output = & ssh @args 2>&1
    $exitCode = $LASTEXITCODE
  } finally {
    $ErrorActionPreference = $previousErrorActionPreference
  }
  $outputText = ($output -join [Environment]::NewLine)
  $outputText | Set-Content -Encoding UTF8 -Path $logPath
  return [ordered]@{
    exit_code = $exitCode
    output = $outputText
    log = $logPath
  }
}

function Convert-KeyValueOutput {
  param([string]$Text)

  $data = @{}
  foreach ($line in ($Text -split "`r?`n")) {
    if ($line -match '^([A-Z0-9_]+)=(.*)$') {
      $data[$matches[1]] = $matches[2]
    }
  }
  return $data
}

if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
  Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
} else {
  Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
}

if (-not (Test-Path $SshKeyPath)) {
  Add-Check -Key "ssh_key" -Label "SSH key" -Status "fail" -Detail "SSH key not found: $SshKeyPath"
} else {
  Add-Check -Key "ssh_key" -Label "SSH key" -Status "pass" -Detail "SSH key exists"
}

$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
$gitStatus = git -C $repoRoot status --short
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
if ($gitStatus) {
  if ($RequireCleanWorktree) {
    Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes and -RequireCleanWorktree is set" -Meta @{ artifact = $gitStatusPath }
  } else {
    Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; ops check is advisory" -Meta @{ artifact = $gitStatusPath }
  }
} else {
  Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
}

if ($remoteUrl -match "192\.168\.3\.90") {
  Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{
    branch = $branch
    commit = $commit
    remote = $remoteUrl
  }
} else {
  Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{
    branch = $branch
    commit = $commit
    remote = $remoteUrl
  }
}

$remoteScript = @"
set -euo pipefail
project='$RemoteProjectPath'
compose='$ComposeFile'
backup_root='$RemoteBackupRoot'
rollback_root='$RemoteRollbackRoot'
cd "`$project"
test "`$PWD" = "`$project"
test -f "`$compose"
echo "REMOTE_PROJECT_OK=1"
backend_state="`$(docker inspect -f '{{.State.Status}}' ai-contract-platform-backend-1 2>/dev/null || true)"
backend_health="`$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' ai-contract-platform-backend-1 2>/dev/null || true)"
frontend_state="`$(docker inspect -f '{{.State.Status}}' ai-contract-platform-frontend-1 2>/dev/null || true)"
postgres_state="`$(docker inspect -f '{{.State.Status}}' ai-contract-platform-postgres-1 2>/dev/null || true)"
postgres_health="`$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' ai-contract-platform-postgres-1 2>/dev/null || true)"
redis_state="`$(docker inspect -f '{{.State.Status}}' ai-contract-platform-redis-1 2>/dev/null || true)"
disk_used="`$(df -P "`$project" | awk 'NR==2 {print `$5}' | tr -d %)"
disk_available="`$(df -P "`$project" | awk 'NR==2 {print `$4}')"
echo "BACKEND_STATE=`$backend_state"
echo "BACKEND_HEALTH=`$backend_health"
echo "FRONTEND_STATE=`$frontend_state"
echo "POSTGRES_STATE=`$postgres_state"
echo "POSTGRES_HEALTH=`$postgres_health"
echo "REDIS_STATE=`$redis_state"
echo "DISK_USED_PERCENT=`$disk_used"
echo "DISK_AVAILABLE_KB=`$disk_available"
if ls "`$backup_root"/frontend-before-*.tgz >/dev/null 2>&1; then
  latest="`$(ls -1t "`$backup_root"/frontend-before-*.tgz | head -1)"
  backup_count="`$(find "`$backup_root" -maxdepth 1 -type f -name 'frontend-before-*.tgz' | wc -l | tr -d ' ')"
  backup_size="`$(stat -c %s "`$latest")"
  backup_epoch="`$(stat -c %Y "`$latest")"
  echo "FRONTEND_BACKUP_COUNT=`$backup_count"
  echo "FRONTEND_LATEST_BACKUP=`$latest"
  echo "FRONTEND_LATEST_BACKUP_SIZE=`$backup_size"
  echo "FRONTEND_LATEST_BACKUP_EPOCH=`$backup_epoch"
else
  echo "FRONTEND_BACKUP_COUNT=0"
  echo "FRONTEND_LATEST_BACKUP="
  echo "FRONTEND_LATEST_BACKUP_SIZE=0"
  echo "FRONTEND_LATEST_BACKUP_EPOCH=0"
fi
if [ -d "`$rollback_root" ]; then
  rollback_backup_count="`$(find "`$rollback_root" -maxdepth 1 -type f -name 'frontend-current-before-rollback-*.tgz' | wc -l | tr -d ' ')"
  echo "ROLLBACK_BACKUP_COUNT=`$rollback_backup_count"
else
  echo "ROLLBACK_BACKUP_COUNT=0"
fi
docker compose -f "`$compose" ps backend frontend postgres redis
"@

$remoteInfo = Invoke-Remote -Name "remote-ops-info" -Command $remoteScript
$remoteInfo.output | Set-Content -Encoding UTF8 -Path $remoteInfoPath
$remoteData = Convert-KeyValueOutput -Text $remoteInfo.output
if ($remoteInfo.exit_code -eq 0 -and $remoteData.REMOTE_PROJECT_OK -eq "1") {
  Add-Check -Key "remote_project" -Label "remote project" -Status "pass" -Detail "Remote project and compose file are accessible" -Meta @{ artifact = $remoteInfoPath }
} else {
  Add-Check -Key "remote_project" -Label "remote project" -Status "fail" -Detail "Remote project or compose preflight failed" -Meta @{
    exit_code = $remoteInfo.exit_code
    artifact = $remoteInfoPath
  }
}

$containerFailures = @()
foreach ($item in @(
  @{ key = "BACKEND_STATE"; expected = "running"; label = "backend" },
  @{ key = "FRONTEND_STATE"; expected = "running"; label = "frontend" },
  @{ key = "POSTGRES_STATE"; expected = "running"; label = "postgres" },
  @{ key = "REDIS_STATE"; expected = "running"; label = "redis" }
)) {
  if ($remoteData[$item.key] -ne $item.expected) {
    $containerFailures += "$($item.label)=$($remoteData[$item.key])"
  }
}
if ($remoteData.BACKEND_HEALTH -ne "healthy") {
  $containerFailures += "backend_health=$($remoteData.BACKEND_HEALTH)"
}
if ($remoteData.POSTGRES_HEALTH -ne "healthy") {
  $containerFailures += "postgres_health=$($remoteData.POSTGRES_HEALTH)"
}
if ($containerFailures.Count -eq 0) {
  Add-Check -Key "docker_services" -Label "Docker services" -Status "pass" -Detail "backend/frontend/postgres/redis states are healthy or running" -Meta @{ artifact = $remoteInfoPath }
} else {
  Add-Check -Key "docker_services" -Label "Docker services" -Status "fail" -Detail ("Unexpected container states: " + ($containerFailures -join ", ")) -Meta @{ artifact = $remoteInfoPath }
}

$diskUsed = if ($remoteData.DISK_USED_PERCENT) { [int]$remoteData.DISK_USED_PERCENT } else { 100 }
if ($diskUsed -ge $DiskFailPercent) {
  Add-Check -Key "disk_usage" -Label "Remote disk usage" -Status "fail" -Detail "Disk usage is $diskUsed%, fail threshold is $DiskFailPercent%" -Meta @{ artifact = $remoteInfoPath }
} elseif ($diskUsed -ge $DiskWarnPercent) {
  Add-Check -Key "disk_usage" -Label "Remote disk usage" -Status "warn" -Required $false -Detail "Disk usage is $diskUsed%, warn threshold is $DiskWarnPercent%" -Meta @{ artifact = $remoteInfoPath }
} else {
  Add-Check -Key "disk_usage" -Label "Remote disk usage" -Status "pass" -Detail "Disk usage is $diskUsed%" -Meta @{ artifact = $remoteInfoPath }
}

$backupCount = if ($remoteData.FRONTEND_BACKUP_COUNT) { [int]$remoteData.FRONTEND_BACKUP_COUNT } else { 0 }
$backupSize = if ($remoteData.FRONTEND_LATEST_BACKUP_SIZE) { [int64]$remoteData.FRONTEND_LATEST_BACKUP_SIZE } else { 0 }
if ($backupCount -lt $MinFrontendBackupCount -or $backupSize -le 0) {
  Add-Check -Key "frontend_backups" -Label "Frontend backups" -Status "fail" -Detail "Frontend backup count or latest backup size is insufficient" -Meta @{
    count = $backupCount
    latest = $remoteData.FRONTEND_LATEST_BACKUP
    latest_size = $backupSize
    artifact = $remoteInfoPath
  }
} else {
  $latestEpoch = if ($remoteData.FRONTEND_LATEST_BACKUP_EPOCH) { [int64]$remoteData.FRONTEND_LATEST_BACKUP_EPOCH } else { 0 }
  $latestAgeHours = if ($latestEpoch -gt 0) {
    [Math]::Round(((Get-Date).ToUniversalTime() - ([DateTimeOffset]::FromUnixTimeSeconds($latestEpoch).UtcDateTime)).TotalHours, 2)
  } else {
    999999
  }
  if ($latestAgeHours -gt $MaxNewestBackupAgeHours) {
    Add-Check -Key "frontend_backups" -Label "Frontend backups" -Status "warn" -Required $false -Detail "Latest frontend backup is $latestAgeHours hours old" -Meta @{
      count = $backupCount
      latest = $remoteData.FRONTEND_LATEST_BACKUP
      latest_size = $backupSize
      latest_age_hours = $latestAgeHours
      artifact = $remoteInfoPath
    }
  } else {
    Add-Check -Key "frontend_backups" -Label "Frontend backups" -Status "pass" -Detail "Frontend backup count=$backupCount, latest backup is $latestAgeHours hours old" -Meta @{
      count = $backupCount
      latest = $remoteData.FRONTEND_LATEST_BACKUP
      latest_size = $backupSize
      latest_age_hours = $latestAgeHours
      artifact = $remoteInfoPath
    }
  }
}

$health = Invoke-HttpProbe -Method "GET" -Url $HealthUrl
Save-Json -Path $healthPath -Value $health
if ($health.ok -and $health.status_code -eq 200) {
  Add-Check -Key "backend_health" -Label "Backend health endpoint" -Status "pass" -Detail "$HealthUrl returned 200" -Meta @{ artifact = $healthPath }
} else {
  Add-Check -Key "backend_health" -Label "Backend health endpoint" -Status "fail" -Detail "$HealthUrl did not return 200" -Meta @{
    status_code = $health.status_code
    artifact = $healthPath
  }
}

$readiness = Invoke-HttpProbe -Method "GET" -Url $ReadinessUrl
Save-Json -Path $readinessPath -Value $readiness
$readinessData = $readiness.body.data
$warnings = if ($readinessData -and $readinessData.summary) { [int]$readinessData.summary.warnings } else { -1 }
if ($readiness.ok -and $readiness.status_code -eq 200 -and $readiness.body.code -eq 0 -and $readinessData.status -eq "ready" -and $warnings -eq 0) {
  Add-Check -Key "system_readiness" -Label "System readiness" -Status "pass" -Detail "readiness=ready and warnings=0" -Meta @{ artifact = $readinessPath }
} else {
  Add-Check -Key "system_readiness" -Label "System readiness" -Status "fail" -Detail "readiness did not satisfy ready/warnings=0" -Meta @{
    status_code = $readiness.status_code
    readiness_status = $readinessData.status
    warnings = $warnings
    artifact = $readinessPath
  }
}

$frontend = Invoke-HttpProbe -Method "GET" -Url $FrontendBaseUrl
Save-Json -Path $frontendPath -Value $frontend
if ($frontend.ok -and $frontend.status_code -eq 200) {
  Add-Check -Key "frontend_home" -Label "Frontend home" -Status "pass" -Detail "$FrontendBaseUrl returned 200" -Meta @{ artifact = $frontendPath }
} else {
  Add-Check -Key "frontend_home" -Label "Frontend home" -Status "fail" -Detail "$FrontendBaseUrl did not return 200" -Meta @{
    status_code = $frontend.status_code
    artifact = $frontendPath
  }
}

$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
$skippedList = @($checks | Where-Object { $_.status -eq "skip" })
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }

$summary = [ordered]@{
  name = "phase7-production-ops-check"
  status = $overallStatus
  generated_at = (Get-Date).ToUniversalTime().ToString("o")
  artifact_dir = $artifactDir
  branch = $branch
  commit = $commit
  remote = $remoteUrl
  targets = [ordered]@{
    server_host = $ServerHost
    remote_project_path = $RemoteProjectPath
    compose_file = $ComposeFile
    frontend_base_url = $FrontendBaseUrl
    backend_base_url = $BackendBaseUrl
    remote_backup_root = $RemoteBackupRoot
    remote_rollback_root = $RemoteRollbackRoot
  }
  thresholds = [ordered]@{
    disk_warn_percent = $DiskWarnPercent
    disk_fail_percent = $DiskFailPercent
    min_frontend_backup_count = $MinFrontendBackupCount
    max_newest_backup_age_hours = $MaxNewestBackupAgeHours
  }
  checks = $checks
  totals = [ordered]@{
    total = $checks.Count
    failed_required = $failedRequired.Count
    warnings = $warningsList.Count
    skipped = $skippedList.Count
  }
}
Save-Json -Path $summaryJson -Value $summary

$markdown = New-Object System.Collections.Generic.List[string]
$markdown.Add("# Phase 7 Production Ops Check") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add("- Status: $overallStatus") | Out-Null
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
$markdown.Add("- Branch: $branch") | Out-Null
$markdown.Add("- Commit: $commit") | Out-Null
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
$markdown.Add("|---|---:|---|---|") | Out-Null
foreach ($check in $checks) {
  $markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
}
$markdown.Add("") | Out-Null
$markdown.Add("Local artifacts:") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add('- `production-ops-check-summary.json`') | Out-Null
$markdown.Add('- `production-ops-check-summary.md`') | Out-Null
$markdown.Add('- `git-status.txt`') | Out-Null
$markdown.Add('- `remote-ops-info.txt`') | Out-Null
$markdown.Add('- `health.json`') | Out-Null
$markdown.Add('- `readiness.json`') | Out-Null
$markdown.Add('- `frontend-home.json`') | Out-Null
$markdown.Add('- `*.log`') | Out-Null
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd

Write-Host "PHASE7_PRODUCTION_OPS_CHECK_$overallStatus $artifactDir"
if ($overallStatus -eq "FAIL") {
  throw "Phase 7 production ops check failed. See $summaryMd"
}