Codex
357 lines
14 KiB
PowerShell
357 lines
14 KiB
PowerShell
param(
|
|
[string]$ServerHost = "192.168.3.90",
|
|
[string]$SshUser = "root",
|
|
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
|
|
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
|
|
[string]$ComposeFile = "docker-compose.server.yml",
|
|
[string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy",
|
|
[string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/phase7-rollback",
|
|
[string]$FrontendBaseUrl = "http://192.168.3.90:3003",
|
|
[string]$BackendBaseUrl = "http://192.168.3.90:8001",
|
|
[string]$ApiUrl = "http://192.168.3.90:8001/api/v1",
|
|
[string]$HealthUrl = "http://192.168.3.90:8001/health",
|
|
[string]$ReadinessUrl = "http://192.168.3.90:8001/api/v1/system/readiness",
|
|
[string]$ArtifactRoot = "",
|
|
[int]$DiskWarnPercent = 80,
|
|
[int]$DiskFailPercent = 90,
|
|
[switch]$RequireCleanWorktree,
|
|
[switch]$AllowDirtyWorktree,
|
|
[switch]$IncludeReleaseGate,
|
|
[switch]$IncludeBackupDrill,
|
|
[switch]$SkipOpsCheck,
|
|
[switch]$SkipCapacityGovernance,
|
|
[switch]$FailOnWarn
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7-production-watchdog"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
|
|
}
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
$summaryJson = Join-Path $artifactDir "production-watchdog-summary.json"
|
|
$summaryMd = Join-Path $artifactDir "production-watchdog-summary.md"
|
|
$alertJson = Join-Path $artifactDir "production-watchdog-alert.json"
|
|
$alertMd = Join-Path $artifactDir "production-watchdog-alert.md"
|
|
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
|
|
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
|
|
function Add-Check {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[ValidateSet("pass", "fail", "warn", "skip")]
|
|
[string]$Status,
|
|
[string]$Detail,
|
|
[bool]$Required = $true,
|
|
[hashtable]$Meta = @{}
|
|
)
|
|
|
|
$checks.Add([ordered]@{
|
|
key = $Key
|
|
label = $Label
|
|
status = $Status
|
|
required = $Required
|
|
detail = $Detail
|
|
meta = $Meta
|
|
}) | Out-Null
|
|
}
|
|
|
|
function Save-Json {
|
|
param(
|
|
[string]$Path,
|
|
[object]$Value
|
|
)
|
|
$Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
|
|
}
|
|
|
|
function Invoke-WatchStep {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[string]$ScriptName,
|
|
[string[]]$Arguments,
|
|
[bool]$Required = $true
|
|
)
|
|
|
|
$scriptPath = Join-Path $repoRoot "scripts\$ScriptName"
|
|
$logPath = Join-Path $artifactDir "$Key.log"
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $scriptPath @Arguments 2>&1
|
|
$exitCode = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
|
|
$outputText = ($output -join [Environment]::NewLine)
|
|
$outputText | Set-Content -Encoding UTF8 -Path $logPath
|
|
$childArtifact = $null
|
|
$artifactMatch = [regex]::Match($outputText, '([A-Z]:\\[^\r\n]+)$', [System.Text.RegularExpressions.RegexOptions]::Multiline)
|
|
if ($artifactMatch.Success) {
|
|
$childArtifact = $artifactMatch.Groups[1].Value.Trim()
|
|
}
|
|
|
|
if ($exitCode -eq 0 -and $outputText -match "_WARN") {
|
|
Add-Check -Key $Key -Label $Label -Status "warn" -Required $Required -Detail "$Label completed with warnings" -Meta @{
|
|
artifact_dir = $childArtifact
|
|
log = $logPath
|
|
exit_code = $exitCode
|
|
}
|
|
} elseif ($exitCode -eq 0) {
|
|
Add-Check -Key $Key -Label $Label -Status "pass" -Required $Required -Detail "$Label passed" -Meta @{
|
|
artifact_dir = $childArtifact
|
|
log = $logPath
|
|
exit_code = $exitCode
|
|
}
|
|
} else {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Required $Required -Detail "$Label failed" -Meta @{
|
|
artifact_dir = $childArtifact
|
|
log = $logPath
|
|
exit_code = $exitCode
|
|
}
|
|
}
|
|
}
|
|
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
|
|
} else {
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
|
|
}
|
|
|
|
$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
|
|
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
|
|
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
|
|
$gitStatus = git -C $repoRoot status --short
|
|
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
|
|
if ($gitStatus) {
|
|
if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes and -RequireCleanWorktree is set" -Meta @{ artifact = $gitStatusPath }
|
|
} else {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; watchdog is running in advisory mode" -Meta @{ artifact = $gitStatusPath }
|
|
}
|
|
} else {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
|
|
}
|
|
|
|
if ($remoteUrl -match "192\.168\.3\.90") {
|
|
Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{
|
|
branch = $branch
|
|
commit = $commit
|
|
remote = $remoteUrl
|
|
}
|
|
} else {
|
|
Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{
|
|
branch = $branch
|
|
commit = $commit
|
|
remote = $remoteUrl
|
|
}
|
|
}
|
|
|
|
$failedPreflight = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
if ($failedPreflight.Count -eq 0) {
|
|
if ($SkipOpsCheck) {
|
|
Add-Check -Key "ops_check" -Label "production ops check" -Status "skip" -Required $false -Detail "Skipped by -SkipOpsCheck"
|
|
} else {
|
|
$opsArgs = @(
|
|
"-ServerHost", $ServerHost,
|
|
"-SshUser", $SshUser,
|
|
"-SshKeyPath", $SshKeyPath,
|
|
"-RemoteProjectPath", $RemoteProjectPath,
|
|
"-ComposeFile", $ComposeFile,
|
|
"-RemoteBackupRoot", $RemoteBackupRoot,
|
|
"-RemoteRollbackRoot", $RemoteRollbackRoot,
|
|
"-FrontendBaseUrl", $FrontendBaseUrl,
|
|
"-BackendBaseUrl", $BackendBaseUrl,
|
|
"-HealthUrl", $HealthUrl,
|
|
"-ReadinessUrl", $ReadinessUrl,
|
|
"-DiskWarnPercent", ([string]$DiskWarnPercent),
|
|
"-DiskFailPercent", ([string]$DiskFailPercent),
|
|
"-ArtifactRoot", (Join-Path $artifactDir "ops-check")
|
|
)
|
|
if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) {
|
|
$opsArgs += "-RequireCleanWorktree"
|
|
}
|
|
Invoke-WatchStep -Key "ops_check" -Label "production ops check" -ScriptName "run-production-ops-check.ps1" -Arguments $opsArgs
|
|
}
|
|
|
|
if ($SkipCapacityGovernance) {
|
|
Add-Check -Key "capacity_governance" -Label "capacity governance dry run" -Status "skip" -Required $false -Detail "Skipped by -SkipCapacityGovernance"
|
|
} else {
|
|
$capacityArgs = @(
|
|
"-ServerHost", $ServerHost,
|
|
"-SshUser", $SshUser,
|
|
"-SshKeyPath", $SshKeyPath,
|
|
"-RemoteProjectPath", $RemoteProjectPath,
|
|
"-RemoteDeployBackupRoot", $RemoteBackupRoot,
|
|
"-RemoteRollbackBackupRoot", $RemoteRollbackRoot,
|
|
"-DiskWarnPercent", ([string]$DiskWarnPercent),
|
|
"-DiskFailPercent", ([string]$DiskFailPercent),
|
|
"-ArtifactRoot", (Join-Path $artifactDir "capacity-governance")
|
|
)
|
|
if ($AllowDirtyWorktree) {
|
|
$capacityArgs += "-AllowDirtyWorktree"
|
|
}
|
|
Invoke-WatchStep -Key "capacity_governance" -Label "capacity governance dry run" -ScriptName "run-capacity-governance.ps1" -Arguments $capacityArgs
|
|
}
|
|
|
|
if ($IncludeReleaseGate) {
|
|
if ([string]::IsNullOrWhiteSpace($env:AI_CONTRACT_ADMIN_PASSWORD)) {
|
|
Add-Check -Key "release_gate" -Label "production release gate" -Status "fail" -Detail "AI_CONTRACT_ADMIN_PASSWORD must be set in the current process"
|
|
} else {
|
|
$releaseArgs = @(
|
|
"-ServerHost", $ServerHost,
|
|
"-FrontendBaseUrl", $FrontendBaseUrl,
|
|
"-BackendBaseUrl", $BackendBaseUrl,
|
|
"-ApiUrl", $ApiUrl,
|
|
"-HealthUrl", $HealthUrl,
|
|
"-ReadinessUrl", $ReadinessUrl,
|
|
"-ArtifactRoot", (Join-Path $artifactDir "release-gate")
|
|
)
|
|
if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) {
|
|
$releaseArgs += "-RequireCleanWorktree"
|
|
}
|
|
if ($IncludeBackupDrill) {
|
|
$releaseArgs += "-IncludeBackupDrill"
|
|
}
|
|
Invoke-WatchStep -Key "release_gate" -Label "production release gate" -ScriptName "run-production-release-gate.ps1" -Arguments $releaseArgs
|
|
}
|
|
} else {
|
|
Add-Check -Key "release_gate" -Label "production release gate" -Status "skip" -Required $false -Detail "Skipped by default; pass -IncludeReleaseGate for full watchdog"
|
|
}
|
|
}
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
|
$skippedList = @($checks | Where-Object { $_.status -eq "skip" })
|
|
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }
|
|
$alertSeverity = if ($failedRequired.Count -gt 0) { "critical" } elseif ($warningsList.Count -gt 0) { "warning" } else { "ok" }
|
|
$alertExitCode = if ($failedRequired.Count -gt 0) { 2 } elseif ($warningsList.Count -gt 0) { 1 } else { 0 }
|
|
|
|
$summary = [ordered]@{
|
|
name = "phase7-production-watchdog"
|
|
status = $overallStatus
|
|
alert_severity = $alertSeverity
|
|
alert_exit_code = $alertExitCode
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
artifact_dir = $artifactDir
|
|
branch = $branch
|
|
commit = $commit
|
|
remote = $remoteUrl
|
|
targets = [ordered]@{
|
|
server_host = $ServerHost
|
|
remote_project_path = $RemoteProjectPath
|
|
compose_file = $ComposeFile
|
|
frontend_base_url = $FrontendBaseUrl
|
|
backend_base_url = $BackendBaseUrl
|
|
api_url = $ApiUrl
|
|
}
|
|
options = [ordered]@{
|
|
require_clean_worktree = [bool]$RequireCleanWorktree
|
|
allow_dirty_worktree = [bool]$AllowDirtyWorktree
|
|
include_release_gate = [bool]$IncludeReleaseGate
|
|
include_backup_drill = [bool]$IncludeBackupDrill
|
|
skip_ops_check = [bool]$SkipOpsCheck
|
|
skip_capacity_governance = [bool]$SkipCapacityGovernance
|
|
fail_on_warn = [bool]$FailOnWarn
|
|
}
|
|
thresholds = [ordered]@{
|
|
disk_warn_percent = $DiskWarnPercent
|
|
disk_fail_percent = $DiskFailPercent
|
|
}
|
|
checks = $checks
|
|
totals = [ordered]@{
|
|
total = $checks.Count
|
|
failed_required = $failedRequired.Count
|
|
warnings = $warningsList.Count
|
|
skipped = $skippedList.Count
|
|
}
|
|
}
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
|
|
$alert = [ordered]@{
|
|
severity = $alertSeverity
|
|
status = $overallStatus
|
|
generated_at = $summary.generated_at
|
|
artifact_dir = $artifactDir
|
|
failed = @($failedRequired | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } })
|
|
warnings = @($warningsList | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } })
|
|
child_artifacts = @($checks | Where-Object { $_.meta.artifact_dir } | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; artifact_dir = $_.meta.artifact_dir } })
|
|
}
|
|
Save-Json -Path $alertJson -Value $alert
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
$markdown.Add("# Phase 7 Production Watchdog") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("- Status: $overallStatus") | Out-Null
|
|
$markdown.Add("- Alert severity: $alertSeverity") | Out-Null
|
|
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
|
$markdown.Add("- Branch: $branch") | Out-Null
|
|
$markdown.Add("- Commit: $commit") | Out-Null
|
|
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
|
$markdown.Add("|---|---:|---|---|") | Out-Null
|
|
foreach ($check in $checks) {
|
|
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
|
}
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("Child artifacts:") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
foreach ($check in $checks) {
|
|
if ($check.meta.artifact_dir) {
|
|
$markdown.Add(('- {0}: `{1}`' -f $check.label, $check.meta.artifact_dir)) | Out-Null
|
|
}
|
|
}
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("Local artifacts:") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add('- `production-watchdog-summary.json`') | Out-Null
|
|
$markdown.Add('- `production-watchdog-summary.md`') | Out-Null
|
|
$markdown.Add('- `production-watchdog-alert.json`') | Out-Null
|
|
$markdown.Add('- `production-watchdog-alert.md`') | Out-Null
|
|
$markdown.Add('- `git-status.txt`') | Out-Null
|
|
$markdown.Add('- `*.log`') | Out-Null
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
|
|
$alertMarkdown = New-Object System.Collections.Generic.List[string]
|
|
$alertMarkdown.Add("# Production Watchdog Alert") | Out-Null
|
|
$alertMarkdown.Add("") | Out-Null
|
|
$alertMarkdown.Add("- Severity: $alertSeverity") | Out-Null
|
|
$alertMarkdown.Add("- Status: $overallStatus") | Out-Null
|
|
$alertMarkdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
|
$alertMarkdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
|
if ($failedRequired.Count -gt 0) {
|
|
$alertMarkdown.Add("") | Out-Null
|
|
$alertMarkdown.Add("Failures:") | Out-Null
|
|
foreach ($check in $failedRequired) {
|
|
$alertMarkdown.Add(("- {0}: {1}" -f $check.label, $check.detail)) | Out-Null
|
|
}
|
|
}
|
|
if ($warningsList.Count -gt 0) {
|
|
$alertMarkdown.Add("") | Out-Null
|
|
$alertMarkdown.Add("Warnings:") | Out-Null
|
|
foreach ($check in $warningsList) {
|
|
$alertMarkdown.Add(("- {0}: {1}" -f $check.label, $check.detail)) | Out-Null
|
|
}
|
|
}
|
|
if ($failedRequired.Count -eq 0 -and $warningsList.Count -eq 0) {
|
|
$alertMarkdown.Add("") | Out-Null
|
|
$alertMarkdown.Add("No failures or warnings.") | Out-Null
|
|
}
|
|
$alertMarkdown | Set-Content -Encoding UTF8 -Path $alertMd
|
|
|
|
Write-Host "PHASE7_PRODUCTION_WATCHDOG_$overallStatus $artifactDir"
|
|
if ($overallStatus -eq "FAIL" -or ($FailOnWarn -and $overallStatus -eq "WARN")) {
|
|
throw "Phase 7 production watchdog $overallStatus. See $summaryMd"
|
|
}
|