Codex
235 lines
10 KiB
PowerShell
235 lines
10 KiB
PowerShell
param(
|
|
[string]$TaskName = "AIContractProductionWatchdog",
|
|
[string]$ArtifactRoot = "",
|
|
[string]$OutputRoot = "",
|
|
[int]$MaxLastRunAgeMinutes = 1440,
|
|
[int]$MaxArtifactAgeMinutes = 1440,
|
|
[switch]$RequireRecentRun,
|
|
[switch]$RequireRecentArtifact,
|
|
[switch]$FailOnWarn
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7.5-watchdog-scheduler"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace($OutputRoot)) {
|
|
$OutputRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7.8-watchdog-task-audit"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($OutputRoot)) {
|
|
$OutputRoot = Join-Path $repoRoot $OutputRoot
|
|
}
|
|
|
|
$artifactDir = Join-Path $OutputRoot $timestamp
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
|
|
$summaryJson = Join-Path $artifactDir "watchdog-task-audit-summary.json"
|
|
$summaryMd = Join-Path $artifactDir "watchdog-task-audit-summary.md"
|
|
$taskInfoJson = Join-Path $artifactDir "scheduled-task-info.json"
|
|
$latestSchedulerSummaryPath = Join-Path $artifactDir "latest-scheduler-summary.json"
|
|
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
|
|
function Add-Check {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[ValidateSet("pass", "fail", "warn", "skip")]
|
|
[string]$Status,
|
|
[string]$Detail,
|
|
[bool]$Required = $true,
|
|
[hashtable]$Meta = @{}
|
|
)
|
|
|
|
$checks.Add([ordered]@{
|
|
key = $Key
|
|
label = $Label
|
|
status = $Status
|
|
required = $Required
|
|
detail = $Detail
|
|
meta = $Meta
|
|
}) | Out-Null
|
|
}
|
|
|
|
function Save-Json {
|
|
param(
|
|
[string]$Path,
|
|
[object]$Value
|
|
)
|
|
$Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
|
|
}
|
|
|
|
function Get-LatestSchedulerArtifact {
|
|
if (-not (Test-Path -LiteralPath $ArtifactRoot)) {
|
|
return $null
|
|
}
|
|
|
|
Get-ChildItem -LiteralPath $ArtifactRoot -Directory |
|
|
Sort-Object LastWriteTime -Descending |
|
|
Where-Object { Test-Path -LiteralPath (Join-Path $_.FullName "watchdog-scheduler-summary.json") } |
|
|
Select-Object -First 1
|
|
}
|
|
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
|
|
} else {
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
|
|
}
|
|
|
|
$task = Get-ScheduledTask -TaskName $TaskName -ErrorAction SilentlyContinue
|
|
$taskInfo = $null
|
|
if ($null -eq $task) {
|
|
Add-Check -Key "scheduled_task_exists" -Label "scheduled task exists" -Status "fail" -Detail "Scheduled task '$TaskName' was not found"
|
|
} else {
|
|
$taskInfo = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop
|
|
$taskRecord = [ordered]@{
|
|
task_name = $task.TaskName
|
|
state = [string]$task.State
|
|
task_path = $task.TaskPath
|
|
description = $task.Description
|
|
last_run_time = if ($taskInfo.LastRunTime) { $taskInfo.LastRunTime.ToString("o") } else { $null }
|
|
last_task_result = $taskInfo.LastTaskResult
|
|
next_run_time = if ($taskInfo.NextRunTime) { $taskInfo.NextRunTime.ToString("o") } else { $null }
|
|
number_of_missed_runs = $taskInfo.NumberOfMissedRuns
|
|
}
|
|
Save-Json -Path $taskInfoJson -Value $taskRecord
|
|
|
|
Add-Check -Key "scheduled_task_exists" -Label "scheduled task exists" -Status "pass" -Detail "Scheduled task '$TaskName' exists" -Meta @{
|
|
artifact = $taskInfoJson
|
|
}
|
|
|
|
if ($task.State -eq "Ready" -or $task.State -eq "Running") {
|
|
Add-Check -Key "scheduled_task_state" -Label "scheduled task state" -Status "pass" -Detail "State=$($task.State)"
|
|
} elseif ($task.State -eq "Disabled") {
|
|
Add-Check -Key "scheduled_task_state" -Label "scheduled task state" -Status "fail" -Detail "State=Disabled"
|
|
} else {
|
|
Add-Check -Key "scheduled_task_state" -Label "scheduled task state" -Status "warn" -Required $false -Detail "State=$($task.State)"
|
|
}
|
|
|
|
if ($taskInfo.LastTaskResult -eq 0) {
|
|
Add-Check -Key "last_task_result" -Label "last task result" -Status "pass" -Detail "LastTaskResult=0"
|
|
} else {
|
|
Add-Check -Key "last_task_result" -Label "last task result" -Status "fail" -Detail "LastTaskResult=$($taskInfo.LastTaskResult)"
|
|
}
|
|
|
|
$lastRunTime = $taskInfo.LastRunTime
|
|
if ($lastRunTime.Year -lt 2000) {
|
|
if ($RequireRecentRun) {
|
|
Add-Check -Key "last_run_age" -Label "last run age" -Status "fail" -Detail "Task has never completed a real run"
|
|
} else {
|
|
Add-Check -Key "last_run_age" -Label "last run age" -Status "warn" -Required $false -Detail "Task has never completed a real run"
|
|
}
|
|
} else {
|
|
$ageMinutes = [math]::Round(((Get-Date) - $lastRunTime).TotalMinutes, 2)
|
|
if ($ageMinutes -le $MaxLastRunAgeMinutes) {
|
|
Add-Check -Key "last_run_age" -Label "last run age" -Status "pass" -Detail "Last run age ${ageMinutes}m"
|
|
} elseif ($RequireRecentRun) {
|
|
Add-Check -Key "last_run_age" -Label "last run age" -Status "fail" -Detail "Last run age ${ageMinutes}m exceeds $MaxLastRunAgeMinutes"
|
|
} else {
|
|
Add-Check -Key "last_run_age" -Label "last run age" -Status "warn" -Required $false -Detail "Last run age ${ageMinutes}m exceeds $MaxLastRunAgeMinutes"
|
|
}
|
|
}
|
|
}
|
|
|
|
$latestArtifact = Get-LatestSchedulerArtifact
|
|
if ($null -eq $latestArtifact) {
|
|
if ($RequireRecentArtifact) {
|
|
Add-Check -Key "latest_artifact" -Label "latest scheduler artifact" -Status "fail" -Detail "No scheduler artifact found in $ArtifactRoot"
|
|
} else {
|
|
Add-Check -Key "latest_artifact" -Label "latest scheduler artifact" -Status "warn" -Required $false -Detail "No scheduler artifact found in $ArtifactRoot"
|
|
}
|
|
$latestSummary = $null
|
|
} else {
|
|
$summaryPath = Join-Path $latestArtifact.FullName "watchdog-scheduler-summary.json"
|
|
Copy-Item -LiteralPath $summaryPath -Destination $latestSchedulerSummaryPath -Force
|
|
$latestSummary = Get-Content -LiteralPath $summaryPath -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
$artifactAgeMinutes = [math]::Round(((Get-Date) - $latestArtifact.LastWriteTime).TotalMinutes, 2)
|
|
if ($artifactAgeMinutes -le $MaxArtifactAgeMinutes) {
|
|
Add-Check -Key "latest_artifact" -Label "latest scheduler artifact" -Status "pass" -Detail "Latest artifact age ${artifactAgeMinutes}m" -Meta @{
|
|
artifact_dir = $latestArtifact.FullName
|
|
summary = $latestSchedulerSummaryPath
|
|
}
|
|
} elseif ($RequireRecentArtifact) {
|
|
Add-Check -Key "latest_artifact" -Label "latest scheduler artifact" -Status "fail" -Detail "Latest artifact age ${artifactAgeMinutes}m exceeds $MaxArtifactAgeMinutes" -Meta @{
|
|
artifact_dir = $latestArtifact.FullName
|
|
summary = $latestSchedulerSummaryPath
|
|
}
|
|
} else {
|
|
Add-Check -Key "latest_artifact" -Label "latest scheduler artifact" -Status "warn" -Required $false -Detail "Latest artifact age ${artifactAgeMinutes}m exceeds $MaxArtifactAgeMinutes" -Meta @{
|
|
artifact_dir = $latestArtifact.FullName
|
|
summary = $latestSchedulerSummaryPath
|
|
}
|
|
}
|
|
|
|
if ($latestSummary.status -eq "PASS") {
|
|
Add-Check -Key "latest_artifact_status" -Label "latest artifact status" -Status "pass" -Detail "Latest scheduler status=PASS"
|
|
} elseif ($latestSummary.status -eq "WARN") {
|
|
Add-Check -Key "latest_artifact_status" -Label "latest artifact status" -Status "warn" -Required $false -Detail "Latest scheduler status=WARN"
|
|
} else {
|
|
Add-Check -Key "latest_artifact_status" -Label "latest artifact status" -Status "fail" -Detail "Latest scheduler status=$($latestSummary.status)"
|
|
}
|
|
}
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
|
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }
|
|
$alertSeverity = if ($failedRequired.Count -gt 0) { "critical" } elseif ($warningsList.Count -gt 0) { "warning" } else { "ok" }
|
|
|
|
$summary = [ordered]@{
|
|
name = "phase7.8-watchdog-task-audit"
|
|
status = $overallStatus
|
|
alert_severity = $alertSeverity
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
artifact_dir = $artifactDir
|
|
task_name = $TaskName
|
|
artifact_root = $ArtifactRoot
|
|
options = [ordered]@{
|
|
max_last_run_age_minutes = $MaxLastRunAgeMinutes
|
|
max_artifact_age_minutes = $MaxArtifactAgeMinutes
|
|
require_recent_run = [bool]$RequireRecentRun
|
|
require_recent_artifact = [bool]$RequireRecentArtifact
|
|
fail_on_warn = [bool]$FailOnWarn
|
|
}
|
|
checks = $checks
|
|
totals = [ordered]@{
|
|
total = $checks.Count
|
|
failed_required = $failedRequired.Count
|
|
warnings = $warningsList.Count
|
|
}
|
|
}
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
$markdown.Add("# Phase 7.8 Watchdog Task Audit") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("- Status: $overallStatus") | Out-Null
|
|
$markdown.Add("- Alert severity: $alertSeverity") | Out-Null
|
|
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
|
$markdown.Add("- Task name: $TaskName") | Out-Null
|
|
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
|
$markdown.Add("|---|---:|---|---|") | Out-Null
|
|
foreach ($check in $checks) {
|
|
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
|
}
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("Local artifacts:") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add('- `watchdog-task-audit-summary.json`') | Out-Null
|
|
$markdown.Add('- `watchdog-task-audit-summary.md`') | Out-Null
|
|
$markdown.Add('- `scheduled-task-info.json` when the task exists') | Out-Null
|
|
$markdown.Add('- `latest-scheduler-summary.json` when a scheduler artifact exists') | Out-Null
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
|
|
Write-Host "PHASE7_8_WATCHDOG_TASK_AUDIT_$overallStatus $artifactDir"
|
|
if ($overallStatus -eq "FAIL" -or ($FailOnWarn -and $overallStatus -eq "WARN")) {
|
|
throw "Phase 7.8 watchdog task audit $overallStatus. See $summaryMd"
|
|
}
|