598 lines
26 KiB
PowerShell
598 lines
26 KiB
PowerShell
param(
|
|
[string]$ServerHost = "192.168.3.90",
|
|
[string]$SshUser = "root",
|
|
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
|
|
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
|
|
[string]$ComposeFile = "docker-compose.server.yml",
|
|
[string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy",
|
|
[string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/phase7-rollback",
|
|
[string]$FrontendBaseUrl = "http://192.168.3.90:3003",
|
|
[string]$BackendBaseUrl = "http://192.168.3.90:8001",
|
|
[string]$ApiUrl = "http://192.168.3.90:8001/api/v1",
|
|
[string]$HealthUrl = "http://192.168.3.90:8001/health",
|
|
[string]$ReadinessUrl = "http://192.168.3.90:8001/api/v1/system/readiness",
|
|
[string]$ArtifactRoot = "",
|
|
[int]$DiskWarnPercent = 80,
|
|
[int]$DiskFailPercent = 90,
|
|
[int]$MaxDeployArtifactAgeHours = 168,
|
|
[int]$MaxRollbackArtifactAgeHours = 168,
|
|
[switch]$RequireCleanWorktree,
|
|
[switch]$AllowDirtyWorktree,
|
|
[switch]$SkipOpsCheck,
|
|
[switch]$SkipReleaseGate,
|
|
[switch]$SkipReleaseGateBuild,
|
|
[switch]$SkipProductionRegression,
|
|
[switch]$IncludeBackupDrill,
|
|
[switch]$SkipCapacityGovernance,
|
|
[switch]$SkipTaskAudit,
|
|
[switch]$SkipEvidenceReport,
|
|
[switch]$FailOnWarn,
|
|
[switch]$FailOnBlocked
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase8-pilot-readiness"
|
|
}
|
|
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
|
|
}
|
|
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
|
|
$summaryJson = Join-Path $artifactDir "pilot-readiness-summary.json"
|
|
$summaryMd = Join-Path $artifactDir "pilot-readiness-summary.md"
|
|
$alertJson = Join-Path $artifactDir "pilot-readiness-alert.json"
|
|
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
|
|
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
|
|
function Add-Check {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[ValidateSet("pass", "fail", "warn", "skip", "blocked")]
|
|
[string]$Status,
|
|
[string]$Detail,
|
|
[bool]$Required = $true,
|
|
[hashtable]$Meta = @{}
|
|
)
|
|
|
|
$checks.Add([ordered]@{
|
|
key = $Key
|
|
label = $Label
|
|
status = $Status
|
|
required = $Required
|
|
detail = $Detail
|
|
meta = $Meta
|
|
}) | Out-Null
|
|
}
|
|
|
|
function Save-Json {
|
|
param(
|
|
[string]$Path,
|
|
[object]$Value
|
|
)
|
|
$Value | ConvertTo-Json -Depth 16 | Set-Content -Encoding UTF8 -Path $Path
|
|
}
|
|
|
|
function ConvertTo-SafeFileName {
|
|
param([string]$Value)
|
|
return ($Value -replace '[^a-zA-Z0-9._-]', '-').Trim("-")
|
|
}
|
|
|
|
function Invoke-HttpProbe {
|
|
param(
|
|
[string]$Method,
|
|
[string]$Url
|
|
)
|
|
|
|
Add-Type -AssemblyName System.Net.Http
|
|
$handler = New-Object System.Net.Http.HttpClientHandler
|
|
$client = New-Object System.Net.Http.HttpClient($handler)
|
|
$client.Timeout = [TimeSpan]::FromSeconds(15)
|
|
try {
|
|
$request = New-Object System.Net.Http.HttpRequestMessage((New-Object System.Net.Http.HttpMethod($Method)), $Url)
|
|
$response = $client.SendAsync($request).GetAwaiter().GetResult()
|
|
$text = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
|
|
$parsed = $null
|
|
if (-not [string]::IsNullOrWhiteSpace($text)) {
|
|
try {
|
|
$parsed = $text | ConvertFrom-Json -ErrorAction Stop
|
|
} catch {
|
|
$parsed = $null
|
|
}
|
|
}
|
|
|
|
return [ordered]@{
|
|
ok = $true
|
|
url = $Url
|
|
method = $Method
|
|
status_code = [int]$response.StatusCode
|
|
body = $parsed
|
|
body_text = if ($parsed) { $null } else { $text.Substring(0, [Math]::Min(500, $text.Length)) }
|
|
}
|
|
} catch {
|
|
return [ordered]@{
|
|
ok = $false
|
|
url = $Url
|
|
method = $Method
|
|
status_code = $null
|
|
error = $_.Exception.Message
|
|
}
|
|
} finally {
|
|
$client.Dispose()
|
|
$handler.Dispose()
|
|
}
|
|
}
|
|
|
|
function Invoke-PilotStep {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[string]$ScriptName,
|
|
[string[]]$Arguments,
|
|
[bool]$Required = $true
|
|
)
|
|
|
|
$scriptPath = Join-Path $repoRoot "scripts\$ScriptName"
|
|
$logPath = Join-Path $artifactDir "$Key.log"
|
|
if (-not (Test-Path -LiteralPath $scriptPath)) {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Required $Required -Detail "Script not found: $ScriptName"
|
|
return
|
|
}
|
|
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $scriptPath @Arguments 2>&1
|
|
$exitCode = $LASTEXITCODE
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
|
|
$outputText = ($output -join [Environment]::NewLine)
|
|
$outputText | Set-Content -Encoding UTF8 -Path $logPath
|
|
$childArtifact = $null
|
|
$artifactMatches = [regex]::Matches($outputText, '([A-Z]:\\[^\r\n]+)$', [System.Text.RegularExpressions.RegexOptions]::Multiline)
|
|
if ($artifactMatches.Count -gt 0) {
|
|
$childArtifact = $artifactMatches[$artifactMatches.Count - 1].Groups[1].Value.Trim()
|
|
}
|
|
|
|
$meta = @{
|
|
log = $logPath
|
|
artifact_dir = $childArtifact
|
|
exit_code = $exitCode
|
|
}
|
|
|
|
if ($exitCode -eq 0 -and $outputText -match "_BLOCKED") {
|
|
Add-Check -Key $Key -Label $Label -Status "blocked" -Required $false -Detail "$Label completed with external blocker" -Meta $meta
|
|
} elseif ($exitCode -eq 0 -and $outputText -match "_WARN") {
|
|
Add-Check -Key $Key -Label $Label -Status "warn" -Required $false -Detail "$Label completed with warnings" -Meta $meta
|
|
} elseif ($exitCode -eq 0) {
|
|
Add-Check -Key $Key -Label $Label -Status "pass" -Required $Required -Detail "$Label passed" -Meta $meta
|
|
} else {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Required $Required -Detail "$Label failed" -Meta $meta
|
|
}
|
|
}
|
|
|
|
function Add-RequiredFileCheck {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[string]$RelativePath
|
|
)
|
|
|
|
$path = Join-Path $repoRoot $RelativePath
|
|
if (Test-Path -LiteralPath $path -PathType Leaf) {
|
|
Add-Check -Key $Key -Label $Label -Status "pass" -Detail "Found $RelativePath" -Meta @{ path = $path }
|
|
} else {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Detail "Missing $RelativePath" -Meta @{ path = $path }
|
|
}
|
|
}
|
|
|
|
function Add-ExistingFileCheck {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[string]$Path
|
|
)
|
|
|
|
if (-not [string]::IsNullOrWhiteSpace($Path) -and (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
|
Add-Check -Key $Key -Label $Label -Status "pass" -Detail "Found $Path" -Meta @{ path = $Path }
|
|
} else {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Detail "Missing required file" -Meta @{ path = $Path }
|
|
}
|
|
}
|
|
|
|
function Add-LatestArtifactCheck {
|
|
param(
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[string]$RelativeRoot,
|
|
[string]$SummaryFile,
|
|
[int]$MaxAgeHours
|
|
)
|
|
|
|
$root = Join-Path $repoRoot $RelativeRoot
|
|
if (-not (Test-Path -LiteralPath $root -PathType Container)) {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Detail "Artifact root is missing: $RelativeRoot"
|
|
return
|
|
}
|
|
|
|
$latest = Get-ChildItem -LiteralPath $root -Directory |
|
|
Sort-Object LastWriteTime -Descending |
|
|
Where-Object { Test-Path -LiteralPath (Join-Path $_.FullName $SummaryFile) } |
|
|
Select-Object -First 1
|
|
|
|
if ($null -eq $latest) {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Detail "No artifact with $SummaryFile found in $RelativeRoot"
|
|
return
|
|
}
|
|
|
|
$summaryPath = Join-Path $latest.FullName $SummaryFile
|
|
$latestCopy = Join-Path $artifactDir ("latest-$Key.json")
|
|
Copy-Item -LiteralPath $summaryPath -Destination $latestCopy -Force
|
|
|
|
$summary = $null
|
|
try {
|
|
$summary = Get-Content -LiteralPath $summaryPath -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
} catch {
|
|
$summary = $null
|
|
}
|
|
|
|
$generatedAt = $latest.LastWriteTime
|
|
if ($summary -and $summary.generated_at) {
|
|
try { $generatedAt = [datetime]$summary.generated_at } catch { $generatedAt = $latest.LastWriteTime }
|
|
}
|
|
$ageHours = [math]::Round(((Get-Date) - $generatedAt.ToLocalTime()).TotalHours, 2)
|
|
$status = if ($summary -and $summary.status) { [string]$summary.status } else { "UNKNOWN" }
|
|
$meta = @{
|
|
artifact_dir = $latest.FullName
|
|
summary = $latestCopy
|
|
status = $status
|
|
age_hours = $ageHours
|
|
}
|
|
|
|
if ($ageHours -gt $MaxAgeHours) {
|
|
Add-Check -Key $Key -Label $Label -Status "warn" -Required $false -Detail "Latest artifact age ${ageHours}h exceeds $MaxAgeHours" -Meta $meta
|
|
} elseif ($status -eq "PASS") {
|
|
Add-Check -Key $Key -Label $Label -Status "pass" -Detail "Latest artifact status=PASS, age=${ageHours}h" -Meta $meta
|
|
} elseif ($status -eq "WARN") {
|
|
Add-Check -Key $Key -Label $Label -Status "warn" -Required $false -Detail "Latest artifact status=WARN, age=${ageHours}h" -Meta $meta
|
|
} else {
|
|
Add-Check -Key $Key -Label $Label -Status "fail" -Detail "Latest artifact status=$status, age=${ageHours}h" -Meta $meta
|
|
}
|
|
}
|
|
|
|
function Add-WebhookCheck {
|
|
$sources = New-Object System.Collections.Generic.List[string]
|
|
foreach ($target in @(
|
|
[System.EnvironmentVariableTarget]::Process,
|
|
[System.EnvironmentVariableTarget]::User,
|
|
[System.EnvironmentVariableTarget]::Machine
|
|
)) {
|
|
$value = [Environment]::GetEnvironmentVariable("AI_CONTRACT_WATCHDOG_WEBHOOK_URL", $target)
|
|
if (-not [string]::IsNullOrWhiteSpace($value)) {
|
|
$sources.Add([string]$target) | Out-Null
|
|
}
|
|
}
|
|
|
|
if ($sources.Count -gt 0) {
|
|
Add-Check -Key "external_webhook" -Label "external alert Webhook" -Status "pass" -Detail "Webhook URL is configured; value redacted" -Meta @{
|
|
sources = $sources.ToArray()
|
|
}
|
|
} else {
|
|
Add-Check -Key "external_webhook" -Label "external alert Webhook" -Status "blocked" -Required $false -Detail "BLOCKED_EXTERNAL_WEBHOOK: AI_CONTRACT_WATCHDOG_WEBHOOK_URL is not configured"
|
|
}
|
|
}
|
|
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
|
|
} else {
|
|
Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
|
|
}
|
|
|
|
$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
|
|
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
|
|
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
|
|
$gitStatus = git -C $repoRoot status --short
|
|
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
|
|
|
|
if ($gitStatus) {
|
|
if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes and -RequireCleanWorktree is set" -Meta @{ artifact = $gitStatusPath }
|
|
} else {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; pilot readiness is advisory" -Meta @{ artifact = $gitStatusPath }
|
|
}
|
|
} else {
|
|
Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
|
|
}
|
|
|
|
if ($remoteUrl -match "192\.168\.3\.90") {
|
|
Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{
|
|
branch = $branch
|
|
commit = $commit
|
|
remote = $remoteUrl
|
|
}
|
|
} else {
|
|
Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{
|
|
branch = $branch
|
|
commit = $commit
|
|
remote = $remoteUrl
|
|
}
|
|
}
|
|
|
|
$phase8PlanPath = Get-ChildItem -LiteralPath (Join-Path $repoRoot "project-docs") -File |
|
|
Where-Object {
|
|
$_.Extension -eq ".md" -and
|
|
$_.Name -like "*8-*2026-05-01.md" -and
|
|
$_.Name -notlike "*8-12*"
|
|
} |
|
|
Sort-Object LastWriteTime -Descending |
|
|
Select-Object -First 1
|
|
Add-ExistingFileCheck -Key "phase8_plan" -Label "phase 8 plan" -Path $(if ($phase8PlanPath) { $phase8PlanPath.FullName } else { "" })
|
|
|
|
foreach ($file in @(
|
|
@{ key = "template_readme"; label = "template pack README"; path = "project-docs\phase8-pilot-templates\README.md" },
|
|
@{ key = "template_account_roles"; label = "account role matrix"; path = "project-docs\phase8-pilot-templates\pilot-account-role-matrix.csv" },
|
|
@{ key = "template_samples"; label = "sample inventory"; path = "project-docs\phase8-pilot-templates\pilot-sample-inventory.csv" },
|
|
@{ key = "template_kb_docs"; label = "KB document inventory"; path = "project-docs\phase8-pilot-templates\pilot-kb-document-inventory.csv" },
|
|
@{ key = "template_contract_templates"; label = "contract template inventory"; path = "project-docs\phase8-pilot-templates\pilot-template-inventory.csv" },
|
|
@{ key = "template_runbook"; label = "scenario runbook"; path = "project-docs\phase8-pilot-templates\pilot-scenario-runbook.md" },
|
|
@{ key = "template_execution_record"; label = "scenario execution record"; path = "project-docs\phase8-pilot-templates\pilot-scenario-execution-record.csv" },
|
|
@{ key = "template_feedback"; label = "feedback register"; path = "project-docs\phase8-pilot-templates\pilot-feedback-register.csv" },
|
|
@{ key = "template_defects"; label = "defect fix record"; path = "project-docs\phase8-pilot-templates\pilot-defect-fix-record.csv" },
|
|
@{ key = "template_daily"; label = "daily report template"; path = "project-docs\phase8-pilot-templates\pilot-daily-report-template.md" },
|
|
@{ key = "template_weekly"; label = "weekly report template"; path = "project-docs\phase8-pilot-templates\pilot-weekly-report-template.md" },
|
|
@{ key = "template_acceptance"; label = "acceptance checklist"; path = "project-docs\phase8-pilot-templates\pilot-acceptance-checklist.md" }
|
|
)) {
|
|
Add-RequiredFileCheck -Key $file.key -Label $file.label -RelativePath $file.path
|
|
}
|
|
|
|
foreach ($script in @(
|
|
@{ key = "deploy_script"; label = "one-click deploy script"; path = "scripts\deploy-frontend-docker.ps1" },
|
|
@{ key = "rollback_script"; label = "one-click rollback script"; path = "scripts\rollback-frontend-docker.ps1" },
|
|
@{ key = "ops_check_script"; label = "ops check script"; path = "scripts\run-production-ops-check.ps1" },
|
|
@{ key = "release_gate_script"; label = "release gate script"; path = "scripts\run-production-release-gate.ps1" },
|
|
@{ key = "capacity_script"; label = "capacity governance script"; path = "scripts\run-capacity-governance.ps1" },
|
|
@{ key = "task_audit_script"; label = "watchdog task audit script"; path = "scripts\run-watchdog-task-audit.ps1" },
|
|
@{ key = "evidence_report_script"; label = "watchdog evidence report script"; path = "scripts\run-watchdog-evidence-report.ps1" }
|
|
)) {
|
|
Add-RequiredFileCheck -Key $script.key -Label $script.label -RelativePath $script.path
|
|
}
|
|
|
|
foreach ($probe in @(
|
|
@{ key = "health_probe"; label = "backend health"; method = "GET"; url = $HealthUrl },
|
|
@{ key = "readiness_probe"; label = "system readiness"; method = "GET"; url = $ReadinessUrl },
|
|
@{ key = "frontend_home_probe"; label = "frontend home"; method = "GET"; url = $FrontendBaseUrl },
|
|
@{ key = "frontend_login_probe"; label = "frontend login"; method = "GET"; url = ($FrontendBaseUrl.TrimEnd("/") + "/login") },
|
|
@{ key = "frontend_pilot_probe"; label = "frontend pilot readiness page"; method = "GET"; url = ($FrontendBaseUrl.TrimEnd("/") + "/pilot-readiness") }
|
|
)) {
|
|
$probeResult = Invoke-HttpProbe -Method $probe.method -Url $probe.url
|
|
$probePath = Join-Path $artifactDir ("probe-" + (ConvertTo-SafeFileName -Value $probe.key) + ".json")
|
|
Save-Json -Path $probePath -Value $probeResult
|
|
if ($probeResult.ok -and $probeResult.status_code -ge 200 -and $probeResult.status_code -lt 400) {
|
|
Add-Check -Key $probe.key -Label $probe.label -Status "pass" -Detail "$($probe.method) $($probe.url) -> $($probeResult.status_code)" -Meta @{ artifact = $probePath }
|
|
} else {
|
|
$detail = if ($probeResult.ok) { "$($probe.method) $($probe.url) -> $($probeResult.status_code)" } else { "$($probe.method) $($probe.url) failed: $($probeResult.error)" }
|
|
Add-Check -Key $probe.key -Label $probe.label -Status "fail" -Detail $detail -Meta @{ artifact = $probePath }
|
|
}
|
|
}
|
|
|
|
Add-LatestArtifactCheck -Key "latest_deploy_artifact" -Label "latest one-click deploy artifact" -RelativeRoot "project-docs\qa-artifacts\phase7-frontend-docker-deploy" -SummaryFile "frontend-docker-deploy-summary.json" -MaxAgeHours $MaxDeployArtifactAgeHours
|
|
Add-LatestArtifactCheck -Key "latest_rollback_artifact" -Label "latest one-click rollback artifact" -RelativeRoot "project-docs\qa-artifacts\phase7-frontend-docker-rollback" -SummaryFile "frontend-docker-rollback-summary.json" -MaxAgeHours $MaxRollbackArtifactAgeHours
|
|
Add-WebhookCheck
|
|
|
|
$failedPreflight = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
if ($failedPreflight.Count -eq 0) {
|
|
if ($SkipOpsCheck) {
|
|
Add-Check -Key "ops_check" -Label "production ops check" -Status "skip" -Required $false -Detail "Skipped by -SkipOpsCheck"
|
|
} else {
|
|
$opsArgs = @(
|
|
"-ServerHost", $ServerHost,
|
|
"-SshUser", $SshUser,
|
|
"-SshKeyPath", $SshKeyPath,
|
|
"-RemoteProjectPath", $RemoteProjectPath,
|
|
"-ComposeFile", $ComposeFile,
|
|
"-RemoteBackupRoot", $RemoteBackupRoot,
|
|
"-RemoteRollbackRoot", $RemoteRollbackRoot,
|
|
"-FrontendBaseUrl", $FrontendBaseUrl,
|
|
"-BackendBaseUrl", $BackendBaseUrl,
|
|
"-HealthUrl", $HealthUrl,
|
|
"-ReadinessUrl", $ReadinessUrl,
|
|
"-DiskWarnPercent", ([string]$DiskWarnPercent),
|
|
"-DiskFailPercent", ([string]$DiskFailPercent),
|
|
"-ArtifactRoot", (Join-Path $artifactDir "ops-check")
|
|
)
|
|
if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) {
|
|
$opsArgs += "-RequireCleanWorktree"
|
|
}
|
|
Invoke-PilotStep -Key "ops_check" -Label "production ops check" -ScriptName "run-production-ops-check.ps1" -Arguments $opsArgs
|
|
}
|
|
|
|
if ($SkipReleaseGate) {
|
|
Add-Check -Key "release_gate" -Label "post-deploy release gate" -Status "skip" -Required $false -Detail "Skipped by -SkipReleaseGate"
|
|
} else {
|
|
$releaseArgs = @(
|
|
"-ServerHost", $ServerHost,
|
|
"-FrontendBaseUrl", $FrontendBaseUrl,
|
|
"-BackendBaseUrl", $BackendBaseUrl,
|
|
"-ApiUrl", $ApiUrl,
|
|
"-HealthUrl", $HealthUrl,
|
|
"-ReadinessUrl", $ReadinessUrl,
|
|
"-ArtifactRoot", (Join-Path $artifactDir "release-gate")
|
|
)
|
|
if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) {
|
|
$releaseArgs += "-RequireCleanWorktree"
|
|
}
|
|
if ($IncludeBackupDrill) {
|
|
$releaseArgs += "-IncludeBackupDrill"
|
|
}
|
|
if ($SkipReleaseGateBuild) {
|
|
$releaseArgs += "-SkipBuild"
|
|
}
|
|
if ($SkipProductionRegression) {
|
|
$releaseArgs += "-SkipProductionRegression"
|
|
Add-Check -Key "pilot_login_regression" -Label "pilot login regression" -Status "skip" -Required $false -Detail "Skipped by -SkipProductionRegression"
|
|
} elseif ([string]::IsNullOrWhiteSpace($env:AI_CONTRACT_ADMIN_PASSWORD)) {
|
|
$releaseArgs += "-SkipProductionRegression"
|
|
Add-Check -Key "pilot_login_regression" -Label "pilot login regression" -Status "blocked" -Required $false -Detail "AI_CONTRACT_ADMIN_PASSWORD is not set in the current process; production regression skipped"
|
|
} else {
|
|
Add-Check -Key "pilot_login_regression" -Label "pilot login regression" -Status "pass" -Detail "Runtime admin password is present for production regression; value redacted"
|
|
}
|
|
Invoke-PilotStep -Key "release_gate" -Label "post-deploy release gate" -ScriptName "run-production-release-gate.ps1" -Arguments $releaseArgs
|
|
}
|
|
|
|
if ($SkipCapacityGovernance) {
|
|
Add-Check -Key "capacity_governance" -Label "capacity governance dry run" -Status "skip" -Required $false -Detail "Skipped by -SkipCapacityGovernance"
|
|
} else {
|
|
$capacityArgs = @(
|
|
"-ServerHost", $ServerHost,
|
|
"-SshUser", $SshUser,
|
|
"-SshKeyPath", $SshKeyPath,
|
|
"-RemoteProjectPath", $RemoteProjectPath,
|
|
"-RemoteDeployBackupRoot", $RemoteBackupRoot,
|
|
"-RemoteRollbackBackupRoot", $RemoteRollbackRoot,
|
|
"-DiskWarnPercent", ([string]$DiskWarnPercent),
|
|
"-DiskFailPercent", ([string]$DiskFailPercent),
|
|
"-ArtifactRoot", (Join-Path $artifactDir "capacity-governance")
|
|
)
|
|
if ($AllowDirtyWorktree -or ((-not $RequireCleanWorktree) -and $gitStatus)) {
|
|
$capacityArgs += "-AllowDirtyWorktree"
|
|
}
|
|
Invoke-PilotStep -Key "capacity_governance" -Label "capacity governance dry run" -ScriptName "run-capacity-governance.ps1" -Arguments $capacityArgs
|
|
}
|
|
|
|
if ($SkipTaskAudit) {
|
|
Add-Check -Key "task_audit" -Label "watchdog task audit" -Status "skip" -Required $false -Detail "Skipped by -SkipTaskAudit"
|
|
} else {
|
|
Invoke-PilotStep -Key "task_audit" -Label "watchdog task audit" -ScriptName "run-watchdog-task-audit.ps1" -Arguments @(
|
|
"-OutputRoot", (Join-Path $artifactDir "task-audit"),
|
|
"-RequireRecentRun",
|
|
"-RequireRecentArtifact"
|
|
)
|
|
}
|
|
|
|
if ($SkipEvidenceReport) {
|
|
Add-Check -Key "evidence_report" -Label "watchdog evidence daily report" -Status "skip" -Required $false -Detail "Skipped by -SkipEvidenceReport"
|
|
} else {
|
|
Invoke-PilotStep -Key "evidence_report" -Label "watchdog evidence daily report" -ScriptName "run-watchdog-evidence-report.ps1" -Arguments @(
|
|
"-ReportKind", "Daily",
|
|
"-OutputRoot", (Join-Path $artifactDir "evidence-report")
|
|
)
|
|
}
|
|
}
|
|
|
|
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
|
|
$blockedList = @($checks | Where-Object { $_.status -eq "blocked" })
|
|
$skippedList = @($checks | Where-Object { $_.status -eq "skip" })
|
|
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($blockedList.Count -gt 0) { "BLOCKED" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }
|
|
$alertSeverity = if ($failedRequired.Count -gt 0) { "critical" } elseif ($blockedList.Count -gt 0) { "blocked" } elseif ($warningsList.Count -gt 0) { "warning" } else { "ok" }
|
|
|
|
$summary = [ordered]@{
|
|
name = "phase8-pilot-readiness"
|
|
status = $overallStatus
|
|
alert_severity = $alertSeverity
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
artifact_dir = $artifactDir
|
|
branch = $branch
|
|
commit = $commit
|
|
remote = $remoteUrl
|
|
targets = [ordered]@{
|
|
server_host = $ServerHost
|
|
remote_project_path = $RemoteProjectPath
|
|
compose_file = $ComposeFile
|
|
frontend_base_url = $FrontendBaseUrl
|
|
backend_base_url = $BackendBaseUrl
|
|
api_url = $ApiUrl
|
|
health_url = $HealthUrl
|
|
readiness_url = $ReadinessUrl
|
|
}
|
|
options = [ordered]@{
|
|
require_clean_worktree = [bool]$RequireCleanWorktree
|
|
allow_dirty_worktree = [bool]$AllowDirtyWorktree
|
|
skip_ops_check = [bool]$SkipOpsCheck
|
|
skip_release_gate = [bool]$SkipReleaseGate
|
|
skip_release_gate_build = [bool]$SkipReleaseGateBuild
|
|
skip_production_regression = [bool]$SkipProductionRegression
|
|
include_backup_drill = [bool]$IncludeBackupDrill
|
|
skip_capacity_governance = [bool]$SkipCapacityGovernance
|
|
skip_task_audit = [bool]$SkipTaskAudit
|
|
skip_evidence_report = [bool]$SkipEvidenceReport
|
|
fail_on_warn = [bool]$FailOnWarn
|
|
fail_on_blocked = [bool]$FailOnBlocked
|
|
}
|
|
thresholds = [ordered]@{
|
|
disk_warn_percent = $DiskWarnPercent
|
|
disk_fail_percent = $DiskFailPercent
|
|
max_deploy_artifact_age_hours = $MaxDeployArtifactAgeHours
|
|
max_rollback_artifact_age_hours = $MaxRollbackArtifactAgeHours
|
|
}
|
|
checks = $checks
|
|
totals = [ordered]@{
|
|
total = $checks.Count
|
|
failed_required = $failedRequired.Count
|
|
warnings = $warningsList.Count
|
|
blocked = $blockedList.Count
|
|
skipped = $skippedList.Count
|
|
}
|
|
}
|
|
Save-Json -Path $summaryJson -Value $summary
|
|
|
|
$alert = [ordered]@{
|
|
severity = $alertSeverity
|
|
status = $overallStatus
|
|
generated_at = $summary.generated_at
|
|
artifact_dir = $artifactDir
|
|
failed = @($failedRequired | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } })
|
|
blocked = @($blockedList | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } })
|
|
warnings = @($warningsList | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } })
|
|
child_artifacts = @($checks | Where-Object { $_.meta.artifact_dir } | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; artifact_dir = $_.meta.artifact_dir } })
|
|
}
|
|
Save-Json -Path $alertJson -Value $alert
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
$markdown.Add("# Phase 8 Pilot Readiness Gate") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("- Status: $overallStatus") | Out-Null
|
|
$markdown.Add("- Alert severity: $alertSeverity") | Out-Null
|
|
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
|
|
$markdown.Add("- Branch: $branch") | Out-Null
|
|
$markdown.Add("- Commit: $commit") | Out-Null
|
|
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
|
|
$markdown.Add("|---|---:|---|---|") | Out-Null
|
|
foreach ($check in $checks) {
|
|
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
|
|
}
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("Child artifacts:") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
foreach ($check in $checks) {
|
|
if ($check.meta.artifact_dir) {
|
|
$markdown.Add(('- {0}: `{1}`' -f $check.label, $check.meta.artifact_dir)) | Out-Null
|
|
}
|
|
}
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("Local artifacts:") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add('- `pilot-readiness-summary.json`') | Out-Null
|
|
$markdown.Add('- `pilot-readiness-summary.md`') | Out-Null
|
|
$markdown.Add('- `pilot-readiness-alert.json`') | Out-Null
|
|
$markdown.Add('- `git-status.txt`') | Out-Null
|
|
$markdown.Add('- `probe-*.json`') | Out-Null
|
|
$markdown.Add('- `*.log`') | Out-Null
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
|
|
|
|
Write-Host "PHASE8_PILOT_READINESS_$overallStatus $artifactDir"
|
|
if ($overallStatus -eq "FAIL" -or ($FailOnWarn -and $overallStatus -eq "WARN") -or ($FailOnBlocked -and $overallStatus -eq "BLOCKED")) {
|
|
throw "Phase 8 pilot readiness $overallStatus. See $summaryMd"
|
|
}
|