From b0229de8110eceeaddbf75471267e3fae7412778 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 1 May 2026 12:50:08 +0800 Subject: [PATCH] chore: add production watchdog automation --- project-docs/运维手册-2026-05-01.md | 16 + .../闭环交付与深度开发方案-2026-05-01.md | 2 +- ...7.4-日常巡检与告警门禁执行记录-2026-05-01.md | 73 ++++ ...段7.4-日常巡检与告警门禁方案-2026-05-01.md | 70 ++++ scripts/run-production-watchdog.ps1 | 356 ++++++++++++++++++ 5 files changed, 516 insertions(+), 1 deletion(-) create mode 100644 project-docs/阶段7.4-日常巡检与告警门禁执行记录-2026-05-01.md create mode 100644 project-docs/阶段7.4-日常巡检与告警门禁方案-2026-05-01.md create mode 100644 scripts/run-production-watchdog.ps1 diff --git a/project-docs/运维手册-2026-05-01.md b/project-docs/运维手册-2026-05-01.md index 73650cea..752ce9ef 100644 --- a/project-docs/运维手册-2026-05-01.md +++ b/project-docs/运维手册-2026-05-01.md @@ -294,6 +294,22 @@ powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-capacity-gov powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-capacity-governance.ps1 -Apply -SkipBackupCleanup -DockerBuilderPruneUntilHours 0 ``` +日常生产值守： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-production-watchdog.ps1 +``` + +完整生产值守： + +```powershell +$env:AI_CONTRACT_ADMIN_PASSWORD = '<由管理员输入的实际密码>' +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-production-watchdog.ps1 -RequireCleanWorktree -IncludeReleaseGate +Remove-Item Env:\AI_CONTRACT_ADMIN_PASSWORD +``` + +值守脚本会生成 `production-watchdog-alert.json` 和 `production-watchdog-alert.md`。告警等级为 `ok`、`warning`、`critical`，后续可接入邮件、企业微信或调度系统。 + 列出可回滚的 frontend 备份： ```powershell diff --git a/project-docs/闭环交付与深度开发方案-2026-05-01.md b/project-docs/闭环交付与深度开发方案-2026-05-01.md index f49b457f..4d1db992 100644 --- a/project-docs/闭环交付与深度开发方案-2026-05-01.md +++ b/project-docs/闭环交付与深度开发方案-2026-05-01.md @@ -20,7 +20,7 @@ 阶段 1 已完成源码闭环与生产入口闭环，生产无人 E2E `18/18` 通过。阶段 2 已完成 Review 重试、Knowledge Base 文档重试、Review -> Chat -> Contract 上下文联动、首页交付工作台和阶段 2 自动化无人闭环，本地与生产 E2E 均 `7/7` 通过。阶段 3 已完成系统 readiness、试点交付准备页、默认账号说明、演示样本数据、依赖健康门禁和阶段 3 自动化无人闭环，本地与生产 E2E 均 `5/5` 通过。 -当前后续重点已从“生产运营自动化与发布门禁固化”推进到“生产运营常态化与一键发布”：阶段 7 首批一键发布闭环已完成，阶段 7.2 一键回滚与生产巡检门禁也已完成首批闭环。阶段 7.3 容量治理已完成，备份保留策略、Docker build cache/旧镜像清理和磁盘告警阈值已固化；服务器磁盘从 `89%` 降至 `67%`。 +当前后续重点已从“生产运营自动化与发布门禁固化”推进到“生产运营常态化与一键发布”：阶段 7 首批一键发布闭环已完成，阶段 7.2 一键回滚与生产巡检门禁也已完成首批闭环。阶段 7.3 容量治理已完成，备份保留策略、Docker build cache/旧镜像清理和磁盘告警阈值已固化；服务器磁盘从 `89%` 降至 `67%`。当前正在进入阶段 7.4 日常巡检与告警门禁，将 ops check、capacity governance dry run 和可选 release gate 聚合为统一生产值守入口。 --- diff --git a/project-docs/阶段7.4-日常巡检与告警门禁执行记录-2026-05-01.md b/project-docs/阶段7.4-日常巡检与告警门禁执行记录-2026-05-01.md new file mode 100644 index 00000000..36b9c153 --- /dev/null +++ b/project-docs/阶段7.4-日常巡检与告警门禁执行记录-2026-05-01.md @@ -0,0 +1,73 @@ +# 阶段 7.4：日常巡检与告警门禁执行记录（2026-05-01） + +> 当前目标：把生产巡检、容量治理和可选 release gate 聚合成一个日常生产值守入口，形成统一告警产物。 + +--- + +## 一、当前执行结论 + +| 主线 | 状态 | 备注 | +|---|---|---| +| gstack 安装校验 | PASS | 已返回 `GSTACK_OK` | +| 阶段 7.4 方案 | READY | 已新增阶段 7.4 方案文档 | +| 日常巡检脚本 | READY | 已新增 `scripts/run-production-watchdog.ps1` | +| 开发期验证 | WARN | `-AllowDirtyWorktree` 演练完成；生产项 PASS，WARN 来自未提交工作区 | +| 完整值守验证 | PENDING | 待执行 | +| 最终门禁 | PENDING | 待执行 | + +--- + +## 二、脚本能力 + +新增脚本： + +```powershell +scripts\run-production-watchdog.ps1 +``` + +默认聚合： + +1. `run-production-ops-check.ps1` +2. `run-capacity-governance.ps1` dry run + +可选聚合： + +1. `run-production-release-gate.ps1` +2. `run-backup-drill.ps1`，通过 release gate 的 `-IncludeBackupDrill` 触发 + +输出产物： + +1. `production-watchdog-summary.json` +2. `production-watchdog-summary.md` +3. `production-watchdog-alert.json` +4. `production-watchdog-alert.md` + +--- + +## 三、开发期验证 + +命令： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-production-watchdog.ps1 -AllowDirtyWorktree +``` + +结果： + +```text +PHASE7_PRODUCTION_WATCHDOG_WARN D:\ai-contract-platform\project-docs\qa-artifacts\phase7-production-watchdog\2026-05-01T12-48-18-008 +``` + +结论： + +1. `gstack preflight` PASS。 +2. `origin remote` PASS，仍指向 `192.168.3.90`。 +3. `production ops check` 生产子项 PASS；远端服务、健康接口、readiness、首页和磁盘阈值均通过。 +4. `capacity governance dry run` 生产子项 PASS；磁盘使用率 67%，当前备份保留策略无删除候选，Docker build cache 为 0B。 +5. 顶层状态为 WARN，原因是开发期允许未提交工作区，子脚本也继承该 advisory WARN。 + +产物： + +```text +project-docs\qa-artifacts\phase7-production-watchdog\2026-05-01T12-48-18-008 +``` diff --git a/project-docs/阶段7.4-日常巡检与告警门禁方案-2026-05-01.md b/project-docs/阶段7.4-日常巡检与告警门禁方案-2026-05-01.md new file mode 100644 index 00000000..92f52984 --- /dev/null +++ b/project-docs/阶段7.4-日常巡检与告警门禁方案-2026-05-01.md @@ -0,0 +1,70 @@ +# 阶段 7.4：日常巡检与告警门禁方案（2026-05-01） + +> 前置状态：阶段 7.2 已完成一键回滚与生产巡检门禁，阶段 7.3 已完成容量治理并解除磁盘 WARN。阶段 7.4 的目标是把这些能力聚合为一个日常生产值守入口，让系统可以被持续巡检、统一告警和按需升级到完整 release gate。 + +--- + +## 一、阶段目标 + +1. 新增统一生产值守脚本。 +2. 默认执行 ops check 和 capacity governance dry run。 +3. 可选执行完整 release gate。 +4. 统一输出 `PASS/WARN/FAIL`、`ok/warning/critical` 告警等级。 +5. 生成 `alert.json` 和 `alert.md`，为后续接入邮件、企业微信、Webhook 或任务调度预留。 +6. 支持 `-FailOnWarn`，让 WARN 在 CI/定时任务中变为非零退出。 + +--- + +## 二、交付范围 + +| 主线 | 目标 | 交付物 | +|---|---|---| +| A. 日常值守入口 | 聚合 ops check 与 capacity governance dry run | `scripts/run-production-watchdog.ps1` | +| B. 完整值守入口 | 可选包含 release gate | `-IncludeReleaseGate` | +| C. 告警产物 | 输出结构化告警 | `production-watchdog-alert.json`、`production-watchdog-alert.md` | +| D. 运维说明 | 固化人工/定时执行方式 | 运维手册与阶段 7.4 执行记录 | + +--- + +## 三、使用方式 + +日常巡检： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-production-watchdog.ps1 +``` + +严格日常巡检： + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-production-watchdog.ps1 -RequireCleanWorktree -FailOnWarn +``` + +完整值守巡检： + +```powershell +$env:AI_CONTRACT_ADMIN_PASSWORD = '<由管理员输入的实际密码>' +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-production-watchdog.ps1 -RequireCleanWorktree -IncludeReleaseGate +Remove-Item Env:\AI_CONTRACT_ADMIN_PASSWORD +``` + +--- + +## 四、告警口径 + +| 状态 | 告警等级 | 说明 | +|---|---|---| +| PASS | `ok` | required 检查全部通过，无 WARN | +| WARN | `warning` | required 检查全部通过，但存在容量、备份年龄或 advisory 风险 | +| FAIL | `critical` | 任一 required 检查失败 | + +--- + +## 五、验收标准 + +1. 日常巡检脚本可复跑。 +2. 生成 summary 和 alert 两类产物。 +3. 默认日常巡检 PASS。 +4. 完整值守巡检包含 release gate 且 PASS。 +5. 工作区最终干净。 +6. 方案、执行记录和脚本提交并推送到 `192.168.3.90` 仓库。 diff --git a/scripts/run-production-watchdog.ps1 b/scripts/run-production-watchdog.ps1 new file mode 100644 index 00000000..ffab9d87 --- /dev/null +++ b/scripts/run-production-watchdog.ps1 @@ -0,0 +1,356 @@ +param( + [string]$ServerHost = "192.168.3.90", + [string]$SshUser = "root", + [string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519", + [string]$RemoteProjectPath = "/opt/ai-contract-platform", + [string]$ComposeFile = "docker-compose.server.yml", + [string]$RemoteBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy", + [string]$RemoteRollbackRoot = "/opt/ai-contract-platform/backups/phase7-rollback", + [string]$FrontendBaseUrl = "http://192.168.3.90:3003", + [string]$BackendBaseUrl = "http://192.168.3.90:8001", + [string]$ApiUrl = "http://192.168.3.90:8001/api/v1", + [string]$HealthUrl = "http://192.168.3.90:8001/health", + [string]$ReadinessUrl = "http://192.168.3.90:8001/api/v1/system/readiness", + [string]$ArtifactRoot = "", + [int]$DiskWarnPercent = 80, + [int]$DiskFailPercent = 90, + [switch]$RequireCleanWorktree, + [switch]$AllowDirtyWorktree, + [switch]$IncludeReleaseGate, + [switch]$IncludeBackupDrill, + [switch]$SkipOpsCheck, + [switch]$SkipCapacityGovernance, + [switch]$FailOnWarn +) + +$ErrorActionPreference = "Stop" + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff" +if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7-production-watchdog" +} +if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot $ArtifactRoot +} +$artifactDir = Join-Path $ArtifactRoot $timestamp +$summaryJson = Join-Path $artifactDir "production-watchdog-summary.json" +$summaryMd = Join-Path $artifactDir "production-watchdog-summary.md" +$alertJson = Join-Path $artifactDir "production-watchdog-alert.json" +$alertMd = Join-Path $artifactDir "production-watchdog-alert.md" +$gitStatusPath = Join-Path $artifactDir "git-status.txt" + +New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null + +$checks = New-Object System.Collections.Generic.List[object] + +function Add-Check { + param( + [string]$Key, + [string]$Label, + [ValidateSet("pass", "fail", "warn", "skip")] + [string]$Status, + [string]$Detail, + [bool]$Required = $true, + [hashtable]$Meta = @{} + ) + + $checks.Add([ordered]@{ + key = $Key + label = $Label + status = $Status + required = $Required + detail = $Detail + meta = $Meta + }) | Out-Null +} + +function Save-Json { + param( + [string]$Path, + [object]$Value + ) + $Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path +} + +function Invoke-WatchStep { + param( + [string]$Key, + [string]$Label, + [string]$ScriptName, + [string[]]$Arguments, + [bool]$Required = $true + ) + + $scriptPath = Join-Path $repoRoot "scripts\$ScriptName" + $logPath = Join-Path $artifactDir "$Key.log" + $previousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $output = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $scriptPath @Arguments 2>&1 + $exitCode = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + + $outputText = ($output -join [Environment]::NewLine) + $outputText | Set-Content -Encoding UTF8 -Path $logPath + $childArtifact = $null + $artifactMatch = [regex]::Match($outputText, '([A-Z]:\\[^\r\n]+)$', [System.Text.RegularExpressions.RegexOptions]::Multiline) + if ($artifactMatch.Success) { + $childArtifact = $artifactMatch.Groups[1].Value.Trim() + } + + if ($exitCode -eq 0 -and $outputText -match "_WARN") { + Add-Check -Key $Key -Label $Label -Status "warn" -Required $Required -Detail "$Label completed with warnings" -Meta @{ + artifact_dir = $childArtifact + log = $logPath + exit_code = $exitCode + } + } elseif ($exitCode -eq 0) { + Add-Check -Key $Key -Label $Label -Status "pass" -Required $Required -Detail "$Label passed" -Meta @{ + artifact_dir = $childArtifact + log = $logPath + exit_code = $exitCode + } + } else { + Add-Check -Key $Key -Label $Label -Status "fail" -Required $Required -Detail "$Label failed" -Meta @{ + artifact_dir = $childArtifact + log = $logPath + exit_code = $exitCode + } + } +} + +if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { + Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing" +} else { + Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK" +} + +$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim() +$commit = (git -C $repoRoot rev-parse HEAD).Trim() +$remoteUrl = (git -C $repoRoot remote get-url origin).Trim() +$gitStatus = git -C $repoRoot status --short +($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath +if ($gitStatus) { + if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) { + Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes and -RequireCleanWorktree is set" -Meta @{ artifact = $gitStatusPath } + } else { + Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; watchdog is running in advisory mode" -Meta @{ artifact = $gitStatusPath } + } +} else { + Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath } +} + +if ($remoteUrl -match "192\.168\.3\.90") { + Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{ + branch = $branch + commit = $commit + remote = $remoteUrl + } +} else { + Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{ + branch = $branch + commit = $commit + remote = $remoteUrl + } +} + +$failedPreflight = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) +if ($failedPreflight.Count -eq 0) { + if ($SkipOpsCheck) { + Add-Check -Key "ops_check" -Label "production ops check" -Status "skip" -Required $false -Detail "Skipped by -SkipOpsCheck" + } else { + $opsArgs = @( + "-ServerHost", $ServerHost, + "-SshUser", $SshUser, + "-SshKeyPath", $SshKeyPath, + "-RemoteProjectPath", $RemoteProjectPath, + "-ComposeFile", $ComposeFile, + "-RemoteBackupRoot", $RemoteBackupRoot, + "-RemoteRollbackRoot", $RemoteRollbackRoot, + "-FrontendBaseUrl", $FrontendBaseUrl, + "-BackendBaseUrl", $BackendBaseUrl, + "-HealthUrl", $HealthUrl, + "-ReadinessUrl", $ReadinessUrl, + "-DiskWarnPercent", ([string]$DiskWarnPercent), + "-DiskFailPercent", ([string]$DiskFailPercent), + "-ArtifactRoot", (Join-Path $artifactDir "ops-check") + ) + if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) { + $opsArgs += "-RequireCleanWorktree" + } + Invoke-WatchStep -Key "ops_check" -Label "production ops check" -ScriptName "run-production-ops-check.ps1" -Arguments $opsArgs + } + + if ($SkipCapacityGovernance) { + Add-Check -Key "capacity_governance" -Label "capacity governance dry run" -Status "skip" -Required $false -Detail "Skipped by -SkipCapacityGovernance" + } else { + $capacityArgs = @( + "-ServerHost", $ServerHost, + "-SshUser", $SshUser, + "-SshKeyPath", $SshKeyPath, + "-RemoteProjectPath", $RemoteProjectPath, + "-RemoteDeployBackupRoot", $RemoteBackupRoot, + "-RemoteRollbackBackupRoot", $RemoteRollbackRoot, + "-DiskWarnPercent", ([string]$DiskWarnPercent), + "-DiskFailPercent", ([string]$DiskFailPercent), + "-ArtifactRoot", (Join-Path $artifactDir "capacity-governance") + ) + if ($AllowDirtyWorktree) { + $capacityArgs += "-AllowDirtyWorktree" + } + Invoke-WatchStep -Key "capacity_governance" -Label "capacity governance dry run" -ScriptName "run-capacity-governance.ps1" -Arguments $capacityArgs + } + + if ($IncludeReleaseGate) { + if ([string]::IsNullOrWhiteSpace($env:AI_CONTRACT_ADMIN_PASSWORD)) { + Add-Check -Key "release_gate" -Label "production release gate" -Status "fail" -Detail "AI_CONTRACT_ADMIN_PASSWORD must be set in the current process" + } else { + $releaseArgs = @( + "-ServerHost", $ServerHost, + "-FrontendBaseUrl", $FrontendBaseUrl, + "-BackendBaseUrl", $BackendBaseUrl, + "-ApiUrl", $ApiUrl, + "-HealthUrl", $HealthUrl, + "-ReadinessUrl", $ReadinessUrl, + "-ArtifactRoot", (Join-Path $artifactDir "release-gate") + ) + if ($RequireCleanWorktree -and -not $AllowDirtyWorktree) { + $releaseArgs += "-RequireCleanWorktree" + } + if ($IncludeBackupDrill) { + $releaseArgs += "-IncludeBackupDrill" + } + Invoke-WatchStep -Key "release_gate" -Label "production release gate" -ScriptName "run-production-release-gate.ps1" -Arguments $releaseArgs + } + } else { + Add-Check -Key "release_gate" -Label "production release gate" -Status "skip" -Required $false -Detail "Skipped by default; pass -IncludeReleaseGate for full watchdog" + } +} + +$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) +$warningsList = @($checks | Where-Object { $_.status -eq "warn" }) +$skippedList = @($checks | Where-Object { $_.status -eq "skip" }) +$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" } +$alertSeverity = if ($failedRequired.Count -gt 0) { "critical" } elseif ($warningsList.Count -gt 0) { "warning" } else { "ok" } +$alertExitCode = if ($failedRequired.Count -gt 0) { 2 } elseif ($warningsList.Count -gt 0) { 1 } else { 0 } + +$summary = [ordered]@{ + name = "phase7-production-watchdog" + status = $overallStatus + alert_severity = $alertSeverity + alert_exit_code = $alertExitCode + generated_at = (Get-Date).ToUniversalTime().ToString("o") + artifact_dir = $artifactDir + branch = $branch + commit = $commit + remote = $remoteUrl + targets = [ordered]@{ + server_host = $ServerHost + remote_project_path = $RemoteProjectPath + compose_file = $ComposeFile + frontend_base_url = $FrontendBaseUrl + backend_base_url = $BackendBaseUrl + api_url = $ApiUrl + } + options = [ordered]@{ + require_clean_worktree = [bool]$RequireCleanWorktree + allow_dirty_worktree = [bool]$AllowDirtyWorktree + include_release_gate = [bool]$IncludeReleaseGate + include_backup_drill = [bool]$IncludeBackupDrill + skip_ops_check = [bool]$SkipOpsCheck + skip_capacity_governance = [bool]$SkipCapacityGovernance + fail_on_warn = [bool]$FailOnWarn + } + thresholds = [ordered]@{ + disk_warn_percent = $DiskWarnPercent + disk_fail_percent = $DiskFailPercent + } + checks = $checks + totals = [ordered]@{ + total = $checks.Count + failed_required = $failedRequired.Count + warnings = $warningsList.Count + skipped = $skippedList.Count + } +} +Save-Json -Path $summaryJson -Value $summary + +$alert = [ordered]@{ + severity = $alertSeverity + status = $overallStatus + generated_at = $summary.generated_at + artifact_dir = $artifactDir + failed = @($failedRequired | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } }) + warnings = @($warningsList | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; detail = $_.detail } }) + child_artifacts = @($checks | Where-Object { $_.meta.artifact_dir } | ForEach-Object { [ordered]@{ key = $_.key; label = $_.label; artifact_dir = $_.meta.artifact_dir } }) +} +Save-Json -Path $alertJson -Value $alert + +$markdown = New-Object System.Collections.Generic.List[string] +$markdown.Add("# Phase 7 Production Watchdog") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("- Status: $overallStatus") | Out-Null +$markdown.Add("- Alert severity: $alertSeverity") | Out-Null +$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null +$markdown.Add("- Branch: $branch") | Out-Null +$markdown.Add("- Commit: $commit") | Out-Null +$markdown.Add("- Artifact dir: $artifactDir") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("| Check | Required | Status | Detail |") | Out-Null +$markdown.Add("|---|---:|---|---|") | Out-Null +foreach ($check in $checks) { + $markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null +} +$markdown.Add("") | Out-Null +$markdown.Add("Child artifacts:") | Out-Null +$markdown.Add("") | Out-Null +foreach ($check in $checks) { + if ($check.meta.artifact_dir) { + $markdown.Add(('- {0}: `{1}`' -f $check.label, $check.meta.artifact_dir)) | Out-Null + } +} +$markdown.Add("") | Out-Null +$markdown.Add("Local artifacts:") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add('- `production-watchdog-summary.json`') | Out-Null +$markdown.Add('- `production-watchdog-summary.md`') | Out-Null +$markdown.Add('- `production-watchdog-alert.json`') | Out-Null +$markdown.Add('- `production-watchdog-alert.md`') | Out-Null +$markdown.Add('- `git-status.txt`') | Out-Null +$markdown.Add('- `*.log`') | Out-Null +$markdown | Set-Content -Encoding UTF8 -Path $summaryMd + +$alertMarkdown = New-Object System.Collections.Generic.List[string] +$alertMarkdown.Add("# Production Watchdog Alert") | Out-Null +$alertMarkdown.Add("") | Out-Null +$alertMarkdown.Add("- Severity: $alertSeverity") | Out-Null +$alertMarkdown.Add("- Status: $overallStatus") | Out-Null +$alertMarkdown.Add("- Generated at: $($summary.generated_at)") | Out-Null +$alertMarkdown.Add("- Artifact dir: $artifactDir") | Out-Null +if ($failedRequired.Count -gt 0) { + $alertMarkdown.Add("") | Out-Null + $alertMarkdown.Add("Failures:") | Out-Null + foreach ($check in $failedRequired) { + $alertMarkdown.Add(("- {0}: {1}" -f $check.label, $check.detail)) | Out-Null + } +} +if ($warningsList.Count -gt 0) { + $alertMarkdown.Add("") | Out-Null + $alertMarkdown.Add("Warnings:") | Out-Null + foreach ($check in $warningsList) { + $alertMarkdown.Add(("- {0}: {1}" -f $check.label, $check.detail)) | Out-Null + } +} +if ($failedRequired.Count -eq 0 -and $warningsList.Count -eq 0) { + $alertMarkdown.Add("") | Out-Null + $alertMarkdown.Add("No failures or warnings.") | Out-Null +} +$alertMarkdown | Set-Content -Encoding UTF8 -Path $alertMd + +Write-Host "PHASE7_PRODUCTION_WATCHDOG_$overallStatus $artifactDir" +if ($overallStatus -eq "FAIL" -or ($FailOnWarn -and $overallStatus -eq "WARN")) { + throw "Phase 7 production watchdog $overallStatus. See $summaryMd" +}