Files
ai-contract-platform/scripts/run-capacity-governance.ps1
T

436 lines
17 KiB
PowerShell

param(
[string]$ServerHost = "192.168.3.90",
[string]$SshUser = "root",
[string]$SshKeyPath = "$HOME\.ssh\water_audit_ed25519",
[string]$RemoteProjectPath = "/opt/ai-contract-platform",
[string]$RemoteDeployBackupRoot = "/opt/ai-contract-platform/backups/phase7-deploy",
[string]$RemoteRollbackBackupRoot = "/opt/ai-contract-platform/backups/phase7-rollback",
[string]$ArtifactRoot = "",
[int]$DiskWarnPercent = 80,
[int]$DiskFailPercent = 90,
[int]$DeployBackupKeep = 10,
[int]$DeployBackupMaxAgeDays = 14,
[int]$RollbackBackupKeep = 10,
[int]$RollbackBackupMaxAgeDays = 14,
[int]$DockerBuilderPruneUntilHours = 24,
[int]$DockerCommandTimeoutSeconds = 180,
[switch]$Apply,
[switch]$SkipBackupCleanup,
[switch]$SkipDockerPrune,
[switch]$AllowDirtyWorktree
)
$ErrorActionPreference = "Stop"
$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..")
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase7-capacity-governance"
}
if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) {
$ArtifactRoot = Join-Path $repoRoot $ArtifactRoot
}
$artifactDir = Join-Path $ArtifactRoot $timestamp
$summaryJson = Join-Path $artifactDir "capacity-governance-summary.json"
$summaryMd = Join-Path $artifactDir "capacity-governance-summary.md"
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
$remoteCapacityPath = Join-Path $artifactDir "remote-capacity-output.txt"
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
$checks = New-Object System.Collections.Generic.List[object]
function Add-Check {
param(
[string]$Key,
[string]$Label,
[ValidateSet("pass", "fail", "warn", "skip")]
[string]$Status,
[string]$Detail,
[bool]$Required = $true,
[hashtable]$Meta = @{}
)
$checks.Add([ordered]@{
key = $Key
label = $Label
status = $Status
required = $Required
detail = $Detail
meta = $Meta
}) | Out-Null
}
function Save-Json {
param(
[string]$Path,
[object]$Value
)
$Value | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 -Path $Path
}
function Convert-KeyValueOutput {
param([string]$Text)
$data = @{}
foreach ($line in ($Text -split "`r?`n")) {
if ($line -match '^([A-Z0-9_]+)=(.*)$') {
$data[$matches[1]] = $matches[2]
}
}
return $data
}
function Invoke-Remote {
param(
[string]$Name,
[string]$Command
)
$target = "${SshUser}@${ServerHost}"
$logPath = Join-Path $artifactDir "$Name.log"
$args = @(
"-i", $SshKeyPath,
"-o", "BatchMode=yes",
"-o", "StrictHostKeyChecking=accept-new",
"-o", "ConnectTimeout=10",
$target,
"bash",
"-s"
)
$previousErrorActionPreference = $ErrorActionPreference
$ErrorActionPreference = "Continue"
try {
$output = $Command | & ssh @args 2>&1
$exitCode = $LASTEXITCODE
} finally {
$ErrorActionPreference = $previousErrorActionPreference
}
$outputText = ($output -join [Environment]::NewLine)
$outputText | Set-Content -Encoding UTF8 -Path $logPath
return [ordered]@{
exit_code = $exitCode
output = $outputText
log = $logPath
}
}
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing"
} else {
Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK"
}
if (-not (Test-Path $SshKeyPath)) {
Add-Check -Key "ssh_key" -Label "SSH key" -Status "fail" -Detail "SSH key not found: $SshKeyPath"
} else {
Add-Check -Key "ssh_key" -Label "SSH key" -Status "pass" -Detail "SSH key exists"
}
$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
$gitStatus = git -C $repoRoot status --short
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
if ($gitStatus) {
if ($AllowDirtyWorktree) {
Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; capacity governance allowed by -AllowDirtyWorktree" -Meta @{ artifact = $gitStatusPath }
} else {
Add-Check -Key "working_tree" -Label "working tree" -Status "fail" -Detail "Working tree has uncommitted changes; pass -AllowDirtyWorktree for development validation" -Meta @{ artifact = $gitStatusPath }
}
} else {
Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath }
}
if ($remoteUrl -match "192\.168\.3\.90") {
Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{
branch = $branch
commit = $commit
remote = $remoteUrl
}
} else {
Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{
branch = $branch
commit = $commit
remote = $remoteUrl
}
}
$failedPreflight = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
if ($failedPreflight.Count -eq 0) {
$applyFlag = if ($Apply) { "true" } else { "false" }
$skipBackupFlag = if ($SkipBackupCleanup) { "true" } else { "false" }
$skipDockerFlag = if ($SkipDockerPrune) { "true" } else { "false" }
$remoteScript = @"
set -euo pipefail
project='$RemoteProjectPath'
deploy_root='$RemoteDeployBackupRoot'
rollback_root='$RemoteRollbackBackupRoot'
apply='$applyFlag'
skip_backup='$skipBackupFlag'
skip_docker='$skipDockerFlag'
deploy_keep='$DeployBackupKeep'
deploy_days='$DeployBackupMaxAgeDays'
rollback_keep='$RollbackBackupKeep'
rollback_days='$RollbackBackupMaxAgeDays'
builder_until_hours='$DockerBuilderPruneUntilHours'
docker_timeout='$DockerCommandTimeoutSeconds'
emit_disk() {
label="`$1"
used="`$(df -P "`$project" | awk 'NR==2 {print `$5}' | tr -d %)"
avail="`$(df -P "`$project" | awk 'NR==2 {print `$4}')"
used_kb="`$(df -P "`$project" | awk 'NR==2 {print `$3}')"
echo "DISK_`${label}_USED_PERCENT=`$used"
echo "DISK_`${label}_AVAILABLE_KB=`$avail"
echo "DISK_`${label}_USED_KB=`$used_kb"
}
scan_backups() {
prefix="`$1"
root="`$2"
pattern="`$3"
keep="`$4"
max_age_days="`$5"
now="`$(date +%s)"
count=0
candidate_count=0
candidate_bytes=0
deleted_count=0
deleted_bytes=0
mkdir -p "`$root"
while read -r epoch_float size path; do
[ -n "`$path" ] || continue
epoch="`$(printf '%s\n' "`$epoch_float" | cut -d. -f1)"
count="`$((count + 1))"
age_days="`$(((now - epoch) / 86400))"
should_delete=false
if [ "`$count" -gt "`$keep" ] && [ "`$age_days" -gt "`$max_age_days" ]; then
should_delete=true
candidate_count="`$((candidate_count + 1))"
candidate_bytes="`$((candidate_bytes + size))"
echo "`${prefix}_DELETE_CANDIDATE=`$path|`$size|`$age_days"
fi
if [ "`$apply" = "true" ] && [ "`$skip_backup" != "true" ] && [ "`$should_delete" = "true" ]; then
case "`$path" in
"`$root"/*) rm -f -- "`$path" ;;
*) echo "`${prefix}_DELETE_REFUSED=`$path" ;;
esac
if [ ! -e "`$path" ]; then
deleted_count="`$((deleted_count + 1))"
deleted_bytes="`$((deleted_bytes + size))"
fi
fi
done < <(find "`$root" -maxdepth 1 -type f -name "`$pattern" -printf '%T@ %s %p\n' 2>/dev/null | sort -nr)
echo "`${prefix}_TOTAL_COUNT=`$count"
echo "`${prefix}_DELETE_CANDIDATE_COUNT=`$candidate_count"
echo "`${prefix}_DELETE_CANDIDATE_BYTES=`$candidate_bytes"
echo "`${prefix}_DELETED_COUNT=`$deleted_count"
echo "`${prefix}_DELETED_BYTES=`$deleted_bytes"
}
cd "`$project"
test "`$PWD" = "`$project"
echo "REMOTE_PROJECT_OK=1"
echo "APPLY_MODE=`$apply"
emit_disk BEFORE
echo "DOCKER_SYSTEM_DF_BEFORE_BEGIN"
timeout 20 docker system df || true
echo "DOCKER_SYSTEM_DF_BEFORE_END"
scan_backups DEPLOY_BACKUP "`$deploy_root" 'frontend-before-*.tgz' "`$deploy_keep" "`$deploy_days"
scan_backups ROLLBACK_BACKUP "`$rollback_root" 'frontend-current-before-rollback-*.tgz' "`$rollback_keep" "`$rollback_days"
if [ "`$apply" = "true" ] && [ "`$skip_docker" != "true" ]; then
echo "DOCKER_BUILDER_PRUNE_BEGIN"
if timeout "`$docker_timeout" docker builder prune -f --filter "until=`${builder_until_hours}h"; then
echo "DOCKER_BUILDER_PRUNE_EXIT=0"
else
echo "DOCKER_BUILDER_PRUNE_EXIT=`$?"
fi
echo "DOCKER_BUILDER_PRUNE_END"
echo "DOCKER_IMAGE_PRUNE_BEGIN"
if timeout "`$docker_timeout" docker image prune -f; then
echo "DOCKER_IMAGE_PRUNE_EXIT=0"
else
echo "DOCKER_IMAGE_PRUNE_EXIT=`$?"
fi
echo "DOCKER_IMAGE_PRUNE_END"
else
echo "DOCKER_PRUNE_SKIPPED=1"
fi
emit_disk AFTER
echo "DOCKER_SYSTEM_DF_AFTER_BEGIN"
timeout 20 docker system df || true
echo "DOCKER_SYSTEM_DF_AFTER_END"
"@
$capacityRun = Invoke-Remote -Name "01-remote-capacity-governance" -Command $remoteScript
$capacityRun.output | Set-Content -Encoding UTF8 -Path $remoteCapacityPath
$remoteData = Convert-KeyValueOutput -Text $capacityRun.output
if ($capacityRun.exit_code -eq 0 -and $remoteData.REMOTE_PROJECT_OK -eq "1") {
Add-Check -Key "remote_capacity_run" -Label "remote capacity run" -Status "pass" -Detail "Remote capacity governance script completed" -Meta @{ artifact = $remoteCapacityPath }
} else {
Add-Check -Key "remote_capacity_run" -Label "remote capacity run" -Status "fail" -Detail "Remote capacity governance script failed" -Meta @{
exit_code = $capacityRun.exit_code
artifact = $remoteCapacityPath
}
}
$beforeDisk = if ($remoteData.DISK_BEFORE_USED_PERCENT) { [int]$remoteData.DISK_BEFORE_USED_PERCENT } else { $null }
$afterDisk = if ($remoteData.DISK_AFTER_USED_PERCENT) { [int]$remoteData.DISK_AFTER_USED_PERCENT } else { $null }
if ($null -eq $afterDisk) {
Add-Check -Key "disk_threshold" -Label "disk threshold" -Status "fail" -Detail "Disk usage could not be read from remote capacity output" -Meta @{
before = $beforeDisk
after = $null
artifact = $remoteCapacityPath
}
} elseif ($afterDisk -ge $DiskFailPercent) {
Add-Check -Key "disk_threshold" -Label "disk threshold" -Status "fail" -Detail "Disk usage is $afterDisk%, fail threshold is $DiskFailPercent%" -Meta @{
before = $beforeDisk
after = $afterDisk
artifact = $remoteCapacityPath
}
} elseif ($afterDisk -ge $DiskWarnPercent) {
Add-Check -Key "disk_threshold" -Label "disk threshold" -Status "warn" -Required $false -Detail "Disk usage is $afterDisk%, warn threshold is $DiskWarnPercent%" -Meta @{
before = $beforeDisk
after = $afterDisk
artifact = $remoteCapacityPath
}
} else {
Add-Check -Key "disk_threshold" -Label "disk threshold" -Status "pass" -Detail "Disk usage is $afterDisk%" -Meta @{
before = $beforeDisk
after = $afterDisk
artifact = $remoteCapacityPath
}
}
$deployCandidates = if ($remoteData.DEPLOY_BACKUP_DELETE_CANDIDATE_COUNT) { [int]$remoteData.DEPLOY_BACKUP_DELETE_CANDIDATE_COUNT } else { 0 }
$rollbackCandidates = if ($remoteData.ROLLBACK_BACKUP_DELETE_CANDIDATE_COUNT) { [int]$remoteData.ROLLBACK_BACKUP_DELETE_CANDIDATE_COUNT } else { 0 }
$deployDeleted = if ($remoteData.DEPLOY_BACKUP_DELETED_COUNT) { [int]$remoteData.DEPLOY_BACKUP_DELETED_COUNT } else { 0 }
$rollbackDeleted = if ($remoteData.ROLLBACK_BACKUP_DELETED_COUNT) { [int]$remoteData.ROLLBACK_BACKUP_DELETED_COUNT } else { 0 }
if ($SkipBackupCleanup) {
Add-Check -Key "backup_retention" -Label "backup retention" -Status "skip" -Required $false -Detail "Skipped by -SkipBackupCleanup"
} elseif ($Apply) {
Add-Check -Key "backup_retention" -Label "backup retention" -Status "pass" -Detail "Backup retention applied; deleted deploy=$deployDeleted rollback=$rollbackDeleted" -Meta @{
deploy_candidates = $deployCandidates
rollback_candidates = $rollbackCandidates
artifact = $remoteCapacityPath
}
} elseif (($deployCandidates + $rollbackCandidates) -gt 0) {
Add-Check -Key "backup_retention" -Label "backup retention" -Status "warn" -Required $false -Detail "Dry run found backup deletion candidates" -Meta @{
deploy_candidates = $deployCandidates
rollback_candidates = $rollbackCandidates
artifact = $remoteCapacityPath
}
} else {
Add-Check -Key "backup_retention" -Label "backup retention" -Status "pass" -Detail "No backup deletion candidates under current retention policy" -Meta @{
deploy_total = $remoteData.DEPLOY_BACKUP_TOTAL_COUNT
rollback_total = $remoteData.ROLLBACK_BACKUP_TOTAL_COUNT
artifact = $remoteCapacityPath
}
}
if ($SkipDockerPrune) {
Add-Check -Key "docker_prune" -Label "Docker prune" -Status "skip" -Required $false -Detail "Skipped by -SkipDockerPrune"
} elseif ($Apply) {
$builderExit = if ($remoteData.DOCKER_BUILDER_PRUNE_EXIT) { [int]$remoteData.DOCKER_BUILDER_PRUNE_EXIT } else { -1 }
$imageExit = if ($remoteData.DOCKER_IMAGE_PRUNE_EXIT) { [int]$remoteData.DOCKER_IMAGE_PRUNE_EXIT } else { -1 }
if ($builderExit -eq 0 -and $imageExit -eq 0) {
Add-Check -Key "docker_prune" -Label "Docker prune" -Status "pass" -Detail "Docker builder prune and dangling image prune completed" -Meta @{ artifact = $remoteCapacityPath }
} else {
Add-Check -Key "docker_prune" -Label "Docker prune" -Status "warn" -Required $false -Detail "Docker prune completed with non-zero exit or timeout" -Meta @{
builder_exit = $builderExit
image_exit = $imageExit
artifact = $remoteCapacityPath
}
}
} else {
Add-Check -Key "docker_prune" -Label "Docker prune" -Status "skip" -Required $false -Detail "Dry run only; pass -Apply to prune Docker builder cache and dangling images"
}
}
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
$warningsList = @($checks | Where-Object { $_.status -eq "warn" })
$skippedList = @($checks | Where-Object { $_.status -eq "skip" })
$overallStatus = if ($failedRequired.Count -gt 0) { "FAIL" } elseif ($warningsList.Count -gt 0) { "WARN" } else { "PASS" }
$summary = [ordered]@{
name = "phase7-capacity-governance"
status = $overallStatus
generated_at = (Get-Date).ToUniversalTime().ToString("o")
artifact_dir = $artifactDir
branch = $branch
commit = $commit
remote = $remoteUrl
targets = [ordered]@{
server_host = $ServerHost
remote_project_path = $RemoteProjectPath
deploy_backup_root = $RemoteDeployBackupRoot
rollback_backup_root = $RemoteRollbackBackupRoot
}
policy = [ordered]@{
disk_warn_percent = $DiskWarnPercent
disk_fail_percent = $DiskFailPercent
deploy_backup_keep = $DeployBackupKeep
deploy_backup_max_age_days = $DeployBackupMaxAgeDays
rollback_backup_keep = $RollbackBackupKeep
rollback_backup_max_age_days = $RollbackBackupMaxAgeDays
docker_builder_prune_until_hours = $DockerBuilderPruneUntilHours
docker_command_timeout_seconds = $DockerCommandTimeoutSeconds
}
options = [ordered]@{
apply = [bool]$Apply
skip_backup_cleanup = [bool]$SkipBackupCleanup
skip_docker_prune = [bool]$SkipDockerPrune
allow_dirty_worktree = [bool]$AllowDirtyWorktree
}
checks = $checks
totals = [ordered]@{
total = $checks.Count
failed_required = $failedRequired.Count
warnings = $warningsList.Count
skipped = $skippedList.Count
}
}
Save-Json -Path $summaryJson -Value $summary
$markdown = New-Object System.Collections.Generic.List[string]
$markdown.Add("# Phase 7 Capacity Governance") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add("- Status: $overallStatus") | Out-Null
$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null
$markdown.Add("- Branch: $branch") | Out-Null
$markdown.Add("- Commit: $commit") | Out-Null
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
$markdown.Add("- Apply mode: $([bool]$Apply)") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add("| Check | Required | Status | Detail |") | Out-Null
$markdown.Add("|---|---:|---|---|") | Out-Null
foreach ($check in $checks) {
$markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null
}
$markdown.Add("") | Out-Null
$markdown.Add("Policy:") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add("- Disk warn/fail: $DiskWarnPercent% / $DiskFailPercent%") | Out-Null
$markdown.Add("- Deploy backup retention: keep $DeployBackupKeep and only delete files older than $DeployBackupMaxAgeDays days") | Out-Null
$markdown.Add("- Rollback backup retention: keep $RollbackBackupKeep and only delete files older than $RollbackBackupMaxAgeDays days") | Out-Null
$markdown.Add("- Docker builder prune: until ${DockerBuilderPruneUntilHours}h") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add("Local artifacts:") | Out-Null
$markdown.Add("") | Out-Null
$markdown.Add('- `capacity-governance-summary.json`') | Out-Null
$markdown.Add('- `capacity-governance-summary.md`') | Out-Null
$markdown.Add('- `git-status.txt`') | Out-Null
$markdown.Add('- `remote-capacity-output.txt`') | Out-Null
$markdown.Add('- `*.log`') | Out-Null
$markdown | Set-Content -Encoding UTF8 -Path $summaryMd
Write-Host "PHASE7_CAPACITY_GOVERNANCE_$overallStatus $artifactDir"
if ($overallStatus -eq "FAIL") {
throw "Phase 7 capacity governance failed. See $summaryMd"
}