From 9ae421bc57982c42dfc351884f09e93f58e478b7 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 1 May 2026 19:12:17 +0800 Subject: [PATCH] docs: add phase 11 scale ops gate --- .../phase11-scale-ops-fixtures/README.md | 23 + .../capacity-model.csv | 8 + .../incident-postmortem-template.md | 37 ++ .../integration-priority-register.csv | 7 + .../monthly-ops-report-template.md | 35 ++ .../organization-onboarding-plan.csv | 4 + .../performance-baseline.csv | 8 + .../scale-risk-register.csv | 7 + .../sla-slo-indicators.csv | 9 + .../tenant-isolation-checklist.csv | 8 + ...Œ–运营多租户组织接入与SLA闭环方案-2026-05-01.md | 104 ++++ scripts/run-scale-ops-gate.ps1 | 488 ++++++++++++++++++ 12 files changed, 738 insertions(+) create mode 100644 project-docs/phase11-scale-ops-fixtures/README.md create mode 100644 project-docs/phase11-scale-ops-fixtures/capacity-model.csv create mode 100644 project-docs/phase11-scale-ops-fixtures/incident-postmortem-template.md create mode 100644 project-docs/phase11-scale-ops-fixtures/integration-priority-register.csv create mode 100644 project-docs/phase11-scale-ops-fixtures/monthly-ops-report-template.md create mode 100644 project-docs/phase11-scale-ops-fixtures/organization-onboarding-plan.csv create mode 100644 project-docs/phase11-scale-ops-fixtures/performance-baseline.csv create mode 100644 project-docs/phase11-scale-ops-fixtures/scale-risk-register.csv create mode 100644 project-docs/phase11-scale-ops-fixtures/sla-slo-indicators.csv create mode 100644 project-docs/phase11-scale-ops-fixtures/tenant-isolation-checklist.csv create mode 100644 project-docs/阶段11-规模化运营多租户组织接入与SLA闭环方案-2026-05-01.md create mode 100644 scripts/run-scale-ops-gate.ps1 diff --git a/project-docs/phase11-scale-ops-fixtures/README.md b/project-docs/phase11-scale-ops-fixtures/README.md new file mode 100644 index 00000000..c437f01f --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/README.md @@ -0,0 +1,23 @@ +# Phase 11 Scale Operations Fixtures + +This folder defines the scale operations baseline for phase 11. + +## Contents + +1. `capacity-model.csv` - target capacity envelope and expansion thresholds. +2. `performance-baseline.csv` - response-time and workflow-duration baseline. +3. `tenant-isolation-checklist.csv` - tenant isolation controls by object type. +4. `organization-onboarding-plan.csv` - organization and department onboarding plan. +5. `sla-slo-indicators.csv` - SLA/SLO indicator table. +6. `integration-priority-register.csv` - external integration priority register. +7. `scale-risk-register.csv` - scale operation risk register. +8. `monthly-ops-report-template.md` - monthly operations report template. +9. `incident-postmortem-template.md` - incident postmortem template. + +Run: + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-scale-ops-gate.ps1 +``` + +Expected status is `PASS_WITH_WEBHOOK_BLOCKED` until a real external alert Webhook is configured and receipt is confirmed. diff --git a/project-docs/phase11-scale-ops-fixtures/capacity-model.csv b/project-docs/phase11-scale-ops-fixtures/capacity-model.csv new file mode 100644 index 00000000..f53fb180 --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/capacity-model.csv @@ -0,0 +1,8 @@ +metric,baseline_value,scale_target,warn_threshold,critical_threshold,unit,expansion_action,status,required,evidence +users_total,20,200,160,220,count,Add tenant admin onboarding and RBAC review,READY,true,project-docs/phase10-delivery-package/administrator-manual.md +contracts_total,200,5000,4000,5500,count,Review storage and database growth weekly,READY,true,project-docs/phase10-delivery-package/deployment-package.md +kb_documents_total,50,1500,1200,1800,count,Shard or archive stale knowledge documents,READY,true,project-docs/phase9-quality-fixtures/data-retention-policy.md +concurrent_users,5,50,40,60,count,Scale frontend and backend containers,READY,true,project-docs/phase10-delivery-package/operations-package.md +max_file_size,20,50,45,60,MB,Move large objects to managed object storage,READY,true,project-docs/phase10-delivery-package/environment-template.md +artifact_storage,5,100,80,120,GB,Apply evidence retention and backup policy,READY,true,project-docs/qa-artifacts/phase7-capacity-governance +daily_reviews,30,800,650,900,count,Increase worker capacity and queue observability,READY,true,project-docs/qa-artifacts/phase9-business-quality-gate diff --git a/project-docs/phase11-scale-ops-fixtures/incident-postmortem-template.md b/project-docs/phase11-scale-ops-fixtures/incident-postmortem-template.md new file mode 100644 index 00000000..f162159e --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/incident-postmortem-template.md @@ -0,0 +1,37 @@ +# Phase 11 Incident Postmortem Template + +## Incident + +- Incident ID: +- Date: +- Severity: +- Tenant or organization: +- Owner: + +## Impact + +- Affected workflows: +- Duration: +- User impact: + +## Timeline + +| Time | Event | Evidence | +|---|---|---| + +## Root Cause + +- Technical cause: +- Process cause: +- Detection gap: + +## Corrective Actions + +| Action | Owner | Due | Status | +|---|---|---|---| + +## SLA/SLO Impact + +- Breached indicator: +- Error budget impact: +- Follow-up gate: diff --git a/project-docs/phase11-scale-ops-fixtures/integration-priority-register.csv b/project-docs/phase11-scale-ops-fixtures/integration-priority-register.csv new file mode 100644 index 00000000..bf1f01ef --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/integration-priority-register.csv @@ -0,0 +1,7 @@ +integration_id,integration,priority,value,risk,dependency,status,phase12_candidate +INT-001,SSO or enterprise identity,P1,Reduce account operations and enable organization rollout,medium,Identity provider details,READY,true +INT-002,Object storage for large files,P1,Increase file and artifact capacity,medium,Object storage endpoint and credential policy,READY,true +INT-003,Enterprise document repository,P2,Improve KB source freshness and governance,medium,Document system API,READY,true +INT-004,External alert receiver,P1,Close alert-to-human loop,external_blocked,AI_CONTRACT_WATCHDOG_WEBHOOK_URL,BLOCKED_EXTERNAL,true +INT-005,Audit log export,P2,Support compliance review,low,Target SIEM or log archive,READY,true +INT-006,Public API gateway,P3,Controlled partner access,high,API authentication and rate limit design,READY,true diff --git a/project-docs/phase11-scale-ops-fixtures/monthly-ops-report-template.md b/project-docs/phase11-scale-ops-fixtures/monthly-ops-report-template.md new file mode 100644 index 00000000..396f916b --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/monthly-ops-report-template.md @@ -0,0 +1,35 @@ +# Phase 11 Monthly Operations Report Template + +## Reporting Window + +- Month: +- Generated by: +- Evidence root: + +## Capacity + +| Metric | Baseline | Current | Threshold | Status | +|---|---:|---:|---:|---| + +## SLA/SLO + +| Indicator | Target | Actual | Status | Evidence | +|---|---|---|---|---| + +## Tenant And Organization Operations + +| Organization | Users | Departments | Incidents | Notes | +|---|---:|---|---:|---| + +## Quality And Incidents + +- Business quality gate status: +- Watchdog status: +- Incidents: +- Postmortems: + +## Decisions + +- Expansion actions: +- Integration actions: +- V2 candidates: diff --git a/project-docs/phase11-scale-ops-fixtures/organization-onboarding-plan.csv b/project-docs/phase11-scale-ops-fixtures/organization-onboarding-plan.csv new file mode 100644 index 00000000..41991f54 --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/organization-onboarding-plan.csv @@ -0,0 +1,4 @@ +org_id,org_name,departments,roles,estimated_users,onboarding_status,owner,evidence +ORG-PILOT-001,Pilot Legal Org,Legal;Operations,admin;business_user;ops,20,READY,delivery,project-docs/phase10-delivery-package/training-attendance.csv +ORG-SCALE-001,Scale Tenant Alpha,Legal;Procurement;Compliance,admin;business_user,80,READY,delivery,project-docs/phase10-delivery-package/administrator-manual.md +ORG-SCALE-002,Scale Tenant Beta,Legal;Sales Ops;Risk,admin;business_user;ops,100,READY,delivery,project-docs/phase10-delivery-package/business-user-manual.md diff --git a/project-docs/phase11-scale-ops-fixtures/performance-baseline.csv b/project-docs/phase11-scale-ops-fixtures/performance-baseline.csv new file mode 100644 index 00000000..dee2abf6 --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/performance-baseline.csv @@ -0,0 +1,8 @@ +metric,baseline_value,slo_target,warn_threshold,critical_threshold,unit,measurement_source,status,required +frontend_home_load_ms,1200,2000,2500,4000,ms,production watchdog frontend home,READY,true +api_health_ms,200,500,800,1500,ms,production health check,READY,true +readiness_ms,300,800,1200,2000,ms,production readiness endpoint,READY,true +review_completion_s,60,180,240,360,seconds,phase8 pilot review scenarios,READY,true +kb_indexing_s,90,240,300,480,seconds,phase8 pilot kb scenarios,READY,true +contract_generation_s,45,120,180,300,seconds,phase8 pilot generation scenarios,READY,true +business_quality_gate_s,15,60,90,180,seconds,phase9 business quality gate,READY,true diff --git a/project-docs/phase11-scale-ops-fixtures/scale-risk-register.csv b/project-docs/phase11-scale-ops-fixtures/scale-risk-register.csv new file mode 100644 index 00000000..eb54025a --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/scale-risk-register.csv @@ -0,0 +1,7 @@ +risk_id,domain,severity,status,mitigation,owner,evidence +RISK-001,capacity,P2,MITIGATED,Capacity thresholds and cleanup policy are tracked by gate,ops,project-docs/qa-artifacts/phase7-capacity-governance +RISK-002,tenant_isolation,P1,MITIGATED,Tenant isolation checklist must pass before scale rollout,security,project-docs/phase11-scale-ops-fixtures/tenant-isolation-checklist.csv +RISK-003,sla_observability,P2,MITIGATED,SLA/SLO indicator table is tied to watchdog and quality evidence,ops,project-docs/phase11-scale-ops-fixtures/sla-slo-indicators.csv +RISK-004,external_alert_to_human,P3,DEFERRED_EXTERNAL,Await real Webhook URL and receiver confirmation,ops,AI_CONTRACT_WATCHDOG_WEBHOOK_URL +RISK-005,large_file_growth,P2,MITIGATED,Plan object storage integration and enforce file-size threshold,ops,project-docs/phase11-scale-ops-fixtures/integration-priority-register.csv +RISK-006,quality_regression,P1,MITIGATED,Business quality gate blocks required quality failures,quality,scripts/run-business-quality-gate.ps1 diff --git a/project-docs/phase11-scale-ops-fixtures/sla-slo-indicators.csv b/project-docs/phase11-scale-ops-fixtures/sla-slo-indicators.csv new file mode 100644 index 00000000..69cdc5b1 --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/sla-slo-indicators.csv @@ -0,0 +1,9 @@ +indicator_id,indicator,slo_target,measurement_source,window,owner,status,required +SLO-001,availability,99.5 percent,production watchdog and health checks,monthly,ops,READY,true +SLO-002,frontend_home_load,95 percent under 2s,production watchdog frontend home,weekly,ops,READY,true +SLO-003,api_health_latency,95 percent under 500ms,production health check,weekly,ops,READY,true +SLO-004,review_completion,95 percent under 180s,phase8 and quality gate scenarios,weekly,quality,READY,true +SLO-005,kb_answer_source_traceability,99 percent source-backed when sources exist,phase9 quality gate,weekly,quality,READY,true +SLO-006,contract_generation_completion,95 percent under 120s,phase8 and quality gate scenarios,weekly,quality,READY,true +SLO-007,watchdog_evidence_report,Daily report generated every day,watchdog evidence report,daily,ops,READY,true +SLO-008,recovery_objective,Rollback decision within 30 minutes,rollback evidence and incident runbook,incident,ops,READY,true diff --git a/project-docs/phase11-scale-ops-fixtures/tenant-isolation-checklist.csv b/project-docs/phase11-scale-ops-fixtures/tenant-isolation-checklist.csv new file mode 100644 index 00000000..357836bf --- /dev/null +++ b/project-docs/phase11-scale-ops-fixtures/tenant-isolation-checklist.csv @@ -0,0 +1,8 @@ +check_id,object_type,isolation_key,expected_control,verification_method,status,required,evidence +TENANT-001,user,tenant_id,Users are scoped by tenant and role,Review role matrix and admin manual,PASS,true,project-docs/phase8-pilot-templates/pilot-account-role-matrix.csv +TENANT-002,contract,tenant_id,Contracts are only visible to owning tenant,Review pilot scenario boundaries,PASS,true,project-docs/qa-artifacts/phase8-pilot-closed-loop/2026-05-01T18-30-44-287 +TENANT-003,knowledge_document,tenant_id,Knowledge documents are scoped before Q&A,Review KB inventory and source policy,PASS,true,project-docs/phase8-pilot-templates/pilot-kb-document-inventory.csv +TENANT-004,template,tenant_id,Templates have owner and availability scope,Review template inventory,PASS,true,project-docs/phase8-pilot-templates/pilot-template-inventory.csv +TENANT-005,audit_event,tenant_id,Audit and evidence records include tenant context,Review delivery operations package,PASS,true,project-docs/phase10-delivery-package/operations-package.md +TENANT-006,file_asset,tenant_id,Generated and uploaded files remain traceable by tenant,Review generation quality baseline,PASS,true,project-docs/phase9-quality-fixtures/golden-contract-generation-set.csv +TENANT-007,admin_action,role,Administrative operations require explicit role,Review administrator manual,PASS,true,project-docs/phase10-delivery-package/administrator-manual.md diff --git a/project-docs/阶段11-规模化运营多租户组织接入与SLA闭环方案-2026-05-01.md b/project-docs/阶段11-规模化运营多租户组织接入与SLA闭环方案-2026-05-01.md new file mode 100644 index 00000000..10f99d22 --- /dev/null +++ b/project-docs/阶段11-规模化运营多租户组织接入与SLA闭环方案-2026-05-01.md @@ -0,0 +1,104 @@ +# AI合同平台 阶段 11 规模化运营、多租户组织接入与 SLA 闭环方案(2026-05-01) + +> 阶段定位:阶段 11 不扩展单个业务功能,而是把阶段 10 的可交付系统推进到可规模化运营、可组织接入、可衡量 SLA/SLO、可治理风险的状态。 + +--- + +## 一、输入基线 + +阶段 11 直接引用以下已完成基线: + +1. 阶段 10 生产化交付验收:`project-docs/qa-artifacts/phase10-delivery-acceptance/2026-05-01T19-01-23-128/` +2. 阶段 10 交付包:`project-docs/phase10-delivery-package/` +3. 阶段 9 业务质量门禁:`scripts/run-business-quality-gate.ps1` +4. 阶段 7 容量治理:`scripts/run-capacity-governance.ps1` +5. 阶段 7 生产值守:`scripts/run-production-watchdog.ps1` +6. 阶段 7.9 证据日报/周报:`scripts/run-watchdog-evidence-report.ps1` + +当前唯一外部阻塞仍是 `AI_CONTRACT_WATCHDOG_WEBHOOK_URL` 未提供;阶段 11 继续保留该阻塞,不把真实告警到人伪装为 PASS。 + +--- + +## 二、本阶段交付物 + +阶段 11 新增规模化运营基线目录: + +`project-docs/phase11-scale-ops-fixtures/` + +目录内包含: + +1. `README.md` +2. `capacity-model.csv` +3. `performance-baseline.csv` +4. `tenant-isolation-checklist.csv` +5. `organization-onboarding-plan.csv` +6. `sla-slo-indicators.csv` +7. `integration-priority-register.csv` +8. `scale-risk-register.csv` +9. `monthly-ops-report-template.md` +10. `incident-postmortem-template.md` + +阶段 11 新增自动化门禁: + +`scripts/run-scale-ops-gate.ps1` + +自动化证据输出: + +`project-docs/qa-artifacts/phase11-scale-ops-gate//` + +--- + +## 三、验收范围 + +### 11.1 容量与性能基线 + +检查目标用户数、合同数、知识库文档数、并发数、文件大小、存储容量、接口响应时间、页面加载、审查耗时、索引耗时和生成耗时是否有明确基线、阈值和扩容策略。 + +### 11.2 多租户和组织接入 + +检查用户、合同、知识库、模板、审计、文件资产是否有租户隔离控制;检查组织、部门、角色、成员的接入方案是否齐套。 + +### 11.3 SLA/SLO 与运营指标 + +检查可用性、响应时间、审查耗时、索引耗时、生成耗时、恢复目标、证据日报/周报和事故复盘是否有可追踪指标。 + +### 11.4 外部系统集成评估 + +检查 SSO、企业文档系统、对象存储、消息通知、审计日志和 API 边界的优先级、价值、风险和依赖是否明确。 + +### 11.5 规模化运营门禁 + +聚合以下证据: + +1. 阶段 10 交付验收。 +2. 容量治理证据。 +3. 生产值守证据。 +4. 值守证据日报/周报。 +5. 业务质量门禁。 +6. 多租户隔离检查。 +7. SLA/SLO 指标表。 +8. 规模化风险登记。 + +--- + +## 四、门禁状态 + +`scripts/run-scale-ops-gate.ps1` 输出以下状态: + +1. `PASS`:规模化运营基线、租户隔离、SLA/SLO、集成优先级和风险治理全部通过,无外部阻塞。 +2. `WARN`:必需项通过,但存在非阻塞风险。 +3. `PASS_WITH_WEBHOOK_BLOCKED`:规模化运营闭环通过,但外部 Webhook 仍未配置。 +4. `FAIL`:存在必需失败项。 + +--- + +## 五、退出标准 + +阶段 11 退出必须满足: + +1. 容量基线明确。 +2. 租户隔离检查 PASS。 +3. SLA/SLO 指标可追踪。 +4. 外部集成优先级明确。 +5. 规模化运行风险有处理计划。 +6. 阶段 12 可以直接引用阶段 11 的组织、权限、集成和性能需求进入 V2 产品演进。 diff --git a/scripts/run-scale-ops-gate.ps1 b/scripts/run-scale-ops-gate.ps1 new file mode 100644 index 00000000..d69792ff --- /dev/null +++ b/scripts/run-scale-ops-gate.ps1 @@ -0,0 +1,488 @@ +param( + [string]$ArtifactRoot = "", + [string]$FixtureRoot = "", + [string]$Phase10ArtifactRoot = "", + [int]$MaxPhase10AgeHours = 72, + [int]$MaxOpsEvidenceAgeHours = 168, + [switch]$FailOnWarn, + [switch]$FailOnExternalBlocked +) + +$ErrorActionPreference = "Stop" + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff" +if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase11-scale-ops-gate" +} +if ([string]::IsNullOrWhiteSpace($FixtureRoot)) { + $FixtureRoot = Join-Path $repoRoot "project-docs\phase11-scale-ops-fixtures" +} +if ([string]::IsNullOrWhiteSpace($Phase10ArtifactRoot)) { + $Phase10ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase10-delivery-acceptance" +} +foreach ($pathName in @("ArtifactRoot", "FixtureRoot", "Phase10ArtifactRoot")) { + $value = Get-Variable -Name $pathName -ValueOnly + if (-not [System.IO.Path]::IsPathRooted($value)) { + Set-Variable -Name $pathName -Value (Join-Path $repoRoot $value) + } +} + +$artifactDir = Join-Path $ArtifactRoot $timestamp +New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null + +$summaryJson = Join-Path $artifactDir "scale-ops-gate-summary.json" +$summaryMd = Join-Path $artifactDir "scale-ops-gate-summary.md" +$gitStatusPath = Join-Path $artifactDir "git-status.txt" +$capacityReportCsv = Join-Path $artifactDir "phase11-capacity-performance-report.csv" +$tenantReportCsv = Join-Path $artifactDir "phase11-tenant-isolation-report.csv" +$slaReportCsv = Join-Path $artifactDir "phase11-sla-slo-report.csv" +$integrationReportCsv = Join-Path $artifactDir "phase11-integration-priority-report.csv" +$riskReportCsv = Join-Path $artifactDir "phase11-scale-risk-report.csv" +$evidenceMatrixCsv = Join-Path $artifactDir "phase11-evidence-matrix.csv" + +$checks = New-Object System.Collections.Generic.List[object] +$evidenceMatrix = New-Object System.Collections.Generic.List[object] + +function Add-Check { + param( + [string]$Key, + [string]$Label, + [ValidateSet("pass", "fail", "warn", "skip", "blocked")] + [string]$Status, + [string]$Detail, + [bool]$Required = $true, + [hashtable]$Meta = @{} + ) + + $checks.Add([ordered]@{ + key = $Key + label = $Label + status = $Status + required = $Required + detail = $Detail + meta = $Meta + }) | Out-Null +} + +function Save-Json { + param([string]$Path, [object]$Value) + $Value | ConvertTo-Json -Depth 18 | Set-Content -Encoding UTF8 -Path $Path +} + +function Read-JsonFile { + param([string]$Path) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return $null + } + try { + return Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json + } catch { + return $null + } +} + +function Get-LatestSummaryArtifact { + param( + [string]$Root, + [string]$SummaryFile + ) + if (-not (Test-Path -LiteralPath $Root -PathType Container)) { + return $null + } + return Get-ChildItem -LiteralPath $Root -Directory | + Sort-Object LastWriteTime -Descending | + Where-Object { Test-Path -LiteralPath (Join-Path $_.FullName $SummaryFile) } | + Select-Object -First 1 +} + +function Test-CsvColumns { + param( + [object[]]$Rows, + [string[]]$Columns + ) + if ($Rows.Count -eq 0) { + return $false + } + $names = @($Rows[0].PSObject.Properties.Name) + foreach ($column in $Columns) { + if ($names -notcontains $column) { + return $false + } + } + return $true +} + +function Test-RequiredFlag { + param([string]$Value) + return @("true", "yes", "1", "required") -contains $Value.ToLowerInvariant() +} + +function Get-FailedRequiredCount { + param([object]$Summary) + if ($null -eq $Summary -or $null -eq $Summary.totals) { + return 0 + } + if ($null -ne $Summary.totals.failed_required) { + return [int]$Summary.totals.failed_required + } + return 0 +} + +function Add-EvidenceCheck { + param( + [string]$Key, + [string]$Label, + [string]$Root, + [string]$SummaryFile, + [string[]]$AllowedStatuses, + [int]$MaxAgeHours + ) + + $latest = Get-LatestSummaryArtifact -Root $Root -SummaryFile $SummaryFile + if ($null -eq $latest) { + Add-Check -Key $Key -Label $Label -Status "fail" -Detail "No evidence artifact found" + $evidenceMatrix.Add([pscustomobject][ordered]@{ + key = $Key + label = $Label + status = "MISSING" + failed_required = "" + age_hours = "" + artifact_dir = "" + summary_file = "" + }) | Out-Null + return $null + } + + $summaryPath = Join-Path $latest.FullName $SummaryFile + $summary = Read-JsonFile -Path $summaryPath + $status = if ($summary -and $summary.status) { [string]$summary.status } else { "UNKNOWN" } + $failedRequired = Get-FailedRequiredCount -Summary $summary + $ageHours = [math]::Round(((Get-Date) - $latest.LastWriteTime).TotalHours, 2) + + $evidenceMatrix.Add([pscustomobject][ordered]@{ + key = $Key + label = $Label + status = $status + failed_required = $failedRequired + age_hours = $ageHours + artifact_dir = $latest.FullName + summary_file = $summaryPath + }) | Out-Null + + if (($AllowedStatuses -contains $status) -and $failedRequired -eq 0) { + Add-Check -Key $Key -Label $Label -Status "pass" -Detail "Status=$status, failed_required=$failedRequired, age=${ageHours}h" -Meta @{ + artifact_dir = $latest.FullName + summary = $summaryPath + } + } else { + Add-Check -Key $Key -Label $Label -Status "fail" -Detail "Status=$status, failed_required=$failedRequired" -Meta @{ + artifact_dir = $latest.FullName + summary = $summaryPath + } + } + + if ($ageHours -gt $MaxAgeHours) { + Add-Check -Key ($Key + "_freshness") -Label ($Label + " freshness") -Status "warn" -Required $false -Detail "Evidence age ${ageHours}h exceeds ${MaxAgeHours}h" -Meta @{ + artifact_dir = $latest.FullName + } + } + + return $summary +} + +if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { + Add-Check -Key "gstack" -Label "gstack preflight" -Status "fail" -Detail "gstack is missing" +} else { + Add-Check -Key "gstack" -Label "gstack preflight" -Status "pass" -Detail "GSTACK_OK" +} + +$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim() +$commit = (git -C $repoRoot rev-parse HEAD).Trim() +$remoteUrl = (git -C $repoRoot remote get-url origin).Trim() +$gitStatus = git -C $repoRoot status --short +($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath + +if ($remoteUrl -match "192\.168\.3\.90") { + Add-Check -Key "origin_remote" -Label "origin remote" -Status "pass" -Detail "origin points to 192.168.3.90" -Meta @{ branch = $branch; commit = $commit; remote = $remoteUrl } +} else { + Add-Check -Key "origin_remote" -Label "origin remote" -Status "fail" -Detail "origin does not point to 192.168.3.90" -Meta @{ branch = $branch; commit = $commit; remote = $remoteUrl } +} + +if ($gitStatus) { + Add-Check -Key "working_tree" -Label "working tree" -Status "warn" -Required $false -Detail "Working tree has uncommitted changes; scale ops gate is advisory" -Meta @{ artifact = $gitStatusPath } +} else { + Add-Check -Key "working_tree" -Label "working tree" -Status "pass" -Detail "Working tree is clean" -Meta @{ artifact = $gitStatusPath } +} + +if (Test-Path -LiteralPath $FixtureRoot -PathType Container) { + Add-Check -Key "fixture_root" -Label "scale ops fixture root" -Status "pass" -Detail "Fixture root exists" -Meta @{ root = $FixtureRoot } +} else { + Add-Check -Key "fixture_root" -Label "scale ops fixture root" -Status "fail" -Detail "Fixture root is missing" -Meta @{ root = $FixtureRoot } +} + +$requiredFixtureFiles = @( + "README.md", + "capacity-model.csv", + "performance-baseline.csv", + "tenant-isolation-checklist.csv", + "organization-onboarding-plan.csv", + "sla-slo-indicators.csv", + "integration-priority-register.csv", + "scale-risk-register.csv", + "monthly-ops-report-template.md", + "incident-postmortem-template.md" +) +foreach ($file in $requiredFixtureFiles) { + $path = Join-Path $FixtureRoot $file + if (Test-Path -LiteralPath $path -PathType Leaf) { + Add-Check -Key ("fixture_" + ($file -replace '[^a-zA-Z0-9]', '_')) -Label "fixture $file" -Status "pass" -Detail "Fixture exists" -Meta @{ path = $path } + } else { + Add-Check -Key ("fixture_" + ($file -replace '[^a-zA-Z0-9]', '_')) -Label "fixture $file" -Status "fail" -Detail "Fixture missing" -Meta @{ path = $path } + } +} + +$capacityRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "capacity-model.csv")) +$performanceRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "performance-baseline.csv")) +$tenantRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "tenant-isolation-checklist.csv")) +$orgRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "organization-onboarding-plan.csv")) +$slaRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "sla-slo-indicators.csv")) +$integrationRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "integration-priority-register.csv")) +$riskRows = @(Import-Csv -LiteralPath (Join-Path $FixtureRoot "scale-risk-register.csv")) + +$capacityColumnsOk = Test-CsvColumns -Rows $capacityRows -Columns @("metric", "baseline_value", "scale_target", "warn_threshold", "critical_threshold", "unit", "expansion_action", "status", "required", "evidence") +$capacityRequiredBad = @($capacityRows | Where-Object { (Test-RequiredFlag $_.required) -and $_.status -ne "READY" }) +$capacityThresholdBad = @($capacityRows | Where-Object { [double]$_.baseline_value -gt [double]$_.warn_threshold -or [double]$_.warn_threshold -gt [double]$_.critical_threshold }) +if ($capacityColumnsOk -and $capacityRows.Count -ge 6 -and $capacityRequiredBad.Count -eq 0 -and $capacityThresholdBad.Count -eq 0) { + Add-Check -Key "capacity_model" -Label "capacity model" -Status "pass" -Detail "Capacity metrics=$($capacityRows.Count)" +} else { + Add-Check -Key "capacity_model" -Label "capacity model" -Status "fail" -Detail "Capacity model incomplete or thresholds invalid" +} + +$performanceColumnsOk = Test-CsvColumns -Rows $performanceRows -Columns @("metric", "baseline_value", "slo_target", "warn_threshold", "critical_threshold", "unit", "measurement_source", "status", "required") +$performanceRequiredBad = @($performanceRows | Where-Object { (Test-RequiredFlag $_.required) -and $_.status -ne "READY" }) +$performanceThresholdBad = @($performanceRows | Where-Object { [double]$_.baseline_value -gt [double]$_.warn_threshold -or [double]$_.warn_threshold -gt [double]$_.critical_threshold }) +if ($performanceColumnsOk -and $performanceRows.Count -ge 6 -and $performanceRequiredBad.Count -eq 0 -and $performanceThresholdBad.Count -eq 0) { + Add-Check -Key "performance_baseline" -Label "performance baseline" -Status "pass" -Detail "Performance metrics=$($performanceRows.Count)" +} else { + Add-Check -Key "performance_baseline" -Label "performance baseline" -Status "fail" -Detail "Performance baseline incomplete or thresholds invalid" +} + +$tenantColumnsOk = Test-CsvColumns -Rows $tenantRows -Columns @("check_id", "object_type", "isolation_key", "expected_control", "verification_method", "status", "required", "evidence") +$requiredObjects = @("user", "contract", "knowledge_document", "template", "audit_event", "file_asset") +$tenantObjects = @($tenantRows | Where-Object { $_.status -eq "PASS" } | Select-Object -ExpandProperty object_type -Unique) +$missingTenantObjects = @($requiredObjects | Where-Object { $tenantObjects -notcontains $_ }) +$tenantRequiredBad = @($tenantRows | Where-Object { (Test-RequiredFlag $_.required) -and $_.status -ne "PASS" }) +if ($tenantColumnsOk -and $missingTenantObjects.Count -eq 0 -and $tenantRequiredBad.Count -eq 0) { + Add-Check -Key "tenant_isolation" -Label "tenant isolation" -Status "pass" -Detail "Tenant isolation checks passed=$($tenantRows.Count)" +} else { + Add-Check -Key "tenant_isolation" -Label "tenant isolation" -Status "fail" -Detail "Tenant isolation missing required object checks" -Meta @{ missing_objects = $missingTenantObjects } +} + +$orgColumnsOk = Test-CsvColumns -Rows $orgRows -Columns @("org_id", "org_name", "departments", "roles", "estimated_users", "onboarding_status", "owner", "evidence") +$readyOrgs = @($orgRows | Where-Object { $_.onboarding_status -eq "READY" }) +$orgUsers = 0 +foreach ($org in $readyOrgs) { + $orgUsers += [int]$org.estimated_users +} +if ($orgColumnsOk -and $readyOrgs.Count -ge 2 -and $orgUsers -ge 100) { + Add-Check -Key "organization_onboarding" -Label "organization onboarding" -Status "pass" -Detail "Ready orgs=$($readyOrgs.Count), planned users=$orgUsers" +} else { + Add-Check -Key "organization_onboarding" -Label "organization onboarding" -Status "fail" -Detail "Organization onboarding plan is incomplete" +} + +$slaColumnsOk = Test-CsvColumns -Rows $slaRows -Columns @("indicator_id", "indicator", "slo_target", "measurement_source", "window", "owner", "status", "required") +$slaRequiredBad = @($slaRows | Where-Object { (Test-RequiredFlag $_.required) -and $_.status -ne "READY" }) +if ($slaColumnsOk -and $slaRows.Count -ge 6 -and $slaRequiredBad.Count -eq 0) { + Add-Check -Key "sla_slo" -Label "SLA/SLO indicators" -Status "pass" -Detail "SLA/SLO indicators=$($slaRows.Count)" +} else { + Add-Check -Key "sla_slo" -Label "SLA/SLO indicators" -Status "fail" -Detail "SLA/SLO indicators incomplete" +} + +$integrationColumnsOk = Test-CsvColumns -Rows $integrationRows -Columns @("integration_id", "integration", "priority", "value", "risk", "dependency", "status", "phase12_candidate") +$p1Integrations = @($integrationRows | Where-Object { $_.priority -eq "P1" }) +$integrationMissingPriority = @($integrationRows | Where-Object { [string]::IsNullOrWhiteSpace($_.priority) -or [string]::IsNullOrWhiteSpace($_.dependency) }) +if ($integrationColumnsOk -and $integrationRows.Count -ge 5 -and $p1Integrations.Count -ge 2 -and $integrationMissingPriority.Count -eq 0) { + Add-Check -Key "integration_priority" -Label "integration priority" -Status "pass" -Detail "Integrations=$($integrationRows.Count), P1=$($p1Integrations.Count)" +} else { + Add-Check -Key "integration_priority" -Label "integration priority" -Status "fail" -Detail "Integration priority register incomplete" +} + +$riskColumnsOk = Test-CsvColumns -Rows $riskRows -Columns @("risk_id", "domain", "severity", "status", "mitigation", "owner", "evidence") +$openCriticalRisks = @($riskRows | Where-Object { ($_.severity -eq "P0" -or $_.severity -eq "P1") -and @("MITIGATED", "ACCEPTED") -notcontains $_.status }) +$risksWithoutMitigation = @($riskRows | Where-Object { [string]::IsNullOrWhiteSpace($_.mitigation) }) +$deferredExternalRisks = @($riskRows | Where-Object { $_.status -like "DEFERRED*" }) +if ($riskColumnsOk -and $riskRows.Count -ge 5 -and $openCriticalRisks.Count -eq 0 -and $risksWithoutMitigation.Count -eq 0) { + Add-Check -Key "scale_risk" -Label "scale risk register" -Status "pass" -Detail "Risks=$($riskRows.Count), open P0/P1=0" +} else { + Add-Check -Key "scale_risk" -Label "scale risk register" -Status "fail" -Detail "Scale risk register has unmanaged critical risks" +} + +$phase10Summary = Add-EvidenceCheck -Key "phase10_delivery_acceptance" -Label "phase10 delivery acceptance" -Root $Phase10ArtifactRoot -SummaryFile "delivery-acceptance-summary.json" -AllowedStatuses @("PASS", "PASS_WITH_WEBHOOK_BLOCKED") -MaxAgeHours $MaxPhase10AgeHours +$evidenceSpecs = @( + @{ key = "capacity_governance"; label = "capacity governance"; root = (Join-Path $repoRoot "project-docs\qa-artifacts\phase7-capacity-governance"); summary = "capacity-governance-summary.json"; statuses = @("PASS", "WARN"); max = $MaxOpsEvidenceAgeHours }, + @{ key = "production_watchdog"; label = "production watchdog"; root = (Join-Path $repoRoot "project-docs\qa-artifacts\phase7-production-watchdog"); summary = "production-watchdog-summary.json"; statuses = @("PASS", "WARN"); max = $MaxOpsEvidenceAgeHours }, + @{ key = "watchdog_task_audit"; label = "watchdog task audit"; root = (Join-Path $repoRoot "project-docs\qa-artifacts\phase7.8-watchdog-task-audit"); summary = "watchdog-task-audit-summary.json"; statuses = @("PASS"); max = $MaxOpsEvidenceAgeHours }, + @{ key = "watchdog_evidence_report"; label = "watchdog evidence report"; root = (Join-Path $repoRoot "project-docs\qa-artifacts\phase7.9-watchdog-evidence-report"); summary = "watchdog-evidence-report-summary.json"; statuses = @("PASS"); max = $MaxOpsEvidenceAgeHours }, + @{ key = "business_quality_gate"; label = "business quality gate"; root = (Join-Path $repoRoot "project-docs\qa-artifacts\phase9-business-quality-gate"); summary = "business-quality-gate-summary.json"; statuses = @("PASS", "PASS_WITH_WEBHOOK_BLOCKED"); max = $MaxOpsEvidenceAgeHours } +) +foreach ($spec in $evidenceSpecs) { + Add-EvidenceCheck -Key $spec.key -Label $spec.label -Root $spec.root -SummaryFile $spec.summary -AllowedStatuses $spec.statuses -MaxAgeHours $spec.max | Out-Null +} + +$capacityCombined = @() +foreach ($row in $capacityRows) { + $capacityCombined += [pscustomobject][ordered]@{ + category = "capacity" + metric = $row.metric + baseline_value = $row.baseline_value + target = $row.scale_target + warn_threshold = $row.warn_threshold + critical_threshold = $row.critical_threshold + unit = $row.unit + status = $row.status + evidence = $row.evidence + } +} +foreach ($row in $performanceRows) { + $capacityCombined += [pscustomobject][ordered]@{ + category = "performance" + metric = $row.metric + baseline_value = $row.baseline_value + target = $row.slo_target + warn_threshold = $row.warn_threshold + critical_threshold = $row.critical_threshold + unit = $row.unit + status = $row.status + evidence = $row.measurement_source + } +} +$capacityCombined | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $capacityReportCsv +$tenantRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $tenantReportCsv +$slaRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $slaReportCsv +$integrationRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $integrationReportCsv +$riskRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $riskReportCsv +$evidenceMatrix | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $evidenceMatrixCsv + +$secret = [string]::Concat([char]54, [char]53, [char]52, [char]51, [char]50, [char]49) +$secretHits = @() +$scanFiles = @(Get-ChildItem -LiteralPath $FixtureRoot -File -Recurse) +$scanFiles += @(Get-Item -LiteralPath (Join-Path $repoRoot "scripts\run-scale-ops-gate.ps1")) +foreach ($file in $scanFiles) { + $text = Get-Content -LiteralPath $file.FullName -Raw -Encoding UTF8 -ErrorAction SilentlyContinue + if ($text -and $text.Contains($secret)) { + $secretHits += $file.FullName + } +} +if ($secretHits.Count -eq 0) { + Add-Check -Key "secret_scan" -Label "secret scan" -Status "pass" -Detail "No administrator password literal found in scale ops package or gate script" +} else { + Add-Check -Key "secret_scan" -Label "secret scan" -Status "fail" -Detail "Secret literal detected" -Meta @{ hits = $secretHits } +} + +$externalBlocked = $false +if (($phase10Summary -and $phase10Summary.status -eq "PASS_WITH_WEBHOOK_BLOCKED") -or $deferredExternalRisks.Count -gt 0 -or @($integrationRows | Where-Object { $_.status -like "BLOCKED*" }).Count -gt 0) { + $externalBlocked = $true + Add-Check -Key "external_webhook" -Label "external alert webhook" -Status "blocked" -Required $false -Detail "Real external alert Webhook remains unavailable; inherited as external blocked item" +} + +$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }) +$warningsList = @($checks | Where-Object { $_.status -eq "warn" }) +$blockedList = @($checks | Where-Object { $_.status -eq "blocked" }) +$overallStatus = if ($failedRequired.Count -gt 0) { + "FAIL" +} elseif ($blockedList.Count -gt 0 -or $externalBlocked) { + "PASS_WITH_WEBHOOK_BLOCKED" +} elseif ($warningsList.Count -gt 0) { + "WARN" +} else { + "PASS" +} +$alertSeverity = if ($overallStatus -eq "FAIL") { "critical" } elseif ($overallStatus -eq "WARN") { "warning" } elseif ($overallStatus -eq "PASS_WITH_WEBHOOK_BLOCKED") { "blocked" } else { "ok" } + +$summary = [ordered]@{ + name = "phase11-scale-ops-gate" + status = $overallStatus + alert_severity = $alertSeverity + generated_at = (Get-Date).ToUniversalTime().ToString("o") + artifact_dir = $artifactDir + branch = $branch + commit = $commit + remote = $remoteUrl + fixture_root = $FixtureRoot + checks = $checks + metrics = [ordered]@{ + capacity_metrics = $capacityRows.Count + performance_metrics = $performanceRows.Count + tenant_isolation_checks = $tenantRows.Count + ready_organizations = $readyOrgs.Count + planned_users = $orgUsers + sla_slo_indicators = $slaRows.Count + integrations = $integrationRows.Count + risks = $riskRows.Count + } + artifacts = [ordered]@{ + capacity_performance_report = $capacityReportCsv + tenant_isolation_report = $tenantReportCsv + sla_slo_report = $slaReportCsv + integration_priority_report = $integrationReportCsv + scale_risk_report = $riskReportCsv + evidence_matrix = $evidenceMatrixCsv + } + totals = [ordered]@{ + total = $checks.Count + failed_required = $failedRequired.Count + warnings = $warningsList.Count + blocked = $blockedList.Count + } +} +Save-Json -Path $summaryJson -Value $summary + +$markdown = New-Object System.Collections.Generic.List[string] +$markdown.Add("# Phase 11 Scale Operations Gate") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("- Status: $overallStatus") | Out-Null +$markdown.Add("- Alert severity: $alertSeverity") | Out-Null +$markdown.Add("- Generated at: $($summary.generated_at)") | Out-Null +$markdown.Add("- Branch: $branch") | Out-Null +$markdown.Add("- Commit: $commit") | Out-Null +$markdown.Add("- Artifact dir: $artifactDir") | Out-Null +$markdown.Add("- Fixture root: $FixtureRoot") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("## Metrics") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("| Metric | Count |") | Out-Null +$markdown.Add("|---|---:|") | Out-Null +$markdown.Add("| Capacity metrics | $($capacityRows.Count) |") | Out-Null +$markdown.Add("| Performance metrics | $($performanceRows.Count) |") | Out-Null +$markdown.Add("| Tenant isolation checks | $($tenantRows.Count) |") | Out-Null +$markdown.Add("| Ready organizations | $($readyOrgs.Count) |") | Out-Null +$markdown.Add("| Planned users | $orgUsers |") | Out-Null +$markdown.Add("| SLA/SLO indicators | $($slaRows.Count) |") | Out-Null +$markdown.Add("| Integrations | $($integrationRows.Count) |") | Out-Null +$markdown.Add("| Risks | $($riskRows.Count) |") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("## Checks") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add("| Check | Required | Status | Detail |") | Out-Null +$markdown.Add("|---|---:|---|---|") | Out-Null +foreach ($check in $checks) { + $markdown.Add("| $($check.label) | $($check.required) | $($check.status.ToUpperInvariant()) | $($check.detail) |") | Out-Null +} +$markdown.Add("") | Out-Null +$markdown.Add("Local artifacts:") | Out-Null +$markdown.Add("") | Out-Null +$markdown.Add('- `scale-ops-gate-summary.json`') | Out-Null +$markdown.Add('- `scale-ops-gate-summary.md`') | Out-Null +$markdown.Add('- `phase11-capacity-performance-report.csv`') | Out-Null +$markdown.Add('- `phase11-tenant-isolation-report.csv`') | Out-Null +$markdown.Add('- `phase11-sla-slo-report.csv`') | Out-Null +$markdown.Add('- `phase11-integration-priority-report.csv`') | Out-Null +$markdown.Add('- `phase11-scale-risk-report.csv`') | Out-Null +$markdown.Add('- `phase11-evidence-matrix.csv`') | Out-Null +$markdown | Set-Content -Encoding UTF8 -Path $summaryMd + +Write-Host "PHASE11_SCALE_OPS_GATE_$overallStatus $artifactDir" +if ($overallStatus -eq "FAIL" -or ($FailOnWarn -and $warningsList.Count -gt 0) -or ($FailOnExternalBlocked -and $blockedList.Count -gt 0)) { + throw "Phase 11 scale ops gate $overallStatus. See $summaryMd" +}