docs: add phase 13 golden eval gate
This commit is contained in:
Codex
@@ -340,7 +340,8 @@ $commonDocFiles = @(
|
||||
"project-docs\phase13-legal-knowledge-template-library\README.md",
|
||||
"project-docs\phase13-legal-knowledge-template-library\governance-rules.md",
|
||||
"project-docs\phase13-legal-knowledge-template-library\field-dictionary.md",
|
||||
"scripts\import-phase13-library.ps1"
|
||||
"scripts\import-phase13-library.ps1",
|
||||
"scripts\run-phase13-golden-eval.ps1"
|
||||
)
|
||||
foreach ($docPath in $commonDocFiles) {
|
||||
$absoluteDocPath = Join-Path $repoRoot $docPath
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
param(
|
||||
[string]$ArtifactRoot = "",
|
||||
[string]$LibraryRoot = "",
|
||||
[int]$MinReviewSamples = 8,
|
||||
[int]$MinGenerationSamples = 8,
|
||||
[int]$MinKbQaSamples = 10,
|
||||
[int]$MinPassRatePercent = 100,
|
||||
[switch]$FailOnLegalReviewRequired
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff"
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
||||
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase13-golden-eval"
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($LibraryRoot)) {
|
||||
$LibraryRoot = Join-Path $repoRoot "project-docs\phase13-legal-knowledge-template-library"
|
||||
}
|
||||
foreach ($pathName in @("ArtifactRoot", "LibraryRoot")) {
|
||||
$value = Get-Variable -Name $pathName -ValueOnly
|
||||
if (-not [System.IO.Path]::IsPathRooted($value)) {
|
||||
Set-Variable -Name $pathName -Value (Join-Path $repoRoot $value)
|
||||
}
|
||||
}
|
||||
|
||||
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
||||
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
||||
|
||||
$summaryJson = Join-Path $artifactDir "phase13-golden-eval-summary.json"
|
||||
$summaryMd = Join-Path $artifactDir "phase13-golden-eval-summary.md"
|
||||
$reviewReportCsv = Join-Path $artifactDir "review-sample-eval-report.csv"
|
||||
$generationReportCsv = Join-Path $artifactDir "generation-sample-eval-report.csv"
|
||||
$kbQaReportCsv = Join-Path $artifactDir "kb-qa-sample-eval-report.csv"
|
||||
$gitStatusPath = Join-Path $artifactDir "git-status.txt"
|
||||
|
||||
$checks = New-Object System.Collections.Generic.List[object]
|
||||
|
||||
function Add-Check {
|
||||
param(
|
||||
[string]$Key,
|
||||
[string]$Label,
|
||||
[ValidateSet("pass", "fail", "warn", "skip", "blocked")]
|
||||
[string]$Status,
|
||||
[string]$Detail,
|
||||
[bool]$Required = $true,
|
||||
[hashtable]$Meta = @{}
|
||||
)
|
||||
|
||||
$checks.Add([ordered]@{
|
||||
key = $Key
|
||||
label = $Label
|
||||
status = $Status
|
||||
required = $Required
|
||||
detail = $Detail
|
||||
meta = $Meta
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
function Save-Json {
|
||||
param([string]$Path, [object]$Value)
|
||||
$Value | ConvertTo-Json -Depth 30 | Set-Content -Encoding UTF8 -Path $Path
|
||||
}
|
||||
|
||||
function Import-CsvRows {
|
||||
param([string]$Path)
|
||||
if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
|
||||
return @()
|
||||
}
|
||||
return @(Import-Csv -LiteralPath $Path)
|
||||
}
|
||||
|
||||
function Split-SemicolonValues {
|
||||
param([string]$Value)
|
||||
if ([string]::IsNullOrWhiteSpace($Value)) {
|
||||
return @()
|
||||
}
|
||||
return @($Value -split ";" | ForEach-Object { $_.Trim() } | Where-Object { $_ })
|
||||
}
|
||||
|
||||
function Test-RequiredFlag {
|
||||
param([string]$Value)
|
||||
if ([string]::IsNullOrWhiteSpace($Value)) {
|
||||
return $false
|
||||
}
|
||||
return @("true", "yes", "1", "required") -contains $Value.ToLowerInvariant()
|
||||
}
|
||||
|
||||
function Resolve-LibraryPath {
|
||||
param([string]$RelativePath)
|
||||
return Join-Path $LibraryRoot ($RelativePath -replace "/", "\")
|
||||
}
|
||||
|
||||
function Read-LibraryText {
|
||||
param([string]$RelativePath)
|
||||
if ([string]::IsNullOrWhiteSpace($RelativePath)) {
|
||||
return ""
|
||||
}
|
||||
$path = Resolve-LibraryPath -RelativePath $RelativePath
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) {
|
||||
return ""
|
||||
}
|
||||
return Get-Content -LiteralPath $path -Raw -Encoding UTF8
|
||||
}
|
||||
|
||||
function Test-TextContainsAll {
|
||||
param([string]$Text, [string[]]$Needles)
|
||||
$missing = New-Object System.Collections.Generic.List[string]
|
||||
foreach ($needle in $Needles) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($needle) -and -not $Text.Contains($needle)) {
|
||||
$missing.Add($needle) | Out-Null
|
||||
}
|
||||
}
|
||||
return $missing.ToArray()
|
||||
}
|
||||
|
||||
function Convert-GateStatusText {
|
||||
param([string]$Status)
|
||||
switch ($Status) {
|
||||
"PASS" { return "通过" }
|
||||
"FAIL" { return "未通过" }
|
||||
"PASS_WITH_LEGAL_REVIEW_REQUIRED" { return "通过但仍需法务复核" }
|
||||
default { return $Status }
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-CheckStatusText {
|
||||
param([string]$Status)
|
||||
switch ($Status) {
|
||||
"pass" { return "通过" }
|
||||
"fail" { return "失败" }
|
||||
"warn" { return "警告" }
|
||||
"skip" { return "跳过" }
|
||||
"blocked" { return "阻塞" }
|
||||
default { return $Status }
|
||||
}
|
||||
}
|
||||
|
||||
function Get-PassRatePercent {
|
||||
param([int]$Passed, [int]$Total)
|
||||
if ($Total -le 0) {
|
||||
return 0
|
||||
}
|
||||
return [math]::Round(($Passed * 100.0) / $Total, 2)
|
||||
}
|
||||
|
||||
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
||||
Add-Check -Key "gstack" -Label "gstack 前置检查" -Status "fail" -Detail "gstack 缺失"
|
||||
} else {
|
||||
Add-Check -Key "gstack" -Label "gstack 前置检查" -Status "pass" -Detail "GSTACK_OK"
|
||||
}
|
||||
|
||||
$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim()
|
||||
$commit = (git -C $repoRoot rev-parse HEAD).Trim()
|
||||
$remoteUrl = (git -C $repoRoot remote get-url origin).Trim()
|
||||
$gitStatus = git -C $repoRoot status --short
|
||||
($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath
|
||||
|
||||
if ($remoteUrl -match "192\.168\.3\.90") {
|
||||
Add-Check -Key "origin_remote" -Label "origin 远端仓库" -Status "pass" -Detail "origin 指向 192.168.3.90"
|
||||
} else {
|
||||
Add-Check -Key "origin_remote" -Label "origin 远端仓库" -Status "fail" -Detail "origin 未指向 192.168.3.90" -Meta @{ remote = $remoteUrl }
|
||||
}
|
||||
|
||||
if (Test-Path -LiteralPath $LibraryRoot -PathType Container) {
|
||||
Add-Check -Key "library_root" -Label "阶段 13 资料包根目录" -Status "pass" -Detail "资料包根目录存在"
|
||||
} else {
|
||||
Add-Check -Key "library_root" -Label "阶段 13 资料包根目录" -Status "fail" -Detail "资料包根目录缺失" -Meta @{ root = $LibraryRoot }
|
||||
}
|
||||
|
||||
$inventoryRoot = Join-Path $LibraryRoot "inventories"
|
||||
$legalSources = Import-CsvRows -Path (Join-Path $inventoryRoot "legal-source-inventory.csv")
|
||||
$knowledgeCards = Import-CsvRows -Path (Join-Path $inventoryRoot "legal-knowledge-card-inventory.csv")
|
||||
$reviewRules = Import-CsvRows -Path (Join-Path $inventoryRoot "review-risk-rule-inventory.csv")
|
||||
$clauses = Import-CsvRows -Path (Join-Path $inventoryRoot "clause-library-inventory.csv")
|
||||
$templates = Import-CsvRows -Path (Join-Path $inventoryRoot "contract-template-inventory.csv")
|
||||
$reviewSamples = Import-CsvRows -Path (Join-Path $inventoryRoot "review-golden-sample-inventory.csv")
|
||||
$generationSamples = Import-CsvRows -Path (Join-Path $inventoryRoot "generation-golden-sample-inventory.csv")
|
||||
$kbQaSamples = Import-CsvRows -Path (Join-Path $inventoryRoot "kb-qa-golden-sample-inventory.csv")
|
||||
|
||||
$sourceMap = @{}
|
||||
foreach ($source in $legalSources) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($source.source_id)) {
|
||||
$sourceMap[$source.source_id] = $true
|
||||
}
|
||||
}
|
||||
$ruleMap = @{}
|
||||
foreach ($rule in $reviewRules) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($rule.rule_id)) {
|
||||
$ruleMap[$rule.rule_id] = $rule
|
||||
}
|
||||
}
|
||||
$clauseMap = @{}
|
||||
foreach ($clause in $clauses) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($clause.clause_id)) {
|
||||
$clauseMap[$clause.clause_id] = $true
|
||||
}
|
||||
}
|
||||
$templateMap = @{}
|
||||
foreach ($template in $templates) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($template.template_id)) {
|
||||
$templateMap[$template.template_id] = $true
|
||||
}
|
||||
}
|
||||
|
||||
if ($reviewSamples.Count -ge $MinReviewSamples) {
|
||||
Add-Check -Key "review_sample_minimum" -Label "审核样本数量下限" -Status "pass" -Detail "审核样本数量=$($reviewSamples.Count)，最低要求=$MinReviewSamples"
|
||||
} else {
|
||||
Add-Check -Key "review_sample_minimum" -Label "审核样本数量下限" -Status "fail" -Detail "审核样本数量=$($reviewSamples.Count)，最低要求=$MinReviewSamples"
|
||||
}
|
||||
if ($generationSamples.Count -ge $MinGenerationSamples) {
|
||||
Add-Check -Key "generation_sample_minimum" -Label "生成样本数量下限" -Status "pass" -Detail "生成样本数量=$($generationSamples.Count)，最低要求=$MinGenerationSamples"
|
||||
} else {
|
||||
Add-Check -Key "generation_sample_minimum" -Label "生成样本数量下限" -Status "fail" -Detail "生成样本数量=$($generationSamples.Count)，最低要求=$MinGenerationSamples"
|
||||
}
|
||||
if ($kbQaSamples.Count -ge $MinKbQaSamples) {
|
||||
Add-Check -Key "kb_qa_sample_minimum" -Label "问答样本数量下限" -Status "pass" -Detail "问答样本数量=$($kbQaSamples.Count)，最低要求=$MinKbQaSamples"
|
||||
} else {
|
||||
Add-Check -Key "kb_qa_sample_minimum" -Label "问答样本数量下限" -Status "fail" -Detail "问答样本数量=$($kbQaSamples.Count)，最低要求=$MinKbQaSamples"
|
||||
}
|
||||
|
||||
$reviewEval = New-Object System.Collections.Generic.List[object]
|
||||
foreach ($sample in $reviewSamples) {
|
||||
$issues = New-Object System.Collections.Generic.List[string]
|
||||
$inputText = Read-LibraryText -RelativePath $sample.input_sample_path
|
||||
$expectedText = Read-LibraryText -RelativePath $sample.expected_findings_path
|
||||
if ([string]::IsNullOrWhiteSpace($inputText)) {
|
||||
$issues.Add("input_sample_missing_or_empty") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedText)) {
|
||||
$issues.Add("expected_findings_missing_or_empty") | Out-Null
|
||||
}
|
||||
|
||||
$boundRules = @($reviewRules | Where-Object { (Split-SemicolonValues -Value $_.review_sample_ids) -contains $sample.sample_id })
|
||||
if ($boundRules.Count -eq 0) {
|
||||
$issues.Add("no_bound_review_rule") | Out-Null
|
||||
}
|
||||
|
||||
foreach ($rule in $boundRules) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($rule.trigger_condition) -and -not $inputText.Contains($rule.trigger_condition)) {
|
||||
$issues.Add("trigger_missing:$($rule.rule_id):$($rule.trigger_condition)") | Out-Null
|
||||
}
|
||||
if (-not $expectedText.Contains($rule.rule_id)) {
|
||||
$issues.Add("expected_rule_missing:$($rule.rule_id)") | Out-Null
|
||||
}
|
||||
if ($rule.severity -ne $sample.expected_severity) {
|
||||
$issues.Add("severity_mismatch:$($rule.rule_id)") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$missingExpectedSources = Test-TextContainsAll -Text $expectedText -Needles (Split-SemicolonValues -Value $sample.expected_source_citations)
|
||||
foreach ($sourceId in $missingExpectedSources) {
|
||||
$issues.Add("expected_source_missing_in_file:$sourceId") | Out-Null
|
||||
}
|
||||
foreach ($sourceId in (Split-SemicolonValues -Value $sample.expected_source_citations)) {
|
||||
if (-not $sourceMap.ContainsKey($sourceId)) {
|
||||
$issues.Add("unknown_source:$sourceId") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$reviewEval.Add([pscustomobject][ordered]@{
|
||||
sample_id = $sample.sample_id
|
||||
asset_type = "review_sample"
|
||||
status = if ($issues.Count -eq 0) { "pass" } else { "fail" }
|
||||
expected_severity = $sample.expected_severity
|
||||
bound_rule_ids = (@($boundRules | Select-Object -ExpandProperty rule_id) -join ";")
|
||||
expected_source_citations = $sample.expected_source_citations
|
||||
issue_count = $issues.Count
|
||||
issues = ($issues -join ";")
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
$generationEval = New-Object System.Collections.Generic.List[object]
|
||||
foreach ($sample in $generationSamples) {
|
||||
$issues = New-Object System.Collections.Generic.List[string]
|
||||
$variablePath = Resolve-LibraryPath -RelativePath $sample.variable_payload_path
|
||||
$expectedText = Read-LibraryText -RelativePath $sample.expected_document_path
|
||||
$variables = $null
|
||||
if (-not (Test-Path -LiteralPath $variablePath -PathType Leaf)) {
|
||||
$issues.Add("variable_payload_missing") | Out-Null
|
||||
} else {
|
||||
try {
|
||||
$variables = Get-Content -LiteralPath $variablePath -Raw -Encoding UTF8 | ConvertFrom-Json
|
||||
} catch {
|
||||
$issues.Add("variable_payload_invalid_json") | Out-Null
|
||||
}
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($expectedText)) {
|
||||
$issues.Add("expected_document_missing_or_empty") | Out-Null
|
||||
}
|
||||
|
||||
if (-not ($clauseMap.ContainsKey($sample.template_id) -or $templateMap.ContainsKey($sample.template_id))) {
|
||||
$issues.Add("unknown_generation_asset:$($sample.template_id)") | Out-Null
|
||||
}
|
||||
if ($null -ne $variables -and $variables.asset_under_test -ne $sample.template_id) {
|
||||
$issues.Add("variable_asset_mismatch:$($variables.asset_under_test)") | Out-Null
|
||||
}
|
||||
if (-not $expectedText.Contains($sample.template_id)) {
|
||||
$issues.Add("expected_asset_missing:$($sample.template_id)") | Out-Null
|
||||
}
|
||||
foreach ($ruleId in (Split-SemicolonValues -Value $sample.expected_review_rule_ids)) {
|
||||
if (-not $ruleMap.ContainsKey($ruleId)) {
|
||||
$issues.Add("unknown_rule:$ruleId") | Out-Null
|
||||
}
|
||||
if (-not $expectedText.Contains($ruleId)) {
|
||||
$issues.Add("expected_rule_missing:$ruleId") | Out-Null
|
||||
}
|
||||
}
|
||||
foreach ($sourceId in (Split-SemicolonValues -Value $sample.expected_source_citations)) {
|
||||
if (-not $sourceMap.ContainsKey($sourceId)) {
|
||||
$issues.Add("unknown_source:$sourceId") | Out-Null
|
||||
}
|
||||
if (-not $expectedText.Contains($sourceId)) {
|
||||
$issues.Add("expected_source_missing:$sourceId") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$generationEval.Add([pscustomobject][ordered]@{
|
||||
sample_id = $sample.sample_id
|
||||
asset_type = "generation_sample"
|
||||
status = if ($issues.Count -eq 0) { "pass" } else { "fail" }
|
||||
asset_under_test = $sample.template_id
|
||||
expected_review_rule_ids = $sample.expected_review_rule_ids
|
||||
expected_source_citations = $sample.expected_source_citations
|
||||
issue_count = $issues.Count
|
||||
issues = ($issues -join ";")
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
$kbQaEval = New-Object System.Collections.Generic.List[object]
|
||||
foreach ($sample in $kbQaSamples) {
|
||||
$issues = New-Object System.Collections.Generic.List[string]
|
||||
$boundCards = @($knowledgeCards | Where-Object { $sample.notes -match [regex]::Escape($_.knowledge_card_id) })
|
||||
if ($boundCards.Count -eq 0) {
|
||||
$issues.Add("no_bound_knowledge_card") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($sample.question) -or $sample.question.Length -lt 8) {
|
||||
$issues.Add("question_too_short") | Out-Null
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($sample.expected_answer_boundary) -or $sample.expected_answer_boundary.Length -lt 12) {
|
||||
$issues.Add("answer_boundary_too_short") | Out-Null
|
||||
}
|
||||
if ((Test-RequiredFlag -Value $sample.expected_citation_required) -and [string]::IsNullOrWhiteSpace($sample.expected_source_ids)) {
|
||||
$issues.Add("citation_required_without_sources") | Out-Null
|
||||
}
|
||||
if ((Test-RequiredFlag -Value $sample.expected_refusal_required) -and $sample.expected_answer_boundary -notmatch "不能|不得|不确定|人工复核|专项复核|必须") {
|
||||
$issues.Add("refusal_boundary_too_weak") | Out-Null
|
||||
}
|
||||
foreach ($sourceId in (Split-SemicolonValues -Value $sample.expected_source_ids)) {
|
||||
if (-not $sourceMap.ContainsKey($sourceId)) {
|
||||
$issues.Add("unknown_source:$sourceId") | Out-Null
|
||||
}
|
||||
}
|
||||
|
||||
$kbQaEval.Add([pscustomobject][ordered]@{
|
||||
sample_id = $sample.sample_id
|
||||
asset_type = "kb_qa_sample"
|
||||
status = if ($issues.Count -eq 0) { "pass" } else { "fail" }
|
||||
bound_knowledge_card_ids = (@($boundCards | Select-Object -ExpandProperty knowledge_card_id) -join ";")
|
||||
expected_source_ids = $sample.expected_source_ids
|
||||
expected_refusal_required = $sample.expected_refusal_required
|
||||
issue_count = $issues.Count
|
||||
issues = ($issues -join ";")
|
||||
}) | Out-Null
|
||||
}
|
||||
|
||||
$reviewPass = @($reviewEval | Where-Object { $_.status -eq "pass" }).Count
|
||||
$generationPass = @($generationEval | Where-Object { $_.status -eq "pass" }).Count
|
||||
$kbQaPass = @($kbQaEval | Where-Object { $_.status -eq "pass" }).Count
|
||||
$totalSamples = $reviewEval.Count + $generationEval.Count + $kbQaEval.Count
|
||||
$totalPass = $reviewPass + $generationPass + $kbQaPass
|
||||
$passRate = Get-PassRatePercent -Passed $totalPass -Total $totalSamples
|
||||
|
||||
if ($reviewEval.Count -gt 0 -and $reviewPass -eq $reviewEval.Count) {
|
||||
Add-Check -Key "review_eval" -Label "审核黄金样本评测" -Status "pass" -Detail "审核样本通过 $reviewPass/$($reviewEval.Count)"
|
||||
} else {
|
||||
Add-Check -Key "review_eval" -Label "审核黄金样本评测" -Status "fail" -Detail "审核样本通过 $reviewPass/$($reviewEval.Count)"
|
||||
}
|
||||
if ($generationEval.Count -gt 0 -and $generationPass -eq $generationEval.Count) {
|
||||
Add-Check -Key "generation_eval" -Label "生成黄金样本评测" -Status "pass" -Detail "生成样本通过 $generationPass/$($generationEval.Count)"
|
||||
} else {
|
||||
Add-Check -Key "generation_eval" -Label "生成黄金样本评测" -Status "fail" -Detail "生成样本通过 $generationPass/$($generationEval.Count)"
|
||||
}
|
||||
if ($kbQaEval.Count -gt 0 -and $kbQaPass -eq $kbQaEval.Count) {
|
||||
Add-Check -Key "kb_qa_eval" -Label "知识库问答样本评测" -Status "pass" -Detail "问答样本通过 $kbQaPass/$($kbQaEval.Count)"
|
||||
} else {
|
||||
Add-Check -Key "kb_qa_eval" -Label "知识库问答样本评测" -Status "fail" -Detail "问答样本通过 $kbQaPass/$($kbQaEval.Count)"
|
||||
}
|
||||
if ($passRate -ge $MinPassRatePercent) {
|
||||
Add-Check -Key "sample_pass_rate" -Label "样本通过率门槛" -Status "pass" -Detail ("通过率={0}%，最低要求={1}%" -f $passRate, $MinPassRatePercent)
|
||||
} else {
|
||||
Add-Check -Key "sample_pass_rate" -Label "样本通过率门槛" -Status "fail" -Detail ("通过率={0}%，最低要求={1}%" -f $passRate, $MinPassRatePercent)
|
||||
}
|
||||
|
||||
$reviewRequiredRows = @(
|
||||
@($reviewSamples | Where-Object { $_.status -eq "LEGAL_REVIEW" -or (Test-RequiredFlag -Value $_.review_required) })
|
||||
@($generationSamples | Where-Object { $_.status -eq "LEGAL_REVIEW" -or (Test-RequiredFlag -Value $_.review_required) })
|
||||
@($kbQaSamples | Where-Object { $_.status -eq "LEGAL_REVIEW" -or (Test-RequiredFlag -Value $_.review_required) })
|
||||
)
|
||||
$legalReviewRequired = $reviewRequiredRows.Count
|
||||
if ($legalReviewRequired -gt 0) {
|
||||
Add-Check -Key "sample_legal_review_required" -Label "样本法务复核要求" -Status "warn" -Required $false -Detail "仍有 $legalReviewRequired 个样本需要法务复核"
|
||||
} else {
|
||||
Add-Check -Key "sample_legal_review_required" -Label "样本法务复核要求" -Status "pass" -Detail "样本已完成法务复核"
|
||||
}
|
||||
|
||||
$reviewEval | Export-Csv -LiteralPath $reviewReportCsv -NoTypeInformation -Encoding UTF8
|
||||
$generationEval | Export-Csv -LiteralPath $generationReportCsv -NoTypeInformation -Encoding UTF8
|
||||
$kbQaEval | Export-Csv -LiteralPath $kbQaReportCsv -NoTypeInformation -Encoding UTF8
|
||||
|
||||
$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }).Count
|
||||
$warnings = @($checks | Where-Object { $_.status -eq "warn" }).Count
|
||||
$overallStatus = if ($failedRequired -gt 0) {
|
||||
"FAIL"
|
||||
} elseif ($legalReviewRequired -gt 0) {
|
||||
"PASS_WITH_LEGAL_REVIEW_REQUIRED"
|
||||
} else {
|
||||
"PASS"
|
||||
}
|
||||
|
||||
$summary = [ordered]@{
|
||||
name = "phase13-golden-eval"
|
||||
status = $overallStatus
|
||||
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
||||
artifact_dir = $artifactDir
|
||||
branch = $branch
|
||||
commit = $commit
|
||||
remote = $remoteUrl
|
||||
library_root = $LibraryRoot
|
||||
checks = $checks.ToArray()
|
||||
metrics = [ordered]@{
|
||||
review_samples = $reviewEval.Count
|
||||
review_samples_passed = $reviewPass
|
||||
generation_samples = $generationEval.Count
|
||||
generation_samples_passed = $generationPass
|
||||
kb_qa_samples = $kbQaEval.Count
|
||||
kb_qa_samples_passed = $kbQaPass
|
||||
total_samples = $totalSamples
|
||||
total_passed = $totalPass
|
||||
pass_rate_percent = $passRate
|
||||
legal_review_required = $legalReviewRequired
|
||||
}
|
||||
artifacts = [ordered]@{
|
||||
review_report = $reviewReportCsv
|
||||
generation_report = $generationReportCsv
|
||||
kb_qa_report = $kbQaReportCsv
|
||||
git_status = $gitStatusPath
|
||||
}
|
||||
totals = [ordered]@{
|
||||
total_checks = $checks.Count
|
||||
failed_required = $failedRequired
|
||||
warnings = $warnings
|
||||
}
|
||||
}
|
||||
Save-Json -Path $summaryJson -Value $summary
|
||||
|
||||
$statusText = Convert-GateStatusText -Status $overallStatus
|
||||
$md = New-Object System.Collections.Generic.List[string]
|
||||
$md.Add("# 阶段 13 黄金样本评测报告") | Out-Null
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add("- 评测状态：$statusText（$overallStatus）") | Out-Null
|
||||
$md.Add("- 生成时间：$($summary.generated_at)") | Out-Null
|
||||
$md.Add("- 分支：$branch") | Out-Null
|
||||
$md.Add("- 提交：$commit") | Out-Null
|
||||
$md.Add("- 产物目录：$artifactDir") | Out-Null
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add("## 指标") | Out-Null
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add("| 指标 | 数量 |") | Out-Null
|
||||
$md.Add("|---|---:|") | Out-Null
|
||||
$md.Add("| 审核样本 | $($reviewEval.Count) |") | Out-Null
|
||||
$md.Add("| 审核样本通过 | $reviewPass |") | Out-Null
|
||||
$md.Add("| 生成样本 | $($generationEval.Count) |") | Out-Null
|
||||
$md.Add("| 生成样本通过 | $generationPass |") | Out-Null
|
||||
$md.Add("| 知识库问答样本 | $($kbQaEval.Count) |") | Out-Null
|
||||
$md.Add("| 知识库问答样本通过 | $kbQaPass |") | Out-Null
|
||||
$md.Add("| 总样本 | $totalSamples |") | Out-Null
|
||||
$md.Add("| 总通过 | $totalPass |") | Out-Null
|
||||
$md.Add(("| 通过率 | {0}% |" -f $passRate)) | Out-Null
|
||||
$md.Add("| 仍需法务复核的样本 | $legalReviewRequired |") | Out-Null
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add("## 检查项") | Out-Null
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add("| 检查项 | 必需 | 状态 | 说明 |") | Out-Null
|
||||
$md.Add("|---|---:|---|---|") | Out-Null
|
||||
foreach ($check in $checks) {
|
||||
$requiredText = if ($check.required) { "是" } else { "否" }
|
||||
$detail = ([string]$check.detail).Replace("|", "/")
|
||||
$checkStatusText = Convert-CheckStatusText -Status $check.status
|
||||
$md.Add("| $($check.label) | $requiredText | $checkStatusText（$($check.status.ToUpperInvariant())） | $detail |") | Out-Null
|
||||
}
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add("本地产物：") | Out-Null
|
||||
$md.Add("") | Out-Null
|
||||
$md.Add('- `phase13-golden-eval-summary.json`') | Out-Null
|
||||
$md.Add('- `phase13-golden-eval-summary.md`') | Out-Null
|
||||
$md.Add('- `review-sample-eval-report.csv`') | Out-Null
|
||||
$md.Add('- `generation-sample-eval-report.csv`') | Out-Null
|
||||
$md.Add('- `kb-qa-sample-eval-report.csv`') | Out-Null
|
||||
$md | Set-Content -Encoding UTF8 -Path $summaryMd
|
||||
|
||||
if ($overallStatus -eq "FAIL") {
|
||||
Write-Host "PHASE13_GOLDEN_EVAL_FAIL $artifactDir"
|
||||
throw "阶段 13 黄金样本评测未通过。查看：$summaryMd"
|
||||
}
|
||||
if ($overallStatus -eq "PASS_WITH_LEGAL_REVIEW_REQUIRED" -and $FailOnLegalReviewRequired) {
|
||||
Write-Host "PHASE13_GOLDEN_EVAL_PASS_WITH_LEGAL_REVIEW_REQUIRED $artifactDir"
|
||||
throw "阶段 13 黄金样本评测仍需法务复核。查看：$summaryMd"
|
||||
}
|
||||
|
||||
Write-Host "PHASE13_GOLDEN_EVAL_$overallStatus $artifactDir"
|
||||
Reference in New Issue
Block a user