diff --git a/project-docs/phase13-legal-knowledge-template-library/README.md b/project-docs/phase13-legal-knowledge-template-library/README.md index 82b03fbd..ba7537c2 100644 --- a/project-docs/phase13-legal-knowledge-template-library/README.md +++ b/project-docs/phase13-legal-knowledge-template-library/README.md @@ -30,5 +30,6 @@ 4. 阶段 13.3 第一批合同审核风险规则已建立,共 8 条,全部保持 `LEGAL_REVIEW`。 5. 阶段 13.4 第一批标准条款已建立,共 8 个,全部保持 `LEGAL_REVIEW` 且 `production_ready=false`。 6. 阶段 13.5 第一批黄金样本已建立,包括 8 个合同审核样本、8 个合同生成样本和 10 个知识库问答样本。 -7. 阶段 13.7 一键导入脚本已建立,默认 `Validate` 模式不改系统,`Review` 模式可导入待法务复核知识库和审核规则,`Production` 模式只导入已复核且生产就绪资料。 -8. 生产发布仍需等待法务复核和业务复核完成。 +7. 阶段 13.5 黄金样本评测脚本已建立,可离线评测审核样本、生成样本和知识库问答样本。 +8. 阶段 13.7 一键导入脚本已建立,默认 `Validate` 模式不改系统,`Review` 模式可导入待法务复核知识库和审核规则,`Production` 模式只导入已复核且生产就绪资料。 +9. 生产发布仍需等待法务复核和业务复核完成。 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/README.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/README.md index cda14e44..258af167 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/README.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/README.md @@ -15,3 +15,11 @@ 2. `generation` 已覆盖 8 个第一批标准条款。 3. `kb-qa` 已覆盖 10 张第一批法律知识卡。 4. 所有样本均保持 `LEGAL_REVIEW`,用于内部验收和法务复核。 + +评测命令: + +```powershell +powershell.exe -NoProfile -ExecutionPolicy Bypass -File scripts\run-phase13-golden-eval.ps1 +``` + +评测脚本会输出审核样本、生成样本和知识库问答样本的结构化评测报告。 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-AUTHORIZATION-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-AUTHORIZATION-001-expected.md index 08f7343b..37afb489 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-AUTHORIZATION-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-AUTHORIZATION-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-COMMON-AUTHORIZATION-001 +- RULE-AUTHORIZATION-SUBJECT-001 - LAW-COMPANY-001 - LAW-CIVIL-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-BREACH-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-BREACH-001-expected.md index 63b64a45..a2335934 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-BREACH-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-BREACH-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-COMMON-BREACH-001 +- RULE-BREACH-LIQUIDATED-DAMAGES-001 - LAW-CIVIL-001 - LAW-CONTRACT-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-DISPUTE-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-DISPUTE-001-expected.md index 55df59a9..de7b900f 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-DISPUTE-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-DISPUTE-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-COMMON-DISPUTE-001 +- RULE-DISPUTE-RESOLUTION-001 - LAW-ARBITRATION-001 - LAW-CIVIL-PROCEDURE-001 - LAW-CIVIL-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-E-SIGN-EVIDENCE-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-E-SIGN-EVIDENCE-001-expected.md index 05fdb0f8..c962770c 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-E-SIGN-EVIDENCE-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-E-SIGN-EVIDENCE-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-COMMON-E-SIGN-EVIDENCE-001 +- RULE-E-SIGN-EVIDENCE-001 - LAW-E-SIGN-001 - LAW-CIVIL-001 - LAW-CIVIL-PROCEDURE-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-FORMAT-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-FORMAT-001-expected.md index eb160b0f..6bab9c07 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-FORMAT-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-FORMAT-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-COMMON-FORMAT-001 +- RULE-FORMAT-CLAUSE-001 - LAW-CIVIL-001 - LAW-CONTRACT-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-RESCISSION-NOTICE-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-RESCISSION-NOTICE-001-expected.md index be5ceb6e..1511ad36 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-RESCISSION-NOTICE-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-COMMON-RESCISSION-NOTICE-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-COMMON-RESCISSION-NOTICE-001 +- RULE-RESCISSION-NOTICE-001 - LAW-CIVIL-001 - LAW-CONTRACT-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-DATA-PI-PROCESSING-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-DATA-PI-PROCESSING-001-expected.md index eda4069e..226a74f8 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-DATA-PI-PROCESSING-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-DATA-PI-PROCESSING-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-DATA-PI-PROCESSING-001 +- RULE-DATA-PI-PROCESSING-001 - LAW-DATA-001 - LAW-DATA-002 - LAW-DATA-003 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-TECH-IP-OWNERSHIP-001-expected.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-TECH-IP-OWNERSHIP-001-expected.md index e4f198ab..30a5a737 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-TECH-IP-OWNERSHIP-001-expected.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/generation/GEN-SAMPLE-CLAUSE-TECH-IP-OWNERSHIP-001-expected.md @@ -5,6 +5,7 @@ 必须引用: - CLAUSE-TECH-IP-OWNERSHIP-001 +- RULE-IP-SOFTWARE-OWNERSHIP-001 - LAW-IP-001 - LAW-CIVIL-001 - LAW-TECH-001 diff --git a/project-docs/phase13-legal-knowledge-template-library/qa-samples/review/REVIEW-SAMPLE-DISPUTE-RESOLUTION-001-input.md b/project-docs/phase13-legal-knowledge-template-library/qa-samples/review/REVIEW-SAMPLE-DISPUTE-RESOLUTION-001-input.md index a7aba663..d071d97c 100644 --- a/project-docs/phase13-legal-knowledge-template-library/qa-samples/review/REVIEW-SAMPLE-DISPUTE-RESOLUTION-001-input.md +++ b/project-docs/phase13-legal-knowledge-template-library/qa-samples/review/REVIEW-SAMPLE-DISPUTE-RESOLUTION-001-input.md @@ -1,3 +1,3 @@ # 审核输入样本:仲裁诉讼和送达冲突 -争议解决条款约定:发生争议时,双方可向甲方所在地人民法院起诉,也可提交北京仲裁解决。合同未写明具体仲裁委员会名称,也未约定送达地址、电子邮箱或地址变更通知规则。 +争议解决和管辖条款约定:发生争议时,双方可向甲方所在地人民法院起诉,也可提交北京仲裁解决。合同未写明具体仲裁委员会名称,也未约定送达地址、电子邮箱或地址变更通知规则。 diff --git a/project-docs/阶段13-法律知识库与合同模板库建设闭环第五批执行记录-2026-05-01.md b/project-docs/阶段13-法律知识库与合同模板库建设闭环第五批执行记录-2026-05-01.md new file mode 100644 index 00000000..70f8ff04 --- /dev/null +++ b/project-docs/阶段13-法律知识库与合同模板库建设闭环第五批执行记录-2026-05-01.md @@ -0,0 +1,110 @@ +# AI合同平台 阶段 13 法律知识库与合同模板库建设闭环第五批执行记录(2026-05-01) + +## 一、阶段结论 + +本批次继续深化 `黄金样本与评测集`,当前状态为 `PASS_WITH_LEGAL_REVIEW_REQUIRED`。 + +第四批已经建立了 8 个审核样本、8 个生成样本和 10 个知识库问答样本。本批次进一步新增离线评测脚本,把“样本存在”升级为“样本可跑、可评分、可输出证据”。评测集不调用生产 API,不导入未复核资料,只做结构化质量评测。 + +## 二、本批次交付 + +新增脚本: + +1. `scripts/run-phase13-golden-eval.ps1` + +更新资料: + +1. `scripts/run-legal-knowledge-library-gate.ps1` +2. `project-docs/phase13-legal-knowledge-template-library/README.md` +3. `project-docs/phase13-legal-knowledge-template-library/qa-samples/README.md` +4. 8 个合同生成样本期望文件,补齐对应审核规则编号。 +5. 1 个争议解决审核样本输入文件,补齐触发词“管辖”。 + +## 三、评测逻辑 + +合同审核样本评测: + +1. 样本必须绑定至少一条审核规则。 +2. 输入样本必须包含规则触发词。 +3. 期望发现文件必须包含规则编号。 +4. 期望风险等级必须与规则等级一致。 +5. 期望发现文件必须包含期望法源编号。 + +合同生成样本评测: + +1. 变量 JSON 必须可解析。 +2. `asset_under_test` 必须与清单中的被测资产一致。 +3. 被测资产必须存在于合同模板清单或标准条款清单。 +4. 期望文档必须包含被测条款编号。 +5. 期望文档必须包含期望审核规则编号和法源编号。 + +知识库问答样本评测: + +1. 每个样本必须通过 `notes` 绑定一张知识卡。 +2. 问题和回答边界必须有足够长度。 +3. 需要引用时必须配置期望法源编号。 +4. 期望拒答或转人工时,回答边界必须包含人工复核、不能、不得、不确定或专项复核等边界语义。 +5. 期望法源编号必须存在于法源清单。 + +## 四、问题修正 + +首次运行评测脚本时暴露 9 个问题: + +1. `REVIEW-SAMPLE-DISPUTE-RESOLUTION-001` 输入样本缺少触发词“管辖”。 +2. 8 个合同生成样本期望文档缺少对应审核规则编号。 + +本批次没有放宽评测脚本,而是修正样本内容,使评测集真正成为可发现资料缺陷的门禁。 + +## 五、自动化评测结果 + +黄金样本评测: + +```text +PHASE13_GOLDEN_EVAL_PASS_WITH_LEGAL_REVIEW_REQUIRED D:\ai-contract-platform\project-docs\qa-artifacts\phase13-golden-eval\2026-05-01T22-27-25-109 +``` + +导入脚本离线校验: + +```text +PHASE13_IMPORT_PASS D:\ai-contract-platform\project-docs\qa-artifacts\phase13-import\2026-05-01T22-27-52-687 +``` + +阶段 13 资料门禁: + +```text +PHASE13_LEGAL_KNOWLEDGE_LIBRARY_GATE_PASS_WITH_LEGAL_REVIEW_REQUIRED D:\ai-contract-platform\project-docs\qa-artifacts\phase13-legal-knowledge-library-gate\2026-05-01T22-28-09-061 +``` + +评测统计: + +| 指标 | 结果 | +|---|---:| +| 合同审核样本 | 8 | +| 合同审核样本通过 | 8 | +| 合同生成样本 | 8 | +| 合同生成样本通过 | 8 | +| 知识库问答样本 | 10 | +| 知识库问答样本通过 | 10 | +| 总样本 | 26 | +| 总通过 | 26 | +| 通过率 | 100% | +| 必需项失败 | 0 | + +## 六、当前保留项 + +| 保留项 | 状态 | 说明 | +|---|---|---| +| 样本法务复核 | LEGAL_REVIEW | 26 个样本仍需法务确认期望结论和拒答边界 | +| 业务变量确认 | BUSINESS_REVIEW_PENDING | 合同生成样本中的期限、比例、责任上限、机构和地点仍需业务填写 | +| 生产评测 | BLOCKED_BY_REVIEW | 未复核资料不得作为生产确定性评测基准 | + +## 七、下一步 + +下一批建议进入 `13.6 法务复核工作台资料包`: + +1. 生成覆盖所有 `LEGAL_REVIEW` 资产的法务复核任务。 +2. 为法源、知识卡、规则、条款和样本建立复核意见模板。 +3. 将复核任务登记进 `expert-review-register.csv`。 +4. 门禁增加“所有待复核资产必须有复核任务”的覆盖检查。 + +阶段 13 第五批当前状态:`PASS_WITH_LEGAL_REVIEW_REQUIRED`。 diff --git a/scripts/run-legal-knowledge-library-gate.ps1 b/scripts/run-legal-knowledge-library-gate.ps1 index d0c05a21..f8cb96c1 100644 --- a/scripts/run-legal-knowledge-library-gate.ps1 +++ b/scripts/run-legal-knowledge-library-gate.ps1 @@ -340,7 +340,8 @@ $commonDocFiles = @( "project-docs\phase13-legal-knowledge-template-library\README.md", "project-docs\phase13-legal-knowledge-template-library\governance-rules.md", "project-docs\phase13-legal-knowledge-template-library\field-dictionary.md", - "scripts\import-phase13-library.ps1" + "scripts\import-phase13-library.ps1", + "scripts\run-phase13-golden-eval.ps1" ) foreach ($docPath in $commonDocFiles) { $absoluteDocPath = Join-Path $repoRoot $docPath diff --git a/scripts/run-phase13-golden-eval.ps1 b/scripts/run-phase13-golden-eval.ps1 new file mode 100644 index 00000000..5e7338a4 --- /dev/null +++ b/scripts/run-phase13-golden-eval.ps1 @@ -0,0 +1,513 @@ +param( + [string]$ArtifactRoot = "", + [string]$LibraryRoot = "", + [int]$MinReviewSamples = 8, + [int]$MinGenerationSamples = 8, + [int]$MinKbQaSamples = 10, + [int]$MinPassRatePercent = 100, + [switch]$FailOnLegalReviewRequired +) + +$ErrorActionPreference = "Stop" + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff" + +if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase13-golden-eval" +} +if ([string]::IsNullOrWhiteSpace($LibraryRoot)) { + $LibraryRoot = Join-Path $repoRoot "project-docs\phase13-legal-knowledge-template-library" +} +foreach ($pathName in @("ArtifactRoot", "LibraryRoot")) { + $value = Get-Variable -Name $pathName -ValueOnly + if (-not [System.IO.Path]::IsPathRooted($value)) { + Set-Variable -Name $pathName -Value (Join-Path $repoRoot $value) + } +} + +$artifactDir = Join-Path $ArtifactRoot $timestamp +New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null + +$summaryJson = Join-Path $artifactDir "phase13-golden-eval-summary.json" +$summaryMd = Join-Path $artifactDir "phase13-golden-eval-summary.md" +$reviewReportCsv = Join-Path $artifactDir "review-sample-eval-report.csv" +$generationReportCsv = Join-Path $artifactDir "generation-sample-eval-report.csv" +$kbQaReportCsv = Join-Path $artifactDir "kb-qa-sample-eval-report.csv" +$gitStatusPath = Join-Path $artifactDir "git-status.txt" + +$checks = New-Object System.Collections.Generic.List[object] + +function Add-Check { + param( + [string]$Key, + [string]$Label, + [ValidateSet("pass", "fail", "warn", "skip", "blocked")] + [string]$Status, + [string]$Detail, + [bool]$Required = $true, + [hashtable]$Meta = @{} + ) + + $checks.Add([ordered]@{ + key = $Key + label = $Label + status = $Status + required = $Required + detail = $Detail + meta = $Meta + }) | Out-Null +} + +function Save-Json { + param([string]$Path, [object]$Value) + $Value | ConvertTo-Json -Depth 30 | Set-Content -Encoding UTF8 -Path $Path +} + +function Import-CsvRows { + param([string]$Path) + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + return @() + } + return @(Import-Csv -LiteralPath $Path) +} + +function Split-SemicolonValues { + param([string]$Value) + if ([string]::IsNullOrWhiteSpace($Value)) { + return @() + } + return @($Value -split ";" | ForEach-Object { $_.Trim() } | Where-Object { $_ }) +} + +function Test-RequiredFlag { + param([string]$Value) + if ([string]::IsNullOrWhiteSpace($Value)) { + return $false + } + return @("true", "yes", "1", "required") -contains $Value.ToLowerInvariant() +} + +function Resolve-LibraryPath { + param([string]$RelativePath) + return Join-Path $LibraryRoot ($RelativePath -replace "/", "\") +} + +function Read-LibraryText { + param([string]$RelativePath) + if ([string]::IsNullOrWhiteSpace($RelativePath)) { + return "" + } + $path = Resolve-LibraryPath -RelativePath $RelativePath + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + return "" + } + return Get-Content -LiteralPath $path -Raw -Encoding UTF8 +} + +function Test-TextContainsAll { + param([string]$Text, [string[]]$Needles) + $missing = New-Object System.Collections.Generic.List[string] + foreach ($needle in $Needles) { + if (-not [string]::IsNullOrWhiteSpace($needle) -and -not $Text.Contains($needle)) { + $missing.Add($needle) | Out-Null + } + } + return $missing.ToArray() +} + +function Convert-GateStatusText { + param([string]$Status) + switch ($Status) { + "PASS" { return "通过" } + "FAIL" { return "未通过" } + "PASS_WITH_LEGAL_REVIEW_REQUIRED" { return "通过但仍需法务复核" } + default { return $Status } + } +} + +function Convert-CheckStatusText { + param([string]$Status) + switch ($Status) { + "pass" { return "通过" } + "fail" { return "失败" } + "warn" { return "警告" } + "skip" { return "跳过" } + "blocked" { return "阻塞" } + default { return $Status } + } +} + +function Get-PassRatePercent { + param([int]$Passed, [int]$Total) + if ($Total -le 0) { + return 0 + } + return [math]::Round(($Passed * 100.0) / $Total, 2) +} + +if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { + Add-Check -Key "gstack" -Label "gstack 前置检查" -Status "fail" -Detail "gstack 缺失" +} else { + Add-Check -Key "gstack" -Label "gstack 前置检查" -Status "pass" -Detail "GSTACK_OK" +} + +$branch = (git -C $repoRoot rev-parse --abbrev-ref HEAD).Trim() +$commit = (git -C $repoRoot rev-parse HEAD).Trim() +$remoteUrl = (git -C $repoRoot remote get-url origin).Trim() +$gitStatus = git -C $repoRoot status --short +($gitStatus -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path $gitStatusPath + +if ($remoteUrl -match "192\.168\.3\.90") { + Add-Check -Key "origin_remote" -Label "origin 远端仓库" -Status "pass" -Detail "origin 指向 192.168.3.90" +} else { + Add-Check -Key "origin_remote" -Label "origin 远端仓库" -Status "fail" -Detail "origin 未指向 192.168.3.90" -Meta @{ remote = $remoteUrl } +} + +if (Test-Path -LiteralPath $LibraryRoot -PathType Container) { + Add-Check -Key "library_root" -Label "阶段 13 资料包根目录" -Status "pass" -Detail "资料包根目录存在" +} else { + Add-Check -Key "library_root" -Label "阶段 13 资料包根目录" -Status "fail" -Detail "资料包根目录缺失" -Meta @{ root = $LibraryRoot } +} + +$inventoryRoot = Join-Path $LibraryRoot "inventories" +$legalSources = Import-CsvRows -Path (Join-Path $inventoryRoot "legal-source-inventory.csv") +$knowledgeCards = Import-CsvRows -Path (Join-Path $inventoryRoot "legal-knowledge-card-inventory.csv") +$reviewRules = Import-CsvRows -Path (Join-Path $inventoryRoot "review-risk-rule-inventory.csv") +$clauses = Import-CsvRows -Path (Join-Path $inventoryRoot "clause-library-inventory.csv") +$templates = Import-CsvRows -Path (Join-Path $inventoryRoot "contract-template-inventory.csv") +$reviewSamples = Import-CsvRows -Path (Join-Path $inventoryRoot "review-golden-sample-inventory.csv") +$generationSamples = Import-CsvRows -Path (Join-Path $inventoryRoot "generation-golden-sample-inventory.csv") +$kbQaSamples = Import-CsvRows -Path (Join-Path $inventoryRoot "kb-qa-golden-sample-inventory.csv") + +$sourceMap = @{} +foreach ($source in $legalSources) { + if (-not [string]::IsNullOrWhiteSpace($source.source_id)) { + $sourceMap[$source.source_id] = $true + } +} +$ruleMap = @{} +foreach ($rule in $reviewRules) { + if (-not [string]::IsNullOrWhiteSpace($rule.rule_id)) { + $ruleMap[$rule.rule_id] = $rule + } +} +$clauseMap = @{} +foreach ($clause in $clauses) { + if (-not [string]::IsNullOrWhiteSpace($clause.clause_id)) { + $clauseMap[$clause.clause_id] = $true + } +} +$templateMap = @{} +foreach ($template in $templates) { + if (-not [string]::IsNullOrWhiteSpace($template.template_id)) { + $templateMap[$template.template_id] = $true + } +} + +if ($reviewSamples.Count -ge $MinReviewSamples) { + Add-Check -Key "review_sample_minimum" -Label "审核样本数量下限" -Status "pass" -Detail "审核样本数量=$($reviewSamples.Count),最低要求=$MinReviewSamples" +} else { + Add-Check -Key "review_sample_minimum" -Label "审核样本数量下限" -Status "fail" -Detail "审核样本数量=$($reviewSamples.Count),最低要求=$MinReviewSamples" +} +if ($generationSamples.Count -ge $MinGenerationSamples) { + Add-Check -Key "generation_sample_minimum" -Label "生成样本数量下限" -Status "pass" -Detail "生成样本数量=$($generationSamples.Count),最低要求=$MinGenerationSamples" +} else { + Add-Check -Key "generation_sample_minimum" -Label "生成样本数量下限" -Status "fail" -Detail "生成样本数量=$($generationSamples.Count),最低要求=$MinGenerationSamples" +} +if ($kbQaSamples.Count -ge $MinKbQaSamples) { + Add-Check -Key "kb_qa_sample_minimum" -Label "问答样本数量下限" -Status "pass" -Detail "问答样本数量=$($kbQaSamples.Count),最低要求=$MinKbQaSamples" +} else { + Add-Check -Key "kb_qa_sample_minimum" -Label "问答样本数量下限" -Status "fail" -Detail "问答样本数量=$($kbQaSamples.Count),最低要求=$MinKbQaSamples" +} + +$reviewEval = New-Object System.Collections.Generic.List[object] +foreach ($sample in $reviewSamples) { + $issues = New-Object System.Collections.Generic.List[string] + $inputText = Read-LibraryText -RelativePath $sample.input_sample_path + $expectedText = Read-LibraryText -RelativePath $sample.expected_findings_path + if ([string]::IsNullOrWhiteSpace($inputText)) { + $issues.Add("input_sample_missing_or_empty") | Out-Null + } + if ([string]::IsNullOrWhiteSpace($expectedText)) { + $issues.Add("expected_findings_missing_or_empty") | Out-Null + } + + $boundRules = @($reviewRules | Where-Object { (Split-SemicolonValues -Value $_.review_sample_ids) -contains $sample.sample_id }) + if ($boundRules.Count -eq 0) { + $issues.Add("no_bound_review_rule") | Out-Null + } + + foreach ($rule in $boundRules) { + if (-not [string]::IsNullOrWhiteSpace($rule.trigger_condition) -and -not $inputText.Contains($rule.trigger_condition)) { + $issues.Add("trigger_missing:$($rule.rule_id):$($rule.trigger_condition)") | Out-Null + } + if (-not $expectedText.Contains($rule.rule_id)) { + $issues.Add("expected_rule_missing:$($rule.rule_id)") | Out-Null + } + if ($rule.severity -ne $sample.expected_severity) { + $issues.Add("severity_mismatch:$($rule.rule_id)") | Out-Null + } + } + + $missingExpectedSources = Test-TextContainsAll -Text $expectedText -Needles (Split-SemicolonValues -Value $sample.expected_source_citations) + foreach ($sourceId in $missingExpectedSources) { + $issues.Add("expected_source_missing_in_file:$sourceId") | Out-Null + } + foreach ($sourceId in (Split-SemicolonValues -Value $sample.expected_source_citations)) { + if (-not $sourceMap.ContainsKey($sourceId)) { + $issues.Add("unknown_source:$sourceId") | Out-Null + } + } + + $reviewEval.Add([pscustomobject][ordered]@{ + sample_id = $sample.sample_id + asset_type = "review_sample" + status = if ($issues.Count -eq 0) { "pass" } else { "fail" } + expected_severity = $sample.expected_severity + bound_rule_ids = (@($boundRules | Select-Object -ExpandProperty rule_id) -join ";") + expected_source_citations = $sample.expected_source_citations + issue_count = $issues.Count + issues = ($issues -join ";") + }) | Out-Null +} + +$generationEval = New-Object System.Collections.Generic.List[object] +foreach ($sample in $generationSamples) { + $issues = New-Object System.Collections.Generic.List[string] + $variablePath = Resolve-LibraryPath -RelativePath $sample.variable_payload_path + $expectedText = Read-LibraryText -RelativePath $sample.expected_document_path + $variables = $null + if (-not (Test-Path -LiteralPath $variablePath -PathType Leaf)) { + $issues.Add("variable_payload_missing") | Out-Null + } else { + try { + $variables = Get-Content -LiteralPath $variablePath -Raw -Encoding UTF8 | ConvertFrom-Json + } catch { + $issues.Add("variable_payload_invalid_json") | Out-Null + } + } + if ([string]::IsNullOrWhiteSpace($expectedText)) { + $issues.Add("expected_document_missing_or_empty") | Out-Null + } + + if (-not ($clauseMap.ContainsKey($sample.template_id) -or $templateMap.ContainsKey($sample.template_id))) { + $issues.Add("unknown_generation_asset:$($sample.template_id)") | Out-Null + } + if ($null -ne $variables -and $variables.asset_under_test -ne $sample.template_id) { + $issues.Add("variable_asset_mismatch:$($variables.asset_under_test)") | Out-Null + } + if (-not $expectedText.Contains($sample.template_id)) { + $issues.Add("expected_asset_missing:$($sample.template_id)") | Out-Null + } + foreach ($ruleId in (Split-SemicolonValues -Value $sample.expected_review_rule_ids)) { + if (-not $ruleMap.ContainsKey($ruleId)) { + $issues.Add("unknown_rule:$ruleId") | Out-Null + } + if (-not $expectedText.Contains($ruleId)) { + $issues.Add("expected_rule_missing:$ruleId") | Out-Null + } + } + foreach ($sourceId in (Split-SemicolonValues -Value $sample.expected_source_citations)) { + if (-not $sourceMap.ContainsKey($sourceId)) { + $issues.Add("unknown_source:$sourceId") | Out-Null + } + if (-not $expectedText.Contains($sourceId)) { + $issues.Add("expected_source_missing:$sourceId") | Out-Null + } + } + + $generationEval.Add([pscustomobject][ordered]@{ + sample_id = $sample.sample_id + asset_type = "generation_sample" + status = if ($issues.Count -eq 0) { "pass" } else { "fail" } + asset_under_test = $sample.template_id + expected_review_rule_ids = $sample.expected_review_rule_ids + expected_source_citations = $sample.expected_source_citations + issue_count = $issues.Count + issues = ($issues -join ";") + }) | Out-Null +} + +$kbQaEval = New-Object System.Collections.Generic.List[object] +foreach ($sample in $kbQaSamples) { + $issues = New-Object System.Collections.Generic.List[string] + $boundCards = @($knowledgeCards | Where-Object { $sample.notes -match [regex]::Escape($_.knowledge_card_id) }) + if ($boundCards.Count -eq 0) { + $issues.Add("no_bound_knowledge_card") | Out-Null + } + if ([string]::IsNullOrWhiteSpace($sample.question) -or $sample.question.Length -lt 8) { + $issues.Add("question_too_short") | Out-Null + } + if ([string]::IsNullOrWhiteSpace($sample.expected_answer_boundary) -or $sample.expected_answer_boundary.Length -lt 12) { + $issues.Add("answer_boundary_too_short") | Out-Null + } + if ((Test-RequiredFlag -Value $sample.expected_citation_required) -and [string]::IsNullOrWhiteSpace($sample.expected_source_ids)) { + $issues.Add("citation_required_without_sources") | Out-Null + } + if ((Test-RequiredFlag -Value $sample.expected_refusal_required) -and $sample.expected_answer_boundary -notmatch "不能|不得|不确定|人工复核|专项复核|必须") { + $issues.Add("refusal_boundary_too_weak") | Out-Null + } + foreach ($sourceId in (Split-SemicolonValues -Value $sample.expected_source_ids)) { + if (-not $sourceMap.ContainsKey($sourceId)) { + $issues.Add("unknown_source:$sourceId") | Out-Null + } + } + + $kbQaEval.Add([pscustomobject][ordered]@{ + sample_id = $sample.sample_id + asset_type = "kb_qa_sample" + status = if ($issues.Count -eq 0) { "pass" } else { "fail" } + bound_knowledge_card_ids = (@($boundCards | Select-Object -ExpandProperty knowledge_card_id) -join ";") + expected_source_ids = $sample.expected_source_ids + expected_refusal_required = $sample.expected_refusal_required + issue_count = $issues.Count + issues = ($issues -join ";") + }) | Out-Null +} + +$reviewPass = @($reviewEval | Where-Object { $_.status -eq "pass" }).Count +$generationPass = @($generationEval | Where-Object { $_.status -eq "pass" }).Count +$kbQaPass = @($kbQaEval | Where-Object { $_.status -eq "pass" }).Count +$totalSamples = $reviewEval.Count + $generationEval.Count + $kbQaEval.Count +$totalPass = $reviewPass + $generationPass + $kbQaPass +$passRate = Get-PassRatePercent -Passed $totalPass -Total $totalSamples + +if ($reviewEval.Count -gt 0 -and $reviewPass -eq $reviewEval.Count) { + Add-Check -Key "review_eval" -Label "审核黄金样本评测" -Status "pass" -Detail "审核样本通过 $reviewPass/$($reviewEval.Count)" +} else { + Add-Check -Key "review_eval" -Label "审核黄金样本评测" -Status "fail" -Detail "审核样本通过 $reviewPass/$($reviewEval.Count)" +} +if ($generationEval.Count -gt 0 -and $generationPass -eq $generationEval.Count) { + Add-Check -Key "generation_eval" -Label "生成黄金样本评测" -Status "pass" -Detail "生成样本通过 $generationPass/$($generationEval.Count)" +} else { + Add-Check -Key "generation_eval" -Label "生成黄金样本评测" -Status "fail" -Detail "生成样本通过 $generationPass/$($generationEval.Count)" +} +if ($kbQaEval.Count -gt 0 -and $kbQaPass -eq $kbQaEval.Count) { + Add-Check -Key "kb_qa_eval" -Label "知识库问答样本评测" -Status "pass" -Detail "问答样本通过 $kbQaPass/$($kbQaEval.Count)" +} else { + Add-Check -Key "kb_qa_eval" -Label "知识库问答样本评测" -Status "fail" -Detail "问答样本通过 $kbQaPass/$($kbQaEval.Count)" +} +if ($passRate -ge $MinPassRatePercent) { + Add-Check -Key "sample_pass_rate" -Label "样本通过率门槛" -Status "pass" -Detail ("通过率={0}%,最低要求={1}%" -f $passRate, $MinPassRatePercent) +} else { + Add-Check -Key "sample_pass_rate" -Label "样本通过率门槛" -Status "fail" -Detail ("通过率={0}%,最低要求={1}%" -f $passRate, $MinPassRatePercent) +} + +$reviewRequiredRows = @( + @($reviewSamples | Where-Object { $_.status -eq "LEGAL_REVIEW" -or (Test-RequiredFlag -Value $_.review_required) }) + @($generationSamples | Where-Object { $_.status -eq "LEGAL_REVIEW" -or (Test-RequiredFlag -Value $_.review_required) }) + @($kbQaSamples | Where-Object { $_.status -eq "LEGAL_REVIEW" -or (Test-RequiredFlag -Value $_.review_required) }) +) +$legalReviewRequired = $reviewRequiredRows.Count +if ($legalReviewRequired -gt 0) { + Add-Check -Key "sample_legal_review_required" -Label "样本法务复核要求" -Status "warn" -Required $false -Detail "仍有 $legalReviewRequired 个样本需要法务复核" +} else { + Add-Check -Key "sample_legal_review_required" -Label "样本法务复核要求" -Status "pass" -Detail "样本已完成法务复核" +} + +$reviewEval | Export-Csv -LiteralPath $reviewReportCsv -NoTypeInformation -Encoding UTF8 +$generationEval | Export-Csv -LiteralPath $generationReportCsv -NoTypeInformation -Encoding UTF8 +$kbQaEval | Export-Csv -LiteralPath $kbQaReportCsv -NoTypeInformation -Encoding UTF8 + +$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }).Count +$warnings = @($checks | Where-Object { $_.status -eq "warn" }).Count +$overallStatus = if ($failedRequired -gt 0) { + "FAIL" +} elseif ($legalReviewRequired -gt 0) { + "PASS_WITH_LEGAL_REVIEW_REQUIRED" +} else { + "PASS" +} + +$summary = [ordered]@{ + name = "phase13-golden-eval" + status = $overallStatus + generated_at = (Get-Date).ToUniversalTime().ToString("o") + artifact_dir = $artifactDir + branch = $branch + commit = $commit + remote = $remoteUrl + library_root = $LibraryRoot + checks = $checks.ToArray() + metrics = [ordered]@{ + review_samples = $reviewEval.Count + review_samples_passed = $reviewPass + generation_samples = $generationEval.Count + generation_samples_passed = $generationPass + kb_qa_samples = $kbQaEval.Count + kb_qa_samples_passed = $kbQaPass + total_samples = $totalSamples + total_passed = $totalPass + pass_rate_percent = $passRate + legal_review_required = $legalReviewRequired + } + artifacts = [ordered]@{ + review_report = $reviewReportCsv + generation_report = $generationReportCsv + kb_qa_report = $kbQaReportCsv + git_status = $gitStatusPath + } + totals = [ordered]@{ + total_checks = $checks.Count + failed_required = $failedRequired + warnings = $warnings + } +} +Save-Json -Path $summaryJson -Value $summary + +$statusText = Convert-GateStatusText -Status $overallStatus +$md = New-Object System.Collections.Generic.List[string] +$md.Add("# 阶段 13 黄金样本评测报告") | Out-Null +$md.Add("") | Out-Null +$md.Add("- 评测状态:$statusText($overallStatus)") | Out-Null +$md.Add("- 生成时间:$($summary.generated_at)") | Out-Null +$md.Add("- 分支:$branch") | Out-Null +$md.Add("- 提交:$commit") | Out-Null +$md.Add("- 产物目录:$artifactDir") | Out-Null +$md.Add("") | Out-Null +$md.Add("## 指标") | Out-Null +$md.Add("") | Out-Null +$md.Add("| 指标 | 数量 |") | Out-Null +$md.Add("|---|---:|") | Out-Null +$md.Add("| 审核样本 | $($reviewEval.Count) |") | Out-Null +$md.Add("| 审核样本通过 | $reviewPass |") | Out-Null +$md.Add("| 生成样本 | $($generationEval.Count) |") | Out-Null +$md.Add("| 生成样本通过 | $generationPass |") | Out-Null +$md.Add("| 知识库问答样本 | $($kbQaEval.Count) |") | Out-Null +$md.Add("| 知识库问答样本通过 | $kbQaPass |") | Out-Null +$md.Add("| 总样本 | $totalSamples |") | Out-Null +$md.Add("| 总通过 | $totalPass |") | Out-Null +$md.Add(("| 通过率 | {0}% |" -f $passRate)) | Out-Null +$md.Add("| 仍需法务复核的样本 | $legalReviewRequired |") | Out-Null +$md.Add("") | Out-Null +$md.Add("## 检查项") | Out-Null +$md.Add("") | Out-Null +$md.Add("| 检查项 | 必需 | 状态 | 说明 |") | Out-Null +$md.Add("|---|---:|---|---|") | Out-Null +foreach ($check in $checks) { + $requiredText = if ($check.required) { "是" } else { "否" } + $detail = ([string]$check.detail).Replace("|", "/") + $checkStatusText = Convert-CheckStatusText -Status $check.status + $md.Add("| $($check.label) | $requiredText | $checkStatusText($($check.status.ToUpperInvariant())) | $detail |") | Out-Null +} +$md.Add("") | Out-Null +$md.Add("本地产物:") | Out-Null +$md.Add("") | Out-Null +$md.Add('- `phase13-golden-eval-summary.json`') | Out-Null +$md.Add('- `phase13-golden-eval-summary.md`') | Out-Null +$md.Add('- `review-sample-eval-report.csv`') | Out-Null +$md.Add('- `generation-sample-eval-report.csv`') | Out-Null +$md.Add('- `kb-qa-sample-eval-report.csv`') | Out-Null +$md | Set-Content -Encoding UTF8 -Path $summaryMd + +if ($overallStatus -eq "FAIL") { + Write-Host "PHASE13_GOLDEN_EVAL_FAIL $artifactDir" + throw "阶段 13 黄金样本评测未通过。查看:$summaryMd" +} +if ($overallStatus -eq "PASS_WITH_LEGAL_REVIEW_REQUIRED" -and $FailOnLegalReviewRequired) { + Write-Host "PHASE13_GOLDEN_EVAL_PASS_WITH_LEGAL_REVIEW_REQUIRED $artifactDir" + throw "阶段 13 黄金样本评测仍需法务复核。查看:$summaryMd" +} + +Write-Host "PHASE13_GOLDEN_EVAL_$overallStatus $artifactDir"