1138 lines
54 KiB
Python
1138 lines
54 KiB
Python
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
import httpx
|
|
|
|
|
|
@dataclass
|
|
class StructuredReviewCase:
|
|
case_id: str
|
|
label: str
|
|
contract_type: Optional[str]
|
|
review_as: str
|
|
file_name: str
|
|
content: str
|
|
expected_contract_type: str
|
|
expected_clause_types: List[str]
|
|
expected_missing_sections: List[str]
|
|
batch_id: str = "B-000"
|
|
source_origin: str = "deidentified_realistic"
|
|
redaction_level: str = "sanitized"
|
|
expected_amount_contains: Optional[str] = None
|
|
expected_term_start: Optional[str] = None
|
|
expected_term_end: Optional[str] = None
|
|
expected_payment_milestones: int = 0
|
|
expected_relationship_types: Optional[List[str]] = None
|
|
expected_absent_relationship_types: Optional[List[str]] = None
|
|
expected_relationship_confidence: Optional[Dict[str, str]] = None
|
|
expected_ip_owner: Optional[str] = None
|
|
expected_liability_carve_outs: Optional[List[str]] = None
|
|
|
|
|
|
CASES = [
|
|
StructuredReviewCase(
|
|
case_id="S-001",
|
|
label="服务合同结构化摘要",
|
|
contract_type="service",
|
|
review_as="party_a",
|
|
file_name="structured-service.txt",
|
|
content="\n".join(
|
|
[
|
|
"服务合同",
|
|
"甲方：甲方科技有限公司",
|
|
"乙方：乙方软件有限公司",
|
|
"合同金额：人民币500000元",
|
|
"服务期限：自2026年5月1日至2027年5月1日",
|
|
"付款条款：甲方应在验收合格并收到发票后10个工作日内付款。",
|
|
"保密条款：乙方应对商业秘密承担保密义务。",
|
|
"违约条款：乙方延迟交付应承担违约责任。",
|
|
]
|
|
),
|
|
expected_contract_type="service",
|
|
expected_clause_types=["payment", "confidentiality", "breach"],
|
|
expected_missing_sections=["争议解决条款"],
|
|
expected_amount_contains="500000",
|
|
expected_term_start="2026-05-01",
|
|
expected_term_end="2027-05-01",
|
|
expected_payment_milestones=1,
|
|
expected_relationship_types=["payment_depends_on_acceptance", "payment_depends_on_invoice"],
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-002",
|
|
label="采购合同缺失条款提示",
|
|
contract_type="procurement",
|
|
review_as="party_a",
|
|
file_name="structured-procurement.txt",
|
|
content="\n".join(
|
|
[
|
|
"采购合同",
|
|
"甲方：甲方商贸有限公司",
|
|
"乙方：乙方供应链有限公司",
|
|
"总价：￥120000元",
|
|
"付款方式：货到后付款，具体节点另行协商。",
|
|
"交付条款：乙方应在30日内交货。",
|
|
]
|
|
),
|
|
expected_contract_type="procurement",
|
|
expected_clause_types=["payment", "delivery"],
|
|
expected_missing_sections=["验收标准或验收流程", "保密范围与期限"],
|
|
expected_amount_contains="120000",
|
|
expected_payment_milestones=1,
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-003",
|
|
label="购销合同编号条款分段",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-sales.txt",
|
|
content="\n".join(
|
|
[
|
|
"购销合同",
|
|
"甲方：甲方制造有限公司",
|
|
"乙方：乙方贸易有限公司",
|
|
"第一条 交货条款",
|
|
"乙方应在收到订单后十五日内完成交货并配合到货验收。",
|
|
"第二条 付款条款",
|
|
"甲方应在验收合格并收到增值税专用发票后七个工作日内付款。",
|
|
"第三条 争议解决",
|
|
"双方同意提交上海仲裁委员会仲裁。",
|
|
]
|
|
),
|
|
expected_contract_type="sales",
|
|
expected_clause_types=["delivery", "payment", "dispute_resolution"],
|
|
expected_missing_sections=["保密范围与期限", "违约责任安排"],
|
|
expected_payment_milestones=1,
|
|
expected_relationship_types=["payment_depends_on_acceptance", "payment_depends_on_invoice"],
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-004",
|
|
label="租赁合同责任限制识别",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-lease.txt",
|
|
content="\n".join(
|
|
[
|
|
"设备租赁合同",
|
|
"甲方：甲方设备有限公司",
|
|
"乙方：乙方工程有限公司",
|
|
"租赁期限：自2026年6月1日至2027年6月1日",
|
|
"第一条 租赁物交付",
|
|
"甲方应在合同生效后五日内完成设备交付。",
|
|
"第二条 责任限制",
|
|
"任何一方的赔偿责任以六个月租金总额为上限，但故意或重大过失除外。",
|
|
"第三条 违约责任",
|
|
"乙方逾期返还设备的，应按日承担违约责任。",
|
|
]
|
|
),
|
|
expected_contract_type="lease",
|
|
expected_clause_types=["delivery", "liability", "breach"],
|
|
expected_missing_sections=["付款与结算安排", "争议解决条款"],
|
|
expected_term_start="2026-06-01",
|
|
expected_term_end="2027-06-01",
|
|
expected_relationship_types=["liability_cap_conflicts_with_breach"],
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-005",
|
|
label="开发合同付款节点拆解",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-development.txt",
|
|
content="\n".join(
|
|
[
|
|
"软件开发合同",
|
|
"甲方：甲方信息有限公司",
|
|
"乙方：乙方科技有限公司",
|
|
"合同金额：人民币800000元",
|
|
"服务期限：自2026年7月1日至2027年1月31日",
|
|
"第一条 付款条款",
|
|
"甲方应在合同签署后5个工作日内支付首期款30%；验收合格并收到发票后10个工作日内支付尾款70%。",
|
|
"第二条 验收条款",
|
|
"乙方交付系统并完成验收后，方可进入尾款结算。",
|
|
]
|
|
),
|
|
expected_contract_type="development",
|
|
expected_clause_types=["payment", "acceptance"],
|
|
expected_missing_sections=["保密范围与期限", "违约责任安排", "争议解决条款"],
|
|
expected_amount_contains="800000",
|
|
expected_term_start="2026-07-01",
|
|
expected_term_end="2027-01-31",
|
|
expected_payment_milestones=2,
|
|
expected_relationship_types=["payment_depends_on_acceptance", "payment_depends_on_invoice"],
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-006",
|
|
label="复杂实施合同关系深化",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-implementation.txt",
|
|
content="\n".join(
|
|
[
|
|
"软件实施服务合同",
|
|
"甲方：甲方集团有限公司",
|
|
"乙方：乙方实施服务有限公司",
|
|
"合同金额：人民币1500000元",
|
|
"服务期限：自2026年8月1日至2027年8月1日",
|
|
"第一条 付款条款",
|
|
"甲方应在合同签署后5个工作日内支付预付款20%；系统验收合格并收到增值税专用发票后10个工作日内支付实施款50%；系统上线稳定运行30日后支付尾款20%；剩余10%作为质保金，在质保期满后10个工作日内支付。",
|
|
"第二条 验收条款",
|
|
"乙方完成实施并通过甲方书面验收后，方可进入后续付款节点。",
|
|
"第三条 保密条款",
|
|
"乙方应对甲方商业秘密、客户数据和项目资料承担严格保密义务。",
|
|
"第四条 知识产权",
|
|
"项目交付成果、源代码及相关文档的知识产权归甲方所有。",
|
|
"第五条 责任限制",
|
|
"任何一方因本合同产生的赔偿责任以合同总金额为上限，但故意或重大过失除外。",
|
|
"第六条 违约责任",
|
|
"乙方延迟交付、泄露保密信息或侵犯第三方知识产权的，应承担违约责任并赔偿损失。",
|
|
]
|
|
),
|
|
expected_contract_type="service",
|
|
expected_clause_types=["payment", "acceptance", "confidentiality", "intellectual_property", "liability", "breach"],
|
|
expected_missing_sections=["争议解决条款"],
|
|
expected_amount_contains="1500000",
|
|
expected_term_start="2026-08-01",
|
|
expected_term_end="2027-08-01",
|
|
expected_payment_milestones=4,
|
|
expected_relationship_types=[
|
|
"payment_depends_on_acceptance",
|
|
"payment_depends_on_invoice",
|
|
"payment_depends_on_go_live",
|
|
"payment_after_warranty_period",
|
|
"liability_cap_conflicts_with_breach",
|
|
"liability_cap_conflicts_with_confidentiality",
|
|
"liability_cap_conflicts_with_ip",
|
|
],
|
|
expected_relationship_confidence={
|
|
"payment_depends_on_acceptance": "high",
|
|
"payment_after_warranty_period": "high",
|
|
"liability_cap_conflicts_with_ip": "high",
|
|
},
|
|
expected_ip_owner="party_a",
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-007",
|
|
label="实施合同责任限制含 carve-out",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-carveout.txt",
|
|
content="\n".join(
|
|
[
|
|
"软件实施服务合同",
|
|
"甲方：甲方集团有限公司",
|
|
"乙方：乙方实施服务有限公司",
|
|
"合同金额：人民币960000元",
|
|
"第一条 付款条款",
|
|
"甲方应在验收合格并收到发票后15个工作日内支付全部合同款。",
|
|
"第二条 保密条款",
|
|
"乙方应对甲方商业秘密承担保密义务。",
|
|
"第三条 知识产权",
|
|
"项目交付成果、源代码及相关文档的知识产权归甲方所有。",
|
|
"第四条 责任限制",
|
|
"任何一方的赔偿责任以合同总金额为上限，但保密义务、知识产权侵权、故意或重大过失除外。",
|
|
"第五条 违约责任",
|
|
"乙方违反保密义务或侵犯知识产权的，应承担违约责任并赔偿损失。",
|
|
]
|
|
),
|
|
expected_contract_type="service",
|
|
expected_clause_types=["payment", "confidentiality", "intellectual_property", "liability", "breach"],
|
|
expected_missing_sections=["验收标准或验收流程", "争议解决条款"],
|
|
expected_amount_contains="960000",
|
|
expected_payment_milestones=1,
|
|
expected_relationship_types=["payment_depends_on_acceptance", "payment_depends_on_invoice", "liability_cap_conflicts_with_breach"],
|
|
expected_absent_relationship_types=["liability_cap_conflicts_with_confidentiality", "liability_cap_conflicts_with_ip"],
|
|
expected_relationship_confidence={"liability_cap_conflicts_with_breach": "weak"},
|
|
expected_ip_owner="party_a",
|
|
expected_liability_carve_outs=["confidentiality", "intellectual_property", "intentional_misconduct", "gross_negligence"],
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-008",
|
|
label="弱提示付款依赖样本",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-weak-acceptance.txt",
|
|
content="\n".join(
|
|
[
|
|
"技术服务合同",
|
|
"甲方：甲方科技有限公司",
|
|
"乙方：乙方顾问有限公司",
|
|
"合同金额：人民币300000元",
|
|
"第一条 付款条款",
|
|
"甲方应在验收合格后尽快支付尾款100%。",
|
|
"第二条 服务内容",
|
|
"乙方负责提供咨询与优化建议。",
|
|
]
|
|
),
|
|
expected_contract_type="service",
|
|
expected_clause_types=["payment"],
|
|
expected_missing_sections=["验收标准或验收流程", "保密范围与期限", "违约责任安排", "争议解决条款"],
|
|
expected_amount_contains="300000",
|
|
expected_payment_milestones=1,
|
|
expected_relationship_types=["payment_depends_on_acceptance"],
|
|
expected_relationship_confidence={"payment_depends_on_acceptance": "weak"},
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-009",
|
|
label="软件许可协议归属与许可关系",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-license.txt",
|
|
content="\n".join(
|
|
[
|
|
"软件许可协议",
|
|
"甲方：甲方制造有限公司",
|
|
"乙方：乙方软件有限公司",
|
|
"合同金额：人民币420000元",
|
|
"服务期限：自2026年9月1日至2027年8月31日",
|
|
"第一条 付款条款",
|
|
"甲方应在合同签署后5个工作日内支付首期许可费50%；收到发票后10个工作日内支付剩余许可费50%。",
|
|
"第二条 知识产权",
|
|
"软件著作权、源代码及相关知识产权归乙方所有，甲方在许可期内享有非独占使用权。",
|
|
"第三条 责任限制",
|
|
"任何一方的赔偿责任以合同总金额为上限，但故意、重大过失和知识产权侵权除外。",
|
|
]
|
|
),
|
|
expected_contract_type="license",
|
|
expected_clause_types=["payment", "intellectual_property", "liability"],
|
|
expected_missing_sections=["验收标准或验收流程", "保密范围与期限", "争议解决条款"],
|
|
expected_amount_contains="420000",
|
|
expected_term_start="2026-09-01",
|
|
expected_term_end="2027-08-31",
|
|
expected_payment_milestones=2,
|
|
expected_relationship_types=["payment_depends_on_invoice"],
|
|
expected_absent_relationship_types=["liability_cap_conflicts_with_ip"],
|
|
expected_ip_owner="party_b",
|
|
expected_liability_carve_outs=["intentional_misconduct", "gross_negligence", "intellectual_property"],
|
|
),
|
|
StructuredReviewCase(
|
|
case_id="S-010",
|
|
label="委托服务合同多节点验收关系",
|
|
contract_type=None,
|
|
review_as="party_a",
|
|
file_name="structured-entrustment.txt",
|
|
content="\n".join(
|
|
[
|
|
"委托服务合同",
|
|
"甲方：甲方运营有限公司",
|
|
"乙方：乙方咨询有限公司",
|
|
"合同金额：人民币560000元",
|
|
"合同期限：自2026年10月1日至2027年3月31日",
|
|
"第一条 付款条款",
|
|
"甲方应在合同签署后5个工作日内支付预付款30%；阶段成果验收合格后7个工作日内支付进度款40%；项目最终验收合格并收到发票后10个工作日内支付尾款30%。",
|
|
"第二条 验收条款",
|
|
"乙方提交阶段成果和最终成果后，应由甲方书面验收确认。",
|
|
"第三条 保密条款",
|
|
"乙方应对项目资料、商业秘密和客户信息承担保密义务。",
|
|
"第四条 争议解决",
|
|
"双方同意提交甲方所在地人民法院管辖。",
|
|
]
|
|
),
|
|
expected_contract_type="entrustment",
|
|
expected_clause_types=["payment", "acceptance", "confidentiality", "dispute_resolution"],
|
|
expected_missing_sections=["违约责任安排"],
|
|
expected_amount_contains="560000",
|
|
expected_term_start="2026-10-01",
|
|
expected_term_end="2027-03-31",
|
|
expected_payment_milestones=3,
|
|
expected_relationship_types=["payment_depends_on_acceptance", "payment_depends_on_invoice"],
|
|
expected_relationship_confidence={"payment_depends_on_acceptance": "high"},
|
|
),
|
|
]
|
|
|
|
SAMPLE_MANIFEST_PATH = Path(__file__).resolve().parents[2] / "project-docs" / "structured-review-samples" / "manifest.json"
|
|
BASELINE_ROOT = SAMPLE_MANIFEST_PATH.parent / "baselines"
|
|
BASELINE_SNAPSHOTS_DIR = BASELINE_ROOT / "snapshots"
|
|
BASELINE_HISTORY_JSONL = BASELINE_ROOT / "review-structured-baseline-history.jsonl"
|
|
|
|
|
|
def load_cases_from_manifest(manifest_path: Path) -> List[StructuredReviewCase]:
|
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
manifest_dir = manifest_path.parent
|
|
cases: List[StructuredReviewCase] = []
|
|
for entry in payload.get("samples", []):
|
|
content_file = entry.pop("content_file")
|
|
content = (manifest_dir / content_file).read_text(encoding="utf-8")
|
|
cases.append(StructuredReviewCase(content=content, **entry))
|
|
return cases
|
|
|
|
|
|
CASES = load_cases_from_manifest(SAMPLE_MANIFEST_PATH)
|
|
|
|
|
|
def register_and_login(client: httpx.Client, api_base: str, password: str) -> Dict[str, str]:
|
|
seed = int(time.time() * 1000)
|
|
username = f"structured.gate.{seed}@example.com"
|
|
display_name = f"Structured Gate {seed}"
|
|
register = client.post(
|
|
f"{api_base}/auth/register",
|
|
json={"username": username, "password": password, "display_name": display_name},
|
|
)
|
|
register.raise_for_status()
|
|
login = client.post(
|
|
f"{api_base}/auth/login",
|
|
json={"account": username, "password": password},
|
|
)
|
|
login.raise_for_status()
|
|
login_data = login.json()["data"]
|
|
return {"username": username, "token": login_data["access_token"]}
|
|
|
|
|
|
def create_review(client: httpx.Client, api_base: str, token: str, case: StructuredReviewCase) -> Dict:
|
|
files = {
|
|
"file": (case.file_name, case.content.encode("utf-8"), "text/plain"),
|
|
}
|
|
data = {
|
|
"review_as": case.review_as,
|
|
}
|
|
if case.contract_type:
|
|
data["contract_type"] = case.contract_type
|
|
response = client.post(
|
|
f"{api_base}/reviews",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
data=data,
|
|
files=files,
|
|
)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
|
|
def fetch_review_result(client: httpx.Client, api_base: str, token: str, review_id: str) -> Dict:
|
|
response = client.get(
|
|
f"{api_base}/reviews/{review_id}/result",
|
|
headers={"Authorization": f"Bearer {token}"},
|
|
)
|
|
response.raise_for_status()
|
|
return response.json()["data"]
|
|
|
|
|
|
def validate_case(case: StructuredReviewCase, result: Dict) -> List[str]:
|
|
problems: List[str] = []
|
|
contract_structure = result.get("contract_structure") or {}
|
|
clauses = result.get("clauses") or []
|
|
risk_items = result.get("risk_items") or []
|
|
confidence_summary = result.get("confidence_summary") or {}
|
|
structured_analysis = confidence_summary.get("structured_analysis") or {}
|
|
|
|
if contract_structure.get("detected_contract_type") != case.expected_contract_type:
|
|
problems.append(
|
|
f"detected_contract_type={contract_structure.get('detected_contract_type')} expected={case.expected_contract_type}"
|
|
)
|
|
|
|
amount_text = contract_structure.get("amount") or ""
|
|
if case.expected_amount_contains and case.expected_amount_contains not in amount_text:
|
|
problems.append(f"amount missing expected fragment: {case.expected_amount_contains}")
|
|
|
|
term_details = contract_structure.get("term_details") or {}
|
|
if case.expected_term_start and term_details.get("start_date") != case.expected_term_start:
|
|
problems.append(f"term start={term_details.get('start_date')} expected={case.expected_term_start}")
|
|
if case.expected_term_end and term_details.get("end_date") != case.expected_term_end:
|
|
problems.append(f"term end={term_details.get('end_date')} expected={case.expected_term_end}")
|
|
|
|
clause_types = {item.get("clause_type") for item in clauses if item.get("clause_type")}
|
|
for expected_type in case.expected_clause_types:
|
|
if expected_type not in clause_types:
|
|
problems.append(f"missing clause type: {expected_type}")
|
|
|
|
missing_sections = set(contract_structure.get("missing_sections") or [])
|
|
for expected_section in case.expected_missing_sections:
|
|
if expected_section not in missing_sections:
|
|
problems.append(f"missing section hint not found: {expected_section}")
|
|
|
|
if contract_structure.get("clause_count", 0) <= 0:
|
|
problems.append("contract_structure.clause_count <= 0")
|
|
if not contract_structure.get("summary"):
|
|
problems.append("missing contract_structure.summary")
|
|
|
|
if not structured_analysis.get("stage"):
|
|
problems.append("missing confidence_summary.structured_analysis.stage")
|
|
if case.expected_payment_milestones and structured_analysis.get("payment_milestone_count", 0) < case.expected_payment_milestones:
|
|
problems.append(
|
|
f"payment_milestone_count={structured_analysis.get('payment_milestone_count')} expected>={case.expected_payment_milestones}"
|
|
)
|
|
|
|
relationships = contract_structure.get("clause_relationships") or []
|
|
relationship_types = {item.get("type") for item in relationships if item.get("type")}
|
|
for expected_type in case.expected_relationship_types or []:
|
|
if expected_type not in relationship_types:
|
|
problems.append(f"missing relationship type: {expected_type}")
|
|
for absent_type in case.expected_absent_relationship_types or []:
|
|
if absent_type in relationship_types:
|
|
problems.append(f"unexpected relationship type: {absent_type}")
|
|
|
|
confidence_map = {item.get("type"): item.get("confidence_tier") for item in relationships if item.get("type")}
|
|
for relationship_type, expected_confidence in (case.expected_relationship_confidence or {}).items():
|
|
if confidence_map.get(relationship_type) != expected_confidence:
|
|
problems.append(
|
|
f"relationship {relationship_type} confidence={confidence_map.get(relationship_type)} expected={expected_confidence}"
|
|
)
|
|
|
|
if case.expected_ip_owner and contract_structure.get("intellectual_property_owner") != case.expected_ip_owner:
|
|
problems.append(
|
|
f"intellectual_property_owner={contract_structure.get('intellectual_property_owner')} expected={case.expected_ip_owner}"
|
|
)
|
|
|
|
carve_outs = set(contract_structure.get("liability_carve_outs") or [])
|
|
for expected_carve_out in case.expected_liability_carve_outs or []:
|
|
if expected_carve_out not in carve_outs:
|
|
problems.append(f"missing liability carve-out: {expected_carve_out}")
|
|
|
|
if risk_items:
|
|
first_risk = risk_items[0]
|
|
evidence = first_risk.get("evidence") or {}
|
|
if not evidence.get("quote"):
|
|
problems.append("first risk item missing evidence.quote")
|
|
if not first_risk.get("certainty"):
|
|
problems.append("first risk item missing certainty")
|
|
|
|
return problems
|
|
|
|
|
|
def build_field_baseline(summary: Dict) -> List[Dict[str, object]]:
|
|
rows: List[Dict[str, object]] = []
|
|
|
|
def append_metric(metric: str, passed: int, total: int):
|
|
rows.append(
|
|
{
|
|
"metric": metric,
|
|
"passed": passed,
|
|
"total": total,
|
|
"accuracy": round((passed / total) * 100, 2) if total else 0.0,
|
|
}
|
|
)
|
|
|
|
contract_type_pass = 0
|
|
contract_type_total = 0
|
|
amount_pass = 0
|
|
amount_total = 0
|
|
term_start_pass = 0
|
|
term_start_total = 0
|
|
term_end_pass = 0
|
|
term_end_total = 0
|
|
payment_milestone_pass = 0
|
|
payment_milestone_total = 0
|
|
ip_owner_pass = 0
|
|
ip_owner_total = 0
|
|
liability_carve_out_pass = 0
|
|
liability_carve_out_total = 0
|
|
clause_type_pass = 0
|
|
clause_type_total = 0
|
|
|
|
for item in summary["cases"]:
|
|
if item.get("status") == "FAIL" and "result" not in item:
|
|
continue
|
|
result = item.get("result") or {}
|
|
contract_structure = result.get("contract_structure") or {}
|
|
case = next(case for case in CASES if case.case_id == item["case_id"])
|
|
|
|
contract_type_total += 1
|
|
if contract_structure.get("detected_contract_type") == case.expected_contract_type:
|
|
contract_type_pass += 1
|
|
|
|
if case.expected_amount_contains:
|
|
amount_total += 1
|
|
if case.expected_amount_contains in (contract_structure.get("amount") or ""):
|
|
amount_pass += 1
|
|
|
|
if case.expected_term_start:
|
|
term_start_total += 1
|
|
if (contract_structure.get("term_details") or {}).get("start_date") == case.expected_term_start:
|
|
term_start_pass += 1
|
|
|
|
if case.expected_term_end:
|
|
term_end_total += 1
|
|
if (contract_structure.get("term_details") or {}).get("end_date") == case.expected_term_end:
|
|
term_end_pass += 1
|
|
|
|
if case.expected_payment_milestones:
|
|
payment_milestone_total += 1
|
|
if len(contract_structure.get("payment_milestones") or []) >= case.expected_payment_milestones:
|
|
payment_milestone_pass += 1
|
|
|
|
if case.expected_ip_owner:
|
|
ip_owner_total += 1
|
|
if contract_structure.get("intellectual_property_owner") == case.expected_ip_owner:
|
|
ip_owner_pass += 1
|
|
|
|
for expected_type in case.expected_clause_types:
|
|
clause_type_total += 1
|
|
if expected_type in {entry.get("clause_type") for entry in result.get("clauses") or [] if entry.get("clause_type")}:
|
|
clause_type_pass += 1
|
|
|
|
for expected_carve_out in case.expected_liability_carve_outs or []:
|
|
liability_carve_out_total += 1
|
|
if expected_carve_out in set(contract_structure.get("liability_carve_outs") or []):
|
|
liability_carve_out_pass += 1
|
|
|
|
append_metric("contract_type", contract_type_pass, contract_type_total)
|
|
append_metric("amount", amount_pass, amount_total)
|
|
append_metric("term_start", term_start_pass, term_start_total)
|
|
append_metric("term_end", term_end_pass, term_end_total)
|
|
append_metric("payment_milestones", payment_milestone_pass, payment_milestone_total)
|
|
append_metric("clause_types", clause_type_pass, clause_type_total)
|
|
append_metric("intellectual_property_owner", ip_owner_pass, ip_owner_total)
|
|
append_metric("liability_carve_outs", liability_carve_out_pass, liability_carve_out_total)
|
|
return rows
|
|
|
|
|
|
def build_relationship_baseline(summary: Dict) -> List[Dict[str, object]]:
|
|
rows: List[Dict[str, object]] = []
|
|
presence_pass = 0
|
|
presence_total = 0
|
|
absence_pass = 0
|
|
absence_total = 0
|
|
confidence_pass = 0
|
|
confidence_total = 0
|
|
high_tier_total = 0
|
|
weak_tier_total = 0
|
|
|
|
for item in summary["cases"]:
|
|
result = item.get("result") or {}
|
|
relationships = result.get("contract_structure", {}).get("clause_relationships") or []
|
|
relationship_types = {entry.get("type") for entry in relationships if entry.get("type")}
|
|
confidence_map = {entry.get("type"): entry.get("confidence_tier") for entry in relationships if entry.get("type")}
|
|
case = next(case for case in CASES if case.case_id == item["case_id"])
|
|
|
|
for expected_type in case.expected_relationship_types or []:
|
|
presence_total += 1
|
|
if expected_type in relationship_types:
|
|
presence_pass += 1
|
|
|
|
for absent_type in case.expected_absent_relationship_types or []:
|
|
absence_total += 1
|
|
if absent_type not in relationship_types:
|
|
absence_pass += 1
|
|
|
|
for relationship_type, expected_confidence in (case.expected_relationship_confidence or {}).items():
|
|
confidence_total += 1
|
|
if expected_confidence == "high":
|
|
high_tier_total += 1
|
|
if expected_confidence == "weak":
|
|
weak_tier_total += 1
|
|
if confidence_map.get(relationship_type) == expected_confidence:
|
|
confidence_pass += 1
|
|
|
|
rows.append({"metric": "relationship_presence", "passed": presence_pass, "total": presence_total, "accuracy": round((presence_pass / presence_total) * 100, 2) if presence_total else 0.0})
|
|
rows.append({"metric": "relationship_absence", "passed": absence_pass, "total": absence_total, "accuracy": round((absence_pass / absence_total) * 100, 2) if absence_total else 0.0})
|
|
rows.append({"metric": "relationship_confidence", "passed": confidence_pass, "total": confidence_total, "accuracy": round((confidence_pass / confidence_total) * 100, 2) if confidence_total else 0.0})
|
|
rows.append({"metric": "relationship_high_confidence_expected", "passed": high_tier_total, "total": high_tier_total, "accuracy": 100.0 if high_tier_total else 0.0})
|
|
rows.append({"metric": "relationship_weak_hint_expected", "passed": weak_tier_total, "total": weak_tier_total, "accuracy": 100.0 if weak_tier_total else 0.0})
|
|
return rows
|
|
|
|
|
|
def build_field_baseline_by_contract(summary: Dict) -> List[Dict[str, object]]:
|
|
rows: List[Dict[str, object]] = []
|
|
grouped: Dict[str, Dict[str, int]] = {}
|
|
for item in summary["cases"]:
|
|
result = item.get("result") or {}
|
|
contract_structure = result.get("contract_structure") or {}
|
|
case = next(case for case in CASES if case.case_id == item["case_id"])
|
|
family = case.expected_contract_type
|
|
bucket = grouped.setdefault(family, {"contract_type_pass": 0, "contract_type_total": 0, "payment_pass": 0, "payment_total": 0, "relationship_pass": 0, "relationship_total": 0})
|
|
bucket["contract_type_total"] += 1
|
|
if contract_structure.get("detected_contract_type") == case.expected_contract_type:
|
|
bucket["contract_type_pass"] += 1
|
|
if case.expected_payment_milestones:
|
|
bucket["payment_total"] += 1
|
|
if len(contract_structure.get("payment_milestones") or []) >= case.expected_payment_milestones:
|
|
bucket["payment_pass"] += 1
|
|
if case.expected_relationship_types:
|
|
bucket["relationship_total"] += len(case.expected_relationship_types)
|
|
relationship_types = {entry.get("type") for entry in contract_structure.get("clause_relationships") or [] if entry.get("type")}
|
|
bucket["relationship_pass"] += sum(1 for relationship_type in case.expected_relationship_types if relationship_type in relationship_types)
|
|
for family, bucket in grouped.items():
|
|
rows.append(
|
|
{
|
|
"contract_family": family,
|
|
"contract_type_passed": bucket["contract_type_pass"],
|
|
"contract_type_total": bucket["contract_type_total"],
|
|
"contract_type_accuracy": round((bucket["contract_type_pass"] / bucket["contract_type_total"]) * 100, 2) if bucket["contract_type_total"] else 0.0,
|
|
"payment_milestone_passed": bucket["payment_pass"],
|
|
"payment_milestone_total": bucket["payment_total"],
|
|
"payment_milestone_accuracy": round((bucket["payment_pass"] / bucket["payment_total"]) * 100, 2) if bucket["payment_total"] else 0.0,
|
|
"relationship_presence_passed": bucket["relationship_pass"],
|
|
"relationship_presence_total": bucket["relationship_total"],
|
|
"relationship_presence_accuracy": round((bucket["relationship_pass"] / bucket["relationship_total"]) * 100, 2) if bucket["relationship_total"] else 0.0,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def build_field_baseline_by_batch(summary: Dict) -> List[Dict[str, object]]:
|
|
rows: List[Dict[str, object]] = []
|
|
grouped: Dict[str, Dict[str, int]] = {}
|
|
for item in summary["cases"]:
|
|
result = item.get("result") or {}
|
|
contract_structure = result.get("contract_structure") or {}
|
|
case = next(case for case in CASES if case.case_id == item["case_id"])
|
|
bucket = grouped.setdefault(case.batch_id, {"contract_type_pass": 0, "contract_type_total": 0, "payment_pass": 0, "payment_total": 0})
|
|
bucket["contract_type_total"] += 1
|
|
if contract_structure.get("detected_contract_type") == case.expected_contract_type:
|
|
bucket["contract_type_pass"] += 1
|
|
if case.expected_payment_milestones:
|
|
bucket["payment_total"] += 1
|
|
if len(contract_structure.get("payment_milestones") or []) >= case.expected_payment_milestones:
|
|
bucket["payment_pass"] += 1
|
|
for batch_id, bucket in grouped.items():
|
|
rows.append(
|
|
{
|
|
"batch_id": batch_id,
|
|
"contract_type_passed": bucket["contract_type_pass"],
|
|
"contract_type_total": bucket["contract_type_total"],
|
|
"contract_type_accuracy": round((bucket["contract_type_pass"] / bucket["contract_type_total"]) * 100, 2) if bucket["contract_type_total"] else 0.0,
|
|
"payment_milestone_passed": bucket["payment_pass"],
|
|
"payment_milestone_total": bucket["payment_total"],
|
|
"payment_milestone_accuracy": round((bucket["payment_pass"] / bucket["payment_total"]) * 100, 2) if bucket["payment_total"] else 0.0,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def build_relationship_baseline_by_type(summary: Dict) -> List[Dict[str, object]]:
|
|
rows: List[Dict[str, object]] = []
|
|
grouped: Dict[str, Dict[str, int]] = {}
|
|
for item in summary["cases"]:
|
|
result = item.get("result") or {}
|
|
relationships = result.get("contract_structure", {}).get("clause_relationships") or []
|
|
relationship_types = {entry.get("type") for entry in relationships if entry.get("type")}
|
|
confidence_map = {entry.get("type"): entry.get("confidence_tier") for entry in relationships if entry.get("type")}
|
|
case = next(case for case in CASES if case.case_id == item["case_id"])
|
|
for relationship_type in case.expected_relationship_types or []:
|
|
bucket = grouped.setdefault(relationship_type, {"present_pass": 0, "present_total": 0, "confidence_pass": 0, "confidence_total": 0})
|
|
bucket["present_total"] += 1
|
|
if relationship_type in relationship_types:
|
|
bucket["present_pass"] += 1
|
|
expected_confidence = (case.expected_relationship_confidence or {}).get(relationship_type)
|
|
if expected_confidence:
|
|
bucket["confidence_total"] += 1
|
|
if confidence_map.get(relationship_type) == expected_confidence:
|
|
bucket["confidence_pass"] += 1
|
|
for relationship_type, bucket in grouped.items():
|
|
rows.append(
|
|
{
|
|
"relationship_type": relationship_type,
|
|
"presence_passed": bucket["present_pass"],
|
|
"presence_total": bucket["present_total"],
|
|
"presence_accuracy": round((bucket["present_pass"] / bucket["present_total"]) * 100, 2) if bucket["present_total"] else 0.0,
|
|
"confidence_passed": bucket["confidence_pass"],
|
|
"confidence_total": bucket["confidence_total"],
|
|
"confidence_accuracy": round((bucket["confidence_pass"] / bucket["confidence_total"]) * 100, 2) if bucket["confidence_total"] else 0.0,
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def build_sample_inventory() -> List[Dict[str, object]]:
|
|
return [
|
|
{
|
|
"case_id": case.case_id,
|
|
"batch_id": case.batch_id,
|
|
"label": case.label,
|
|
"contract_family": case.expected_contract_type,
|
|
"source_origin": case.source_origin,
|
|
"redaction_level": case.redaction_level,
|
|
"expected_relationships": len(case.expected_relationship_types or []),
|
|
"expected_payment_milestones": case.expected_payment_milestones,
|
|
}
|
|
for case in CASES
|
|
]
|
|
|
|
|
|
def build_md(summary: Dict) -> str:
|
|
lines = [
|
|
"# Review Structured Acceptance Gate",
|
|
"",
|
|
f"- Status: {summary['status']}",
|
|
f"- Generated at: {summary['generated_at']}",
|
|
f"- API Base: `{summary['api_base']}`",
|
|
f"- Artifact dir: `{summary['artifact_dir']}`",
|
|
"",
|
|
"| Case | Status | Detail |",
|
|
"|---|---|---|",
|
|
]
|
|
for case in summary["cases"]:
|
|
lines.append(f"| {case['case_id']} | {case['status']} | {case['detail'].replace('|', '/')} |")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def build_baseline_md(field_rows: List[Dict[str, object]], relationship_rows: List[Dict[str, object]]) -> str:
|
|
lines = [
|
|
"# Structured Accuracy Baseline",
|
|
"",
|
|
"## Field Accuracy",
|
|
"",
|
|
"| Metric | Passed | Total | Accuracy |",
|
|
"|---|---:|---:|---:|",
|
|
]
|
|
for row in field_rows:
|
|
lines.append(f"| {row['metric']} | {row['passed']} | {row['total']} | {row['accuracy']}% |")
|
|
lines.extend(
|
|
[
|
|
"",
|
|
"## Relationship Accuracy",
|
|
"",
|
|
"| Metric | Passed | Total | Accuracy |",
|
|
"|---|---:|---:|---:|",
|
|
]
|
|
)
|
|
for row in relationship_rows:
|
|
lines.append(f"| {row['metric']} | {row['passed']} | {row['total']} | {row['accuracy']}% |")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def build_dashboard_html(
|
|
summary: Dict[str, object],
|
|
field_rows: List[Dict[str, object]],
|
|
relationship_rows: List[Dict[str, object]],
|
|
contract_rows: List[Dict[str, object]],
|
|
batch_rows: List[Dict[str, object]],
|
|
relationship_type_rows: List[Dict[str, object]],
|
|
) -> str:
|
|
data = {
|
|
"generated_at": summary["generated_at"],
|
|
"sample_count": len(summary.get("sample_inventory") or []),
|
|
"field_rows": field_rows,
|
|
"relationship_rows": relationship_rows,
|
|
"contract_rows": contract_rows,
|
|
"batch_rows": batch_rows,
|
|
"relationship_type_rows": relationship_type_rows,
|
|
}
|
|
return f"""<!doctype html>
|
|
<html lang="zh-CN">
|
|
<head>
|
|
<meta charset="utf-8" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
|
<title>Structured Review Baseline Dashboard</title>
|
|
<style>
|
|
body {{ font-family: Arial, sans-serif; margin: 0; padding: 24px; background: #f5f7fb; color: #111827; }}
|
|
h1, h2 {{ margin: 0 0 12px; }}
|
|
p {{ margin: 0 0 8px; color: #475569; }}
|
|
.grid {{ display: grid; gap: 16px; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); margin-bottom: 24px; }}
|
|
.card {{ background: #fff; border: 1px solid #dbe3f0; border-radius: 12px; padding: 16px; box-shadow: 0 8px 24px rgba(15, 23, 42, 0.06); }}
|
|
.metric {{ font-size: 28px; font-weight: 700; margin-top: 8px; }}
|
|
table {{ width: 100%; border-collapse: collapse; background: #fff; border-radius: 12px; overflow: hidden; }}
|
|
th, td {{ padding: 10px 12px; border-bottom: 1px solid #e5e7eb; text-align: left; font-size: 14px; }}
|
|
th {{ background: #f8fafc; }}
|
|
.section {{ margin-bottom: 24px; }}
|
|
.bar-wrap {{ background: #eef2ff; border-radius: 999px; height: 10px; overflow: hidden; }}
|
|
.bar {{ height: 10px; background: linear-gradient(90deg, #2563eb, #22c55e); border-radius: 999px; }}
|
|
.muted {{ color: #64748b; font-size: 12px; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Structured Review Baseline Dashboard</h1>
|
|
<p>Latest snapshot: {summary["generated_at"]}</p>
|
|
<div class="grid">
|
|
<div class="card"><div class="muted">Samples</div><div class="metric">{len(summary.get("sample_inventory") or [])}</div></div>
|
|
<div class="card"><div class="muted">Field Metrics</div><div class="metric">{len(field_rows)}</div></div>
|
|
<div class="card"><div class="muted">Relationship Metrics</div><div class="metric">{len(relationship_rows)}</div></div>
|
|
<div class="card"><div class="muted">Contract Families</div><div class="metric">{len(contract_rows)}</div></div>
|
|
</div>
|
|
<div id="app" data-json='{json.dumps(data, ensure_ascii=False).replace("'", "'")}'></div>
|
|
<script>
|
|
const host = document.getElementById('app');
|
|
const data = JSON.parse(host.dataset.json);
|
|
function table(title, keyField, rows, metrics) {{
|
|
let html = `<div class="section"><h2>${{title}}</h2><table><thead><tr><th>${{keyField}}</th>`;
|
|
for (const metric of metrics) html += `<th>${{metric}}</th><th>Trend</th>`;
|
|
html += `</tr></thead><tbody>`;
|
|
for (const row of rows) {{
|
|
html += `<tr><td>${{row[keyField] ?? row.metric}}</td>`;
|
|
for (const metric of metrics) {{
|
|
const value = row[metric];
|
|
const numeric = typeof value === 'number' ? value : parseFloat(value);
|
|
const safe = Number.isFinite(numeric) ? numeric : 0;
|
|
html += `<td>${{value}}</td><td><div class="bar-wrap"><div class="bar" style="width:${{Math.max(0, Math.min(100, safe))}}%"></div></div></td>`;
|
|
}}
|
|
html += `</tr>`;
|
|
}}
|
|
html += `</tbody></table></div>`;
|
|
return html;
|
|
}}
|
|
host.innerHTML =
|
|
table('Field Accuracy', 'metric', data.field_rows, ['accuracy']) +
|
|
table('Relationship Accuracy', 'metric', data.relationship_rows, ['accuracy']) +
|
|
table('By Contract Family', 'contract_family', data.contract_rows, ['contract_type_accuracy','payment_milestone_accuracy','relationship_presence_accuracy']) +
|
|
table('By Batch', 'batch_id', data.batch_rows, ['contract_type_accuracy','payment_milestone_accuracy']) +
|
|
table('By Relationship Type', 'relationship_type', data.relationship_type_rows, ['presence_accuracy','confidence_accuracy']);
|
|
</script>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def write_csv(path: Path, rows: List[Dict[str, object]]) -> None:
|
|
if not rows:
|
|
path.write_text("", encoding="utf-8")
|
|
return
|
|
headers = list(rows[0].keys())
|
|
lines = [",".join(headers)]
|
|
for row in rows:
|
|
values = []
|
|
for header in headers:
|
|
value = row.get(header, "")
|
|
text = str(value).replace('"', '""')
|
|
if "," in text or '"' in text or "\n" in text:
|
|
text = f'"{text}"'
|
|
values.append(text)
|
|
lines.append(",".join(values))
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def index_rows(rows: List[Dict[str, object]], key_field: str) -> Dict[str, Dict[str, object]]:
|
|
return {str(row[key_field]): row for row in rows if key_field in row}
|
|
|
|
|
|
def build_trend_rows(
|
|
current_rows: List[Dict[str, object]],
|
|
previous_rows: List[Dict[str, object]],
|
|
key_field: str,
|
|
metric_fields: List[str],
|
|
) -> List[Dict[str, object]]:
|
|
previous_index = index_rows(previous_rows, key_field)
|
|
trend_rows: List[Dict[str, object]] = []
|
|
for row in current_rows:
|
|
key = str(row[key_field])
|
|
previous_row = previous_index.get(key, {})
|
|
trend_row: Dict[str, object] = {key_field: key}
|
|
for metric_field in metric_fields:
|
|
current_value = row.get(metric_field)
|
|
previous_value = previous_row.get(metric_field)
|
|
trend_row[f"current_{metric_field}"] = current_value
|
|
trend_row[f"previous_{metric_field}"] = previous_value if previous_value is not None else "N/A"
|
|
if isinstance(current_value, (int, float)) and isinstance(previous_value, (int, float)):
|
|
trend_row[f"delta_{metric_field}"] = round(float(current_value) - float(previous_value), 2)
|
|
else:
|
|
trend_row[f"delta_{metric_field}"] = "N/A"
|
|
trend_rows.append(trend_row)
|
|
return trend_rows
|
|
|
|
|
|
def load_previous_snapshot(baseline_root: Path = BASELINE_ROOT) -> Optional[Dict[str, object]]:
|
|
latest_path = baseline_root / "latest-summary.json"
|
|
if not latest_path.exists():
|
|
return None
|
|
return json.loads(latest_path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def persist_baseline_history(summary: Dict[str, object], baseline_root: Path = BASELINE_ROOT) -> None:
|
|
snapshots_dir = baseline_root / "snapshots"
|
|
history_jsonl = baseline_root / "review-structured-baseline-history.jsonl"
|
|
baseline_root.mkdir(parents=True, exist_ok=True)
|
|
snapshots_dir.mkdir(parents=True, exist_ok=True)
|
|
previous_snapshot = load_previous_snapshot(baseline_root)
|
|
generated_at = str(summary["generated_at"])
|
|
safe_stamp = generated_at.replace(":", "-").replace(".", "-")
|
|
|
|
snapshot_payload = {
|
|
"generated_at": generated_at,
|
|
"artifact_dir": summary["artifact_dir"],
|
|
"sample_count": len(summary.get("sample_inventory") or []),
|
|
"field_baseline": summary.get("field_baseline") or [],
|
|
"relationship_baseline": summary.get("relationship_baseline") or [],
|
|
"field_baseline_by_contract": summary.get("field_baseline_by_contract") or [],
|
|
"field_baseline_by_batch": summary.get("field_baseline_by_batch") or [],
|
|
"relationship_baseline_by_type": summary.get("relationship_baseline_by_type") or [],
|
|
"sample_inventory": summary.get("sample_inventory") or [],
|
|
}
|
|
|
|
snapshot_path = snapshots_dir / f"{safe_stamp}-review-structured-baseline-snapshot.json"
|
|
snapshot_path.write_text(json.dumps(snapshot_payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
history_row = {
|
|
"generated_at": generated_at,
|
|
"artifact_dir": summary["artifact_dir"],
|
|
"snapshot_path": str(snapshot_path),
|
|
"sample_count": len(summary.get("sample_inventory") or []),
|
|
"field_metric_count": len(summary.get("field_baseline") or []),
|
|
"relationship_metric_count": len(summary.get("relationship_baseline") or []),
|
|
}
|
|
with history_jsonl.open("a", encoding="utf-8") as fh:
|
|
fh.write(json.dumps(history_row, ensure_ascii=False) + "\n")
|
|
|
|
latest_path = baseline_root / "latest-summary.json"
|
|
latest_path.write_text(json.dumps(snapshot_payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
write_csv(baseline_root / "latest-field-baseline.csv", summary.get("field_baseline") or [])
|
|
write_csv(baseline_root / "latest-relationship-baseline.csv", summary.get("relationship_baseline") or [])
|
|
write_csv(baseline_root / "latest-field-baseline-by-contract.csv", summary.get("field_baseline_by_contract") or [])
|
|
write_csv(baseline_root / "latest-field-baseline-by-batch.csv", summary.get("field_baseline_by_batch") or [])
|
|
write_csv(baseline_root / "latest-relationship-baseline-by-type.csv", summary.get("relationship_baseline_by_type") or [])
|
|
write_csv(baseline_root / "latest-sample-inventory.csv", summary.get("sample_inventory") or [])
|
|
|
|
field_trend_rows = build_trend_rows(
|
|
summary.get("field_baseline") or [],
|
|
(previous_snapshot or {}).get("field_baseline") or [],
|
|
"metric",
|
|
["accuracy"],
|
|
)
|
|
relationship_trend_rows = build_trend_rows(
|
|
summary.get("relationship_baseline") or [],
|
|
(previous_snapshot or {}).get("relationship_baseline") or [],
|
|
"metric",
|
|
["accuracy"],
|
|
)
|
|
contract_trend_rows = build_trend_rows(
|
|
summary.get("field_baseline_by_contract") or [],
|
|
(previous_snapshot or {}).get("field_baseline_by_contract") or [],
|
|
"contract_family",
|
|
["contract_type_accuracy", "payment_milestone_accuracy", "relationship_presence_accuracy"],
|
|
)
|
|
batch_trend_rows = build_trend_rows(
|
|
summary.get("field_baseline_by_batch") or [],
|
|
(previous_snapshot or {}).get("field_baseline_by_batch") or [],
|
|
"batch_id",
|
|
["contract_type_accuracy", "payment_milestone_accuracy"],
|
|
)
|
|
relationship_type_trend_rows = build_trend_rows(
|
|
summary.get("relationship_baseline_by_type") or [],
|
|
(previous_snapshot or {}).get("relationship_baseline_by_type") or [],
|
|
"relationship_type",
|
|
["presence_accuracy", "confidence_accuracy"],
|
|
)
|
|
|
|
write_csv(baseline_root / "baseline-trend-overall.csv", field_trend_rows)
|
|
write_csv(baseline_root / "baseline-trend-relationship-overall.csv", relationship_trend_rows)
|
|
write_csv(baseline_root / "baseline-trend-by-contract.csv", contract_trend_rows)
|
|
write_csv(baseline_root / "baseline-trend-by-batch.csv", batch_trend_rows)
|
|
write_csv(baseline_root / "baseline-trend-by-relationship-type.csv", relationship_type_trend_rows)
|
|
dashboard_html = build_dashboard_html(
|
|
summary,
|
|
summary.get("field_baseline") or [],
|
|
summary.get("relationship_baseline") or [],
|
|
summary.get("field_baseline_by_contract") or [],
|
|
summary.get("field_baseline_by_batch") or [],
|
|
summary.get("relationship_baseline_by_type") or [],
|
|
)
|
|
(baseline_root / "baseline-trend-dashboard.html").write_text(dashboard_html, encoding="utf-8")
|
|
|
|
trend_summary_md = baseline_root / "baseline-trend-summary.md"
|
|
lines = [
|
|
"# Structured Baseline Trend Summary",
|
|
"",
|
|
f"- Latest snapshot: `{generated_at}`",
|
|
f"- Sample count: `{len(summary.get('sample_inventory') or [])}`",
|
|
f"- Previous snapshot: `{(previous_snapshot or {}).get('generated_at') or 'N/A'}`",
|
|
"",
|
|
"Generated trend files:",
|
|
"",
|
|
"1. `baseline-trend-overall.csv`",
|
|
"2. `baseline-trend-relationship-overall.csv`",
|
|
"3. `baseline-trend-by-contract.csv`",
|
|
"4. `baseline-trend-by-batch.csv`",
|
|
"5. `baseline-trend-by-relationship-type.csv`",
|
|
]
|
|
trend_summary_md.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def run_gate(api_base: str, artifact_dir: Path, password: str) -> Dict:
|
|
artifact_dir.mkdir(parents=True, exist_ok=True)
|
|
client = httpx.Client(timeout=120.0)
|
|
auth = register_and_login(client, api_base, password)
|
|
summary = {
|
|
"name": "review-structured-acceptance-gate",
|
|
"status": "PASS",
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"api_base": api_base,
|
|
"artifact_dir": str(artifact_dir),
|
|
"registered_user": auth["username"],
|
|
"cases": [],
|
|
}
|
|
for case in CASES:
|
|
submit_json = create_review(client, api_base, auth["token"], case)
|
|
if submit_json.get("code") != 0:
|
|
summary["status"] = "FAIL"
|
|
summary["cases"].append(
|
|
{
|
|
"case_id": case.case_id,
|
|
"label": case.label,
|
|
"status": "FAIL",
|
|
"detail": submit_json.get("message") or "create review failed",
|
|
}
|
|
)
|
|
continue
|
|
review_id = submit_json["data"]["id"]
|
|
result = fetch_review_result(client, api_base, auth["token"], review_id)
|
|
problems = validate_case(case, result)
|
|
case_status = "PASS" if not problems else "FAIL"
|
|
if problems:
|
|
summary["status"] = "FAIL"
|
|
summary["cases"].append(
|
|
{
|
|
"case_id": case.case_id,
|
|
"label": case.label,
|
|
"status": case_status,
|
|
"detail": "; ".join(problems) if problems else f"review_id={review_id}",
|
|
"review_id": review_id,
|
|
"result": result,
|
|
}
|
|
)
|
|
field_baseline = build_field_baseline(summary)
|
|
relationship_baseline = build_relationship_baseline(summary)
|
|
field_baseline_by_contract = build_field_baseline_by_contract(summary)
|
|
field_baseline_by_batch = build_field_baseline_by_batch(summary)
|
|
relationship_baseline_by_type = build_relationship_baseline_by_type(summary)
|
|
sample_inventory = build_sample_inventory()
|
|
summary["field_baseline"] = field_baseline
|
|
summary["relationship_baseline"] = relationship_baseline
|
|
summary["field_baseline_by_contract"] = field_baseline_by_contract
|
|
summary["field_baseline_by_batch"] = field_baseline_by_batch
|
|
summary["relationship_baseline_by_type"] = relationship_baseline_by_type
|
|
summary["sample_inventory"] = sample_inventory
|
|
return summary
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Run structured review acceptance gate against the reviews API.")
|
|
parser.add_argument("--api-base", required=True)
|
|
parser.add_argument("--artifact-dir", required=True)
|
|
parser.add_argument("--password", default="123456")
|
|
parser.add_argument("--baseline-root", default=str(BASELINE_ROOT))
|
|
args = parser.parse_args()
|
|
|
|
artifact_dir = Path(args.artifact_dir)
|
|
baseline_root = Path(args.baseline_root)
|
|
summary = run_gate(api_base=args.api_base.rstrip("/"), artifact_dir=artifact_dir, password=args.password)
|
|
summary_path = artifact_dir / "review-structured-acceptance-summary.json"
|
|
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
md_path = artifact_dir / "review-structured-acceptance-summary.md"
|
|
md_path.write_text(build_md(summary), encoding="utf-8")
|
|
field_csv_path = artifact_dir / "review-structured-field-baseline.csv"
|
|
write_csv(field_csv_path, summary.get("field_baseline") or [])
|
|
relationship_csv_path = artifact_dir / "review-structured-relationship-baseline.csv"
|
|
write_csv(relationship_csv_path, summary.get("relationship_baseline") or [])
|
|
field_by_contract_csv_path = artifact_dir / "review-structured-field-baseline-by-contract.csv"
|
|
write_csv(field_by_contract_csv_path, summary.get("field_baseline_by_contract") or [])
|
|
field_by_batch_csv_path = artifact_dir / "review-structured-field-baseline-by-batch.csv"
|
|
write_csv(field_by_batch_csv_path, summary.get("field_baseline_by_batch") or [])
|
|
relationship_by_type_csv_path = artifact_dir / "review-structured-relationship-baseline-by-type.csv"
|
|
write_csv(relationship_by_type_csv_path, summary.get("relationship_baseline_by_type") or [])
|
|
sample_inventory_csv_path = artifact_dir / "review-structured-sample-inventory.csv"
|
|
write_csv(sample_inventory_csv_path, summary.get("sample_inventory") or [])
|
|
baseline_md_path = artifact_dir / "review-structured-baseline.md"
|
|
baseline_md_path.write_text(
|
|
build_baseline_md(summary.get("field_baseline") or [], summary.get("relationship_baseline") or []),
|
|
encoding="utf-8",
|
|
)
|
|
(artifact_dir / "review-structured-baseline-dashboard.html").write_text(
|
|
build_dashboard_html(
|
|
summary,
|
|
summary.get("field_baseline") or [],
|
|
summary.get("relationship_baseline") or [],
|
|
summary.get("field_baseline_by_contract") or [],
|
|
summary.get("field_baseline_by_batch") or [],
|
|
summary.get("relationship_baseline_by_type") or [],
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
persist_baseline_history(summary, baseline_root=baseline_root)
|
|
print(f"REVIEW_STRUCTURED_ACCEPTANCE_{summary['status']} {artifact_dir}")
|
|
return 0 if summary["status"] == "PASS" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|