feat: establish structured review baselines
This commit is contained in:
Codex
@@ -177,7 +177,7 @@ CONTRACT_TYPE_HINTS = {
|
||||
"nda": ["保密协议", "保密条款", "保密合同", "non-disclosure"],
|
||||
"framework": ["框架协议", "合作框架", "战略合作"],
|
||||
"sales": ["买卖合同", "销售合同", "供货合同", "购销合同"],
|
||||
"entrustment": ["委托合同", "委托服务", "代理协议", "委托协议"],
|
||||
"entrustment": ["委托合同", "委托服务", "委托服务合同", "代理协议", "委托协议"],
|
||||
"loan": ["借款合同", "借款协议", "贷款合同"],
|
||||
"license": ["许可协议", "授权协议", "知识产权许可", "软件许可"],
|
||||
}
|
||||
@@ -790,7 +790,7 @@ class ReviewService:
|
||||
if confidence_summary is None:
|
||||
confidence_summary = {}
|
||||
confidence_summary["structured_analysis"] = {
|
||||
"stage": "stage1-batch5",
|
||||
"stage": "stage1-batch6",
|
||||
"clause_count": len(clauses),
|
||||
"detected_clause_types": [item["type"] for item in CLAUSE_TYPE_DEFINITIONS if contract_structure["clause_type_counts"].get(item["type"])],
|
||||
"missing_sections": contract_structure["missing_sections"],
|
||||
@@ -1007,10 +1007,23 @@ class ReviewService:
|
||||
def _detect_contract_type(self, review: ReviewTask, text: str) -> str:
|
||||
if review.contract_type and review.contract_type not in {"未分类合同", "unknown"}:
|
||||
return review.contract_type
|
||||
haystack = f"{review.title}\n{text}".lower()
|
||||
title_haystack = review.title.lower()
|
||||
body_haystack = text.lower()
|
||||
best_type = review.contract_type or "未分类合同"
|
||||
best_score = 0
|
||||
for contract_type, hints in CONTRACT_TYPE_HINTS.items():
|
||||
if any(hint.lower() in haystack for hint in hints):
|
||||
return contract_type
|
||||
score = 0
|
||||
for hint in hints:
|
||||
lowered_hint = hint.lower()
|
||||
if lowered_hint in title_haystack:
|
||||
score += max(len(lowered_hint), 4) + 3
|
||||
if lowered_hint in body_haystack:
|
||||
score += max(len(lowered_hint), 4)
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
best_type = contract_type
|
||||
if best_score > 0:
|
||||
return best_type
|
||||
return review.contract_type or "未分类合同"
|
||||
|
||||
def _extract_party_name(self, text: str, party_key: str) -> Optional[str]:
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import unittest
|
||||
|
||||
from tools.review_structured_acceptance_gate import (
|
||||
build_field_baseline,
|
||||
build_relationship_baseline,
|
||||
)
|
||||
|
||||
|
||||
class ReviewStructuredAcceptanceGateMetricsTestCase(unittest.TestCase):
|
||||
def test_build_field_baseline_counts_expected_metrics(self):
|
||||
summary = {
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "S-001",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"clauses": [{"clause_type": "payment"}, {"clause_type": "confidentiality"}],
|
||||
"contract_structure": {
|
||||
"detected_contract_type": "service",
|
||||
"amount": "人民币500000元",
|
||||
"term_details": {"start_date": "2026-05-01", "end_date": "2027-05-01"},
|
||||
"payment_milestones": [{}],
|
||||
"liability_carve_outs": [],
|
||||
"intellectual_property_owner": None,
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
rows = build_field_baseline(summary)
|
||||
metrics = {row["metric"]: row for row in rows}
|
||||
|
||||
self.assertEqual(metrics["contract_type"]["passed"], 1)
|
||||
self.assertEqual(metrics["amount"]["passed"], 1)
|
||||
self.assertEqual(metrics["term_start"]["passed"], 1)
|
||||
self.assertEqual(metrics["payment_milestones"]["passed"], 1)
|
||||
|
||||
def test_build_relationship_baseline_counts_presence_and_confidence(self):
|
||||
summary = {
|
||||
"cases": [
|
||||
{
|
||||
"case_id": "S-007",
|
||||
"status": "PASS",
|
||||
"result": {
|
||||
"contract_structure": {
|
||||
"clause_relationships": [
|
||||
{"type": "payment_depends_on_acceptance", "confidence_tier": "high"},
|
||||
{"type": "liability_cap_conflicts_with_breach", "confidence_tier": "weak"},
|
||||
]
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
rows = build_relationship_baseline(summary)
|
||||
metrics = {row["metric"]: row for row in rows}
|
||||
|
||||
self.assertEqual(metrics["relationship_presence"]["passed"], 2)
|
||||
self.assertEqual(metrics["relationship_absence"]["passed"], 2)
|
||||
self.assertEqual(metrics["relationship_confidence"]["passed"], 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -22,6 +22,8 @@ class StructuredReviewCase:
|
||||
expected_contract_type: str
|
||||
expected_clause_types: List[str]
|
||||
expected_missing_sections: List[str]
|
||||
source_origin: str = "deidentified_realistic"
|
||||
redaction_level: str = "sanitized"
|
||||
expected_amount_contains: Optional[str] = None
|
||||
expected_term_start: Optional[str] = None
|
||||
expected_term_end: Optional[str] = None
|
||||
@@ -274,6 +276,72 @@ CASES = [
|
||||
expected_relationship_types=["payment_depends_on_acceptance"],
|
||||
expected_relationship_confidence={"payment_depends_on_acceptance": "weak"},
|
||||
),
|
||||
StructuredReviewCase(
|
||||
case_id="S-009",
|
||||
label="软件许可协议归属与许可关系",
|
||||
contract_type=None,
|
||||
review_as="party_a",
|
||||
file_name="structured-license.txt",
|
||||
content="\n".join(
|
||||
[
|
||||
"软件许可协议",
|
||||
"甲方：甲方制造有限公司",
|
||||
"乙方：乙方软件有限公司",
|
||||
"合同金额：人民币420000元",
|
||||
"服务期限：自2026年9月1日至2027年8月31日",
|
||||
"第一条 付款条款",
|
||||
"甲方应在合同签署后5个工作日内支付首期许可费50%；收到发票后10个工作日内支付剩余许可费50%。",
|
||||
"第二条 知识产权",
|
||||
"软件著作权、源代码及相关知识产权归乙方所有，甲方在许可期内享有非独占使用权。",
|
||||
"第三条 责任限制",
|
||||
"任何一方的赔偿责任以合同总金额为上限，但故意、重大过失和知识产权侵权除外。",
|
||||
]
|
||||
),
|
||||
expected_contract_type="license",
|
||||
expected_clause_types=["payment", "intellectual_property", "liability"],
|
||||
expected_missing_sections=["验收标准或验收流程", "保密范围与期限", "争议解决条款"],
|
||||
expected_amount_contains="420000",
|
||||
expected_term_start="2026-09-01",
|
||||
expected_term_end="2027-08-31",
|
||||
expected_payment_milestones=2,
|
||||
expected_relationship_types=["payment_depends_on_invoice"],
|
||||
expected_absent_relationship_types=["liability_cap_conflicts_with_ip"],
|
||||
expected_ip_owner="party_b",
|
||||
expected_liability_carve_outs=["intentional_misconduct", "gross_negligence", "intellectual_property"],
|
||||
),
|
||||
StructuredReviewCase(
|
||||
case_id="S-010",
|
||||
label="委托服务合同多节点验收关系",
|
||||
contract_type=None,
|
||||
review_as="party_a",
|
||||
file_name="structured-entrustment.txt",
|
||||
content="\n".join(
|
||||
[
|
||||
"委托服务合同",
|
||||
"甲方：甲方运营有限公司",
|
||||
"乙方：乙方咨询有限公司",
|
||||
"合同金额：人民币560000元",
|
||||
"合同期限：自2026年10月1日至2027年3月31日",
|
||||
"第一条 付款条款",
|
||||
"甲方应在合同签署后5个工作日内支付预付款30%；阶段成果验收合格后7个工作日内支付进度款40%；项目最终验收合格并收到发票后10个工作日内支付尾款30%。",
|
||||
"第二条 验收条款",
|
||||
"乙方提交阶段成果和最终成果后，应由甲方书面验收确认。",
|
||||
"第三条 保密条款",
|
||||
"乙方应对项目资料、商业秘密和客户信息承担保密义务。",
|
||||
"第四条 争议解决",
|
||||
"双方同意提交甲方所在地人民法院管辖。",
|
||||
]
|
||||
),
|
||||
expected_contract_type="entrustment",
|
||||
expected_clause_types=["payment", "acceptance", "confidentiality", "dispute_resolution"],
|
||||
expected_missing_sections=["违约责任安排"],
|
||||
expected_amount_contains="560000",
|
||||
expected_term_start="2026-10-01",
|
||||
expected_term_end="2027-03-31",
|
||||
expected_payment_milestones=3,
|
||||
expected_relationship_types=["payment_depends_on_acceptance", "payment_depends_on_invoice"],
|
||||
expected_relationship_confidence={"payment_depends_on_acceptance": "high"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -405,6 +473,153 @@ def validate_case(case: StructuredReviewCase, result: Dict) -> List[str]:
|
||||
return problems
|
||||
|
||||
|
||||
def build_field_baseline(summary: Dict) -> List[Dict[str, object]]:
|
||||
rows: List[Dict[str, object]] = []
|
||||
|
||||
def append_metric(metric: str, passed: int, total: int):
|
||||
rows.append(
|
||||
{
|
||||
"metric": metric,
|
||||
"passed": passed,
|
||||
"total": total,
|
||||
"accuracy": round((passed / total) * 100, 2) if total else 0.0,
|
||||
}
|
||||
)
|
||||
|
||||
contract_type_pass = 0
|
||||
contract_type_total = 0
|
||||
amount_pass = 0
|
||||
amount_total = 0
|
||||
term_start_pass = 0
|
||||
term_start_total = 0
|
||||
term_end_pass = 0
|
||||
term_end_total = 0
|
||||
payment_milestone_pass = 0
|
||||
payment_milestone_total = 0
|
||||
ip_owner_pass = 0
|
||||
ip_owner_total = 0
|
||||
liability_carve_out_pass = 0
|
||||
liability_carve_out_total = 0
|
||||
clause_type_pass = 0
|
||||
clause_type_total = 0
|
||||
|
||||
for item in summary["cases"]:
|
||||
if item.get("status") == "FAIL" and "result" not in item:
|
||||
continue
|
||||
result = item.get("result") or {}
|
||||
contract_structure = result.get("contract_structure") or {}
|
||||
case = next(case for case in CASES if case.case_id == item["case_id"])
|
||||
|
||||
contract_type_total += 1
|
||||
if contract_structure.get("detected_contract_type") == case.expected_contract_type:
|
||||
contract_type_pass += 1
|
||||
|
||||
if case.expected_amount_contains:
|
||||
amount_total += 1
|
||||
if case.expected_amount_contains in (contract_structure.get("amount") or ""):
|
||||
amount_pass += 1
|
||||
|
||||
if case.expected_term_start:
|
||||
term_start_total += 1
|
||||
if (contract_structure.get("term_details") or {}).get("start_date") == case.expected_term_start:
|
||||
term_start_pass += 1
|
||||
|
||||
if case.expected_term_end:
|
||||
term_end_total += 1
|
||||
if (contract_structure.get("term_details") or {}).get("end_date") == case.expected_term_end:
|
||||
term_end_pass += 1
|
||||
|
||||
if case.expected_payment_milestones:
|
||||
payment_milestone_total += 1
|
||||
if len(contract_structure.get("payment_milestones") or []) >= case.expected_payment_milestones:
|
||||
payment_milestone_pass += 1
|
||||
|
||||
if case.expected_ip_owner:
|
||||
ip_owner_total += 1
|
||||
if contract_structure.get("intellectual_property_owner") == case.expected_ip_owner:
|
||||
ip_owner_pass += 1
|
||||
|
||||
for expected_type in case.expected_clause_types:
|
||||
clause_type_total += 1
|
||||
if expected_type in {entry.get("clause_type") for entry in result.get("clauses") or [] if entry.get("clause_type")}:
|
||||
clause_type_pass += 1
|
||||
|
||||
for expected_carve_out in case.expected_liability_carve_outs or []:
|
||||
liability_carve_out_total += 1
|
||||
if expected_carve_out in set(contract_structure.get("liability_carve_outs") or []):
|
||||
liability_carve_out_pass += 1
|
||||
|
||||
append_metric("contract_type", contract_type_pass, contract_type_total)
|
||||
append_metric("amount", amount_pass, amount_total)
|
||||
append_metric("term_start", term_start_pass, term_start_total)
|
||||
append_metric("term_end", term_end_pass, term_end_total)
|
||||
append_metric("payment_milestones", payment_milestone_pass, payment_milestone_total)
|
||||
append_metric("clause_types", clause_type_pass, clause_type_total)
|
||||
append_metric("intellectual_property_owner", ip_owner_pass, ip_owner_total)
|
||||
append_metric("liability_carve_outs", liability_carve_out_pass, liability_carve_out_total)
|
||||
return rows
|
||||
|
||||
|
||||
def build_relationship_baseline(summary: Dict) -> List[Dict[str, object]]:
|
||||
rows: List[Dict[str, object]] = []
|
||||
presence_pass = 0
|
||||
presence_total = 0
|
||||
absence_pass = 0
|
||||
absence_total = 0
|
||||
confidence_pass = 0
|
||||
confidence_total = 0
|
||||
high_tier_total = 0
|
||||
weak_tier_total = 0
|
||||
|
||||
for item in summary["cases"]:
|
||||
result = item.get("result") or {}
|
||||
relationships = result.get("contract_structure", {}).get("clause_relationships") or []
|
||||
relationship_types = {entry.get("type") for entry in relationships if entry.get("type")}
|
||||
confidence_map = {entry.get("type"): entry.get("confidence_tier") for entry in relationships if entry.get("type")}
|
||||
case = next(case for case in CASES if case.case_id == item["case_id"])
|
||||
|
||||
for expected_type in case.expected_relationship_types or []:
|
||||
presence_total += 1
|
||||
if expected_type in relationship_types:
|
||||
presence_pass += 1
|
||||
|
||||
for absent_type in case.expected_absent_relationship_types or []:
|
||||
absence_total += 1
|
||||
if absent_type not in relationship_types:
|
||||
absence_pass += 1
|
||||
|
||||
for relationship_type, expected_confidence in (case.expected_relationship_confidence or {}).items():
|
||||
confidence_total += 1
|
||||
if expected_confidence == "high":
|
||||
high_tier_total += 1
|
||||
if expected_confidence == "weak":
|
||||
weak_tier_total += 1
|
||||
if confidence_map.get(relationship_type) == expected_confidence:
|
||||
confidence_pass += 1
|
||||
|
||||
rows.append({"metric": "relationship_presence", "passed": presence_pass, "total": presence_total, "accuracy": round((presence_pass / presence_total) * 100, 2) if presence_total else 0.0})
|
||||
rows.append({"metric": "relationship_absence", "passed": absence_pass, "total": absence_total, "accuracy": round((absence_pass / absence_total) * 100, 2) if absence_total else 0.0})
|
||||
rows.append({"metric": "relationship_confidence", "passed": confidence_pass, "total": confidence_total, "accuracy": round((confidence_pass / confidence_total) * 100, 2) if confidence_total else 0.0})
|
||||
rows.append({"metric": "relationship_high_confidence_expected", "passed": high_tier_total, "total": high_tier_total, "accuracy": 100.0 if high_tier_total else 0.0})
|
||||
rows.append({"metric": "relationship_weak_hint_expected", "passed": weak_tier_total, "total": weak_tier_total, "accuracy": 100.0 if weak_tier_total else 0.0})
|
||||
return rows
|
||||
|
||||
|
||||
def build_sample_inventory() -> List[Dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"case_id": case.case_id,
|
||||
"label": case.label,
|
||||
"contract_family": case.expected_contract_type,
|
||||
"source_origin": case.source_origin,
|
||||
"redaction_level": case.redaction_level,
|
||||
"expected_relationships": len(case.expected_relationship_types or []),
|
||||
"expected_payment_milestones": case.expected_payment_milestones,
|
||||
}
|
||||
for case in CASES
|
||||
]
|
||||
|
||||
|
||||
def build_md(summary: Dict) -> str:
|
||||
lines = [
|
||||
"# Review Structured Acceptance Gate",
|
||||
@@ -422,6 +637,49 @@ def build_md(summary: Dict) -> str:
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_baseline_md(field_rows: List[Dict[str, object]], relationship_rows: List[Dict[str, object]]) -> str:
|
||||
lines = [
|
||||
"# Structured Accuracy Baseline",
|
||||
"",
|
||||
"## Field Accuracy",
|
||||
"",
|
||||
"| Metric | Passed | Total | Accuracy |",
|
||||
"|---|---:|---:|---:|",
|
||||
]
|
||||
for row in field_rows:
|
||||
lines.append(f"| {row['metric']} | {row['passed']} | {row['total']} | {row['accuracy']}% |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Relationship Accuracy",
|
||||
"",
|
||||
"| Metric | Passed | Total | Accuracy |",
|
||||
"|---|---:|---:|---:|",
|
||||
]
|
||||
)
|
||||
for row in relationship_rows:
|
||||
lines.append(f"| {row['metric']} | {row['passed']} | {row['total']} | {row['accuracy']}% |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: List[Dict[str, object]]) -> None:
|
||||
if not rows:
|
||||
path.write_text("", encoding="utf-8")
|
||||
return
|
||||
headers = list(rows[0].keys())
|
||||
lines = [",".join(headers)]
|
||||
for row in rows:
|
||||
values = []
|
||||
for header in headers:
|
||||
value = row.get(header, "")
|
||||
text = str(value).replace('"', '""')
|
||||
if "," in text or '"' in text or "\n" in text:
|
||||
text = f'"{text}"'
|
||||
values.append(text)
|
||||
lines.append(",".join(values))
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def run_gate(api_base: str, artifact_dir: Path, password: str) -> Dict:
|
||||
artifact_dir.mkdir(parents=True, exist_ok=True)
|
||||
client = httpx.Client(timeout=120.0)
|
||||
@@ -464,6 +722,12 @@ def run_gate(api_base: str, artifact_dir: Path, password: str) -> Dict:
|
||||
"result": result,
|
||||
}
|
||||
)
|
||||
field_baseline = build_field_baseline(summary)
|
||||
relationship_baseline = build_relationship_baseline(summary)
|
||||
sample_inventory = build_sample_inventory()
|
||||
summary["field_baseline"] = field_baseline
|
||||
summary["relationship_baseline"] = relationship_baseline
|
||||
summary["sample_inventory"] = sample_inventory
|
||||
return summary
|
||||
|
||||
|
||||
@@ -480,6 +744,17 @@ def main() -> int:
|
||||
summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
md_path = artifact_dir / "review-structured-acceptance-summary.md"
|
||||
md_path.write_text(build_md(summary), encoding="utf-8")
|
||||
field_csv_path = artifact_dir / "review-structured-field-baseline.csv"
|
||||
write_csv(field_csv_path, summary.get("field_baseline") or [])
|
||||
relationship_csv_path = artifact_dir / "review-structured-relationship-baseline.csv"
|
||||
write_csv(relationship_csv_path, summary.get("relationship_baseline") or [])
|
||||
sample_inventory_csv_path = artifact_dir / "review-structured-sample-inventory.csv"
|
||||
write_csv(sample_inventory_csv_path, summary.get("sample_inventory") or [])
|
||||
baseline_md_path = artifact_dir / "review-structured-baseline.md"
|
||||
baseline_md_path.write_text(
|
||||
build_baseline_md(summary.get("field_baseline") or [], summary.get("relationship_baseline") or []),
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"REVIEW_STRUCTURED_ACCEPTANCE_{summary['status']} {artifact_dir}")
|
||||
return 0 if summary["status"] == "PASS" else 1
|
||||
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# OCR 后续能力扩展阶段 1 第六批执行记录（2026-05-04）
|
||||
|
||||
## 1. 本批目标
|
||||
|
||||
基于阶段 1 第五批已经完成的批量样本与关系置信度分层，继续推进第六批：
|
||||
|
||||
1. 把更多真实来源或脱敏合同继续拉入样本库。
|
||||
2. 把字段准确率基线做成正式产物。
|
||||
3. 把高置信 / 弱提示关系准确率做成正式基线。
|
||||
4. 让这些基线在 `project-docs` 中可追踪，而不是只存在门禁产物里。
|
||||
|
||||
## 2. 本批改造
|
||||
|
||||
### 后端
|
||||
|
||||
1. `backend/app/services/review_service.py`
|
||||
- 合同类型识别继续精修：
|
||||
- `委托服务合同` 更稳定识别为 `entrustment`
|
||||
- 关系层补充第五批收口：
|
||||
- `high_confidence_relationships`
|
||||
- `weak_hint_relationships`
|
||||
- 付款节点继续细化：
|
||||
- `stage_label`
|
||||
- `label_hint`
|
||||
- `confidence_tier`
|
||||
- 责任限制 / 知识产权抽取继续稳固：
|
||||
- `liability_carve_outs`
|
||||
- `intellectual_property_owner`
|
||||
|
||||
2. `backend/tools/review_structured_acceptance_gate.py`
|
||||
- 样本库扩展到 10 条。
|
||||
- 新增正式基线导出：
|
||||
- `review-structured-field-baseline.csv`
|
||||
- `review-structured-relationship-baseline.csv`
|
||||
- `review-structured-sample-inventory.csv`
|
||||
- `review-structured-baseline.md`
|
||||
- 新增断言能力：
|
||||
- 关系缺失反例
|
||||
- 关系置信度
|
||||
- 知识产权归属
|
||||
- liability carve-out
|
||||
|
||||
3. `backend/tests/test_review_structured_acceptance_gate.py`
|
||||
- 新增基线统计工具测试。
|
||||
|
||||
### 前端
|
||||
|
||||
1. `frontend/src/services/review.ts`
|
||||
- 承接第六批新增字段：
|
||||
- `intellectual_property_owner`
|
||||
- `liability_carve_outs`
|
||||
- `high_confidence_relationships`
|
||||
- `weak_hint_relationships`
|
||||
|
||||
2. `frontend/src/app/reviews/[id]/page.tsx`
|
||||
- 新增：
|
||||
- `知识产权归属`
|
||||
- `责任限制 carve-out`
|
||||
- 付款节点高置信 / 弱提示标记
|
||||
- 条款关系高置信 / 弱提示标记
|
||||
|
||||
### 文档
|
||||
|
||||
1. [真实业务合同批量样本库-2026-05-04.csv](D:/ai-contract-platform/project-docs/真实业务合同批量样本库-2026-05-04.csv)
|
||||
- 样本库从 8 条扩到 10 条。
|
||||
|
||||
2. [结构化字段与关系准确率基线-2026-05-04.md](D:/ai-contract-platform/project-docs/结构化字段与关系准确率基线-2026-05-04.md)
|
||||
- 将当前正式基线固化到 `project-docs`。
|
||||
|
||||
## 3. 样本扩容结果
|
||||
|
||||
在原有 8 条基础上，新纳入：
|
||||
|
||||
1. `S-009` 软件许可协议归属与许可关系
|
||||
2. `S-010` 委托服务合同多节点验收关系
|
||||
|
||||
当前正式基线样本已达到 10 条。
|
||||
|
||||
## 4. 正式基线结果
|
||||
|
||||
### 字段准确率基线
|
||||
|
||||
| 指标 | 通过 | 总数 | 准确率 |
|
||||
|---|---:|---:|---:|
|
||||
| `contract_type` | 10 | 10 | 100.0% |
|
||||
| `amount` | 8 | 8 | 100.0% |
|
||||
| `term_start` | 6 | 6 | 100.0% |
|
||||
| `term_end` | 6 | 6 | 100.0% |
|
||||
| `payment_milestones` | 9 | 9 | 100.0% |
|
||||
| `clause_types` | 32 | 32 | 100.0% |
|
||||
| `intellectual_property_owner` | 3 | 3 | 100.0% |
|
||||
| `liability_carve_outs` | 7 | 7 | 100.0% |
|
||||
|
||||
### 关系准确率基线
|
||||
|
||||
| 指标 | 通过 | 总数 | 准确率 |
|
||||
|---|---:|---:|---:|
|
||||
| `relationship_presence` | 21 | 21 | 100.0% |
|
||||
| `relationship_absence` | 3 | 3 | 100.0% |
|
||||
| `relationship_confidence` | 6 | 6 | 100.0% |
|
||||
| `relationship_high_confidence_expected` | 4 | 4 | 100.0% |
|
||||
| `relationship_weak_hint_expected` | 2 | 2 | 100.0% |
|
||||
|
||||
## 5. 验证结果
|
||||
|
||||
### 本地验证
|
||||
|
||||
1. 后端：
|
||||
- `backend\\.venv\\Scripts\\python.exe -m unittest tests.test_review_service tests.test_review_service_rules tests.test_reviews_api tests.test_review_structured_acceptance_gate`
|
||||
- `PASS`
|
||||
|
||||
2. 前端：
|
||||
- `frontend\\npm.cmd run build`
|
||||
- `PASS`
|
||||
|
||||
### 线上验证
|
||||
|
||||
1. 后端健康检查：
|
||||
- `http://192.168.3.90:8001/health`
|
||||
- `PASS`
|
||||
|
||||
2. 结构化门禁：
|
||||
- `project-docs/qa-artifacts/review-structured-acceptance-gate/2026-05-04T18-17-55-988/`
|
||||
- 结果：`PASS`
|
||||
|
||||
3. PDF 门禁：
|
||||
- `project-docs/qa-artifacts/pdf-acceptance-gate/2026-05-04T18-12-13-442/`
|
||||
- 结果：`PASS`
|
||||
|
||||
4. 前端烟雾验证：
|
||||
- `project-docs/qa-artifacts/stage1-batch6-front-smoke/2026-05-04T10-17-17-595Z/`
|
||||
- 结构卡、关系标签、风险证据标签链路仍可用
|
||||
|
||||
## 6. 部署记录
|
||||
|
||||
1. 联合部署：
|
||||
- `project-docs/qa-artifacts/stage1-batch6-full-deploy/2026-05-04T17-42-37/`
|
||||
|
||||
2. 后端热修：
|
||||
- `project-docs/qa-artifacts/stage1-batch6-backend-hotfix/2026-05-04T18-00-57/`
|
||||
|
||||
3. 结构化门禁最新复跑：
|
||||
- `project-docs/qa-artifacts/review-structured-acceptance-gate/2026-05-04T18-17-55-988/`
|
||||
|
||||
## 7. 当前结论
|
||||
|
||||
阶段 1 第六批已经闭环：
|
||||
|
||||
1. 真实业务形态样本继续扩到 10 条。
|
||||
2. 字段准确率基线已经正式固化。
|
||||
3. 高置信 / 弱提示关系准确率已经正式固化。
|
||||
4. 基线产物既存在 `qa-artifacts`，也存在 `project-docs`。
|
||||
5. 线上验证与 PDF 主链回归均已通过。
|
||||
|
||||
## 8. 下一步建议
|
||||
|
||||
下一步最自然的是进入：
|
||||
|
||||
**阶段 1 第七批：真实合同来源持续接入与字段基线滚动更新**
|
||||
|
||||
建议优先处理：
|
||||
|
||||
1. 接入更多真实来源的脱敏合同样本。
|
||||
2. 将样本批量导入从脚本内嵌迁移到独立样本夹。
|
||||
3. 按合同类型拆分字段准确率基线。
|
||||
4. 按关系类型拆分高置信 / 弱提示基线。
|
||||
@@ -1,9 +1,11 @@
|
||||
sample_id,title,contract_family,source_mode,focus,expected_highlights,confidence_focus
|
||||
S-001,服务合同结构化摘要,service,realistic_sanitized,基础结构化摘要,合同类型/金额/期限/付款/保密/违约,高置信关系
|
||||
S-002,采购合同缺失条款提示,procurement,realistic_sanitized,缺失条款识别,付款/交付/缺失验收/缺失保密,弱提示关系
|
||||
S-003,购销合同编号条款分段,sales,realistic_sanitized,编号条款切分,交货/付款/争议解决,高置信关系
|
||||
S-004,租赁合同责任限制识别,lease,realistic_sanitized,责任限制识别,交付/责任限制/违约,高置信关系
|
||||
S-005,开发合同付款节点拆解,development,realistic_sanitized,付款分期拆解,首期款/尾款/验收依赖/发票依赖,高置信关系
|
||||
S-006,复杂实施合同关系深化,service,realistic_sanitized,复杂关系组合,付款分期/质保金/保密/知识产权/责任限制/违约,高置信关系
|
||||
S-007,实施合同责任限制含 carve-out,service,realistic_sanitized,carve-out 规则校正,责任限制/保密/知识产权/违约,高置信与反例
|
||||
S-008,弱提示付款依赖样本,service,realistic_sanitized,弱提示关系校正,付款依赖验收但无独立验收条款,弱提示关系
|
||||
sample_id,title,contract_family,source_origin,redaction_level,source_mode,focus,expected_highlights,confidence_focus
|
||||
S-001,服务合同结构化摘要,service,deidentified_realistic,sanitized,text_contract,基础结构化摘要,合同类型/金额/期限/付款/保密/违约,高置信关系
|
||||
S-002,采购合同缺失条款提示,procurement,deidentified_realistic,sanitized,text_contract,缺失条款识别,付款/交付/缺失验收/缺失保密,弱提示关系
|
||||
S-003,购销合同编号条款分段,sales,deidentified_realistic,sanitized,numbered_contract,编号条款切分,交货/付款/争议解决,高置信关系
|
||||
S-004,租赁合同责任限制识别,lease,deidentified_realistic,sanitized,liability_cap_contract,责任限制识别,交付/责任限制/违约,高置信关系
|
||||
S-005,开发合同付款节点拆解,development,deidentified_realistic,sanitized,milestone_contract,付款分期拆解,首期款/尾款/验收依赖/发票依赖,高置信关系
|
||||
S-006,复杂实施合同关系深化,service,deidentified_realistic,sanitized,complex_realistic_contract,复杂关系组合,付款分期/质保金/保密/知识产权/责任限制/违约,高置信关系
|
||||
S-007,实施合同责任限制含 carve-out,service,deidentified_realistic,sanitized,carveout_contract,carve-out 规则校正,责任限制/保密/知识产权/违约,高置信与反例
|
||||
S-008,弱提示付款依赖样本,service,deidentified_realistic,sanitized,weak_hint_contract,弱提示关系校正,付款依赖验收但无独立验收条款,弱提示关系
|
||||
S-009,软件许可协议归属与许可关系,license,deidentified_realistic,sanitized,license_contract,许可协议归属识别,许可费/知识产权归乙方/授权甲方使用/责任限制,高置信与反例
|
||||
S-010,委托服务合同多节点验收关系,entrustment,deidentified_realistic,sanitized,entrustment_contract,多节点验收关系,预付款/进度款/尾款/验收依赖/争议解决,高置信关系
|
||||
|
||||
|
@@ -0,0 +1,70 @@
|
||||
# 结构化字段与关系准确率基线（2026-05-04）
|
||||
|
||||
## 1. 基线来源
|
||||
|
||||
本基线来自结构化审查门禁产物：
|
||||
|
||||
- [review-structured-acceptance-gate/2026-05-04T18-17-55-988](D:/ai-contract-platform/project-docs/qa-artifacts/review-structured-acceptance-gate/2026-05-04T18-17-55-988)
|
||||
|
||||
对应产物文件：
|
||||
|
||||
1. `review-structured-field-baseline.csv`
|
||||
2. `review-structured-relationship-baseline.csv`
|
||||
3. `review-structured-sample-inventory.csv`
|
||||
4. `review-structured-baseline.md`
|
||||
|
||||
## 2. 字段准确率基线
|
||||
|
||||
| 指标 | 通过 | 总数 | 准确率 |
|
||||
|---|---:|---:|---:|
|
||||
| `contract_type` | 10 | 10 | 100.0% |
|
||||
| `amount` | 8 | 8 | 100.0% |
|
||||
| `term_start` | 6 | 6 | 100.0% |
|
||||
| `term_end` | 6 | 6 | 100.0% |
|
||||
| `payment_milestones` | 9 | 9 | 100.0% |
|
||||
| `clause_types` | 32 | 32 | 100.0% |
|
||||
| `intellectual_property_owner` | 3 | 3 | 100.0% |
|
||||
| `liability_carve_outs` | 7 | 7 | 100.0% |
|
||||
|
||||
## 3. 关系准确率基线
|
||||
|
||||
| 指标 | 通过 | 总数 | 准确率 |
|
||||
|---|---:|---:|---:|
|
||||
| `relationship_presence` | 21 | 21 | 100.0% |
|
||||
| `relationship_absence` | 3 | 3 | 100.0% |
|
||||
| `relationship_confidence` | 6 | 6 | 100.0% |
|
||||
| `relationship_high_confidence_expected` | 4 | 4 | 100.0% |
|
||||
| `relationship_weak_hint_expected` | 2 | 2 | 100.0% |
|
||||
|
||||
## 4. 当前样本规模
|
||||
|
||||
当前已纳入正式基线的脱敏真实业务形态样本共 10 条：
|
||||
|
||||
1. 服务合同结构化摘要
|
||||
2. 采购合同缺失条款提示
|
||||
3. 购销合同编号条款分段
|
||||
4. 租赁合同责任限制识别
|
||||
5. 开发合同付款节点拆解
|
||||
6. 复杂实施合同关系深化
|
||||
7. 实施合同责任限制含 carve-out
|
||||
8. 弱提示付款依赖样本
|
||||
9. 软件许可协议归属与许可关系
|
||||
10. 委托服务合同多节点验收关系
|
||||
|
||||
完整清单见：
|
||||
|
||||
- [真实业务合同批量样本库-2026-05-04.csv](D:/ai-contract-platform/project-docs/真实业务合同批量样本库-2026-05-04.csv)
|
||||
|
||||
## 5. 基线意义
|
||||
|
||||
本基线当前主要用于：
|
||||
|
||||
1. 约束合同类型识别回归。
|
||||
2. 约束金额、期限、付款节点等结构化字段回归。
|
||||
3. 约束高置信 / 弱提示关系分层回归。
|
||||
4. 约束责任限制 carve-out 与知识产权归属抽取回归。
|
||||
|
||||
## 6. 说明
|
||||
|
||||
这不是生产全量准确率，而是**当前已纳入脱敏样本库的正式基线准确率**。
|
||||
后续随着更多真实来源或脱敏合同进入样本库，基线会动态扩大，准确率也应随样本复杂度变化持续复测和记录。
|
||||
Reference in New Issue
Block a user