diff --git a/backend/app/api/prompt_assets.py b/backend/app/api/prompt_assets.py index 5e1e1d8d..600a97c5 100644 --- a/backend/app/api/prompt_assets.py +++ b/backend/app/api/prompt_assets.py @@ -16,6 +16,7 @@ ROUTE_PERMISSION_ACTIONS = { "detail": "prompt_assets:detail", "create": "prompt_assets:create", "create_version": "prompt_assets:versions:create", + "evaluate": "prompt_assets:evaluations:run", "publish": "prompt_assets:publish", "unpublish": "prompt_assets:unpublish", "rollback": "prompt_assets:rollback", @@ -156,6 +157,32 @@ def create_prompt_asset_version( return success_response(result) +@router.post("/{asset_id}/evaluations/run") +def run_prompt_asset_evaluation( + asset_id: str, + prompt_asset_service: PromptAssetService = Depends(get_prompt_asset_service), + audit_log_service: AuditLogService = Depends(get_audit_log_service), + current_user: dict = Depends(require_current_user), +): + result = prompt_asset_service.evaluate_current_prompt_asset(current_user, asset_id) + audit_log_service.record_event( + tenant_id=current_user.get("tenant_id"), + user_id=current_user.get("id"), + action="run_prompt_asset_evaluation", + resource_type="prompt_asset", + resource_id=result["asset"]["id"], + detail_json={ + "prompt_key": result["asset"]["prompt_key"], + "prompt_type": result["asset"]["prompt_type"], + "version_label": result["version"]["version_label"], + "evaluation_status": result["version"]["evaluation_status"], + "evaluation_score": result["version"]["evaluation_score"], + "suite_key": result["evaluation_report"].get("suite_key"), + }, + ) + return success_response(result) + + @router.post("/{asset_id}/publish") def publish_prompt_asset( asset_id: str, diff --git a/backend/app/services/prompt_asset_service.py b/backend/app/services/prompt_asset_service.py index 411f563b..75c7b6d3 100644 --- a/backend/app/services/prompt_asset_service.py +++ b/backend/app/services/prompt_asset_service.py @@ -25,6 +25,14 @@ class PromptAssetService: } EVALUATION_STATUSES = {"not_evaluated", "pending", "passed", "failed"} PUBLISHABLE_EVALUATION_STATUSES = {"passed"} + PROMPT_GOLDEN_SUITES = { + "contract_generation": {"suite_key": "phase13_generation", "sample_count": 8}, + "contract_review": {"suite_key": "phase13_review", "sample_count": 8}, + "legal_chat": {"suite_key": "phase13_kb_qa", "sample_count": 10}, + "clause_recommendation": {"suite_key": "phase13_clause_generation", "sample_count": 6}, + "wording_rewrite": {"suite_key": "phase15_wording_rewrite", "sample_count": 3}, + "regulation_impact": {"suite_key": "future_regulation_impact", "sample_count": 0}, + } def __init__(self, db: Session): self.db = db @@ -178,6 +186,27 @@ class PromptAssetService: self._copy_evaluation_to_asset(updated, version) return self._to_item(self.repository.update(updated)) + def evaluate_current_prompt_asset(self, current_user: Dict[str, Any], asset_id: str) -> dict: + self._require_admin(current_user) + asset = self._require_asset(current_user, asset_id) + version = self._require_current_version(current_user, asset) + report = self._build_prompt_evaluation_report(asset, version) + + version.evaluation_report_json = report + version.evaluation_score = float(report["quality_score"]) + version.evaluation_status = "passed" if report["passed"] else "failed" + version = self.version_repository.update(version) + + asset.updated_by = current_user.get("id") + asset.updated_at = self._now() + self._copy_evaluation_to_asset(asset, version) + asset = self.repository.update(asset) + return { + "asset": self._to_item(asset), + "version": self._to_version_item(version), + "evaluation_report": report, + } + def publish_prompt_asset( self, current_user: Dict[str, Any], @@ -409,6 +438,108 @@ class PromptAssetService: asset.latest_evaluation_score = version.evaluation_score asset.latest_evaluation_report_json = version.evaluation_report_json + def _build_prompt_evaluation_report(self, asset: PromptAsset, version: PromptAssetVersion) -> dict[str, Any]: + suite = self.PROMPT_GOLDEN_SUITES.get(asset.prompt_type, {"suite_key": "custom_prompt_suite", "sample_count": 1}) + sample_count = int(suite["sample_count"]) + checks = self._build_prompt_quality_checks(asset, version) + blocking_failures = [item for item in checks if item["required"] and item["status"] != "pass"] + warning_failures = [item for item in checks if not item["required"] and item["status"] != "pass"] + passed = not blocking_failures + quality_score = max(0.0, 1.0 - (0.25 * len(blocking_failures)) - (0.05 * len(warning_failures))) + if not passed: + quality_score = min(quality_score, 0.79) + passed_samples = sample_count if passed else max(0, sample_count - len(blocking_failures)) + pass_rate = 1.0 if sample_count == 0 else round(passed_samples / sample_count, 4) + return { + "schema_version": "phase15-prompt-evaluation-report.v1", + "evaluator": "prompt_asset_api_static_gate", + "prompt_asset_id": asset.id, + "prompt_asset_version_id": version.id, + "prompt_key": asset.prompt_key, + "prompt_type": asset.prompt_type, + "version_label": version.version_label, + "suite_key": suite["suite_key"], + "sample_count": sample_count, + "passed_samples": passed_samples, + "failed_samples": sample_count - passed_samples, + "pass_rate": pass_rate, + "quality_score": round(quality_score, 4), + "passed": passed, + "failed_items": [item for item in checks if item["status"] != "pass"], + "checks": checks, + "evaluated_at": self._now().isoformat(), + "source": "prompt_assets.evaluate_current_version", + } + + def _build_prompt_quality_checks(self, asset: PromptAsset, version: PromptAssetVersion) -> list[dict[str, Any]]: + content = f"{version.system_prompt or ''}\n{version.prompt_content or ''}".strip() + normalized = content.lower() + checks = [ + { + "key": "prompt_content_minimum_length", + "label": "Prompt content is long enough for production use", + "required": True, + "status": "pass" if len(content) >= 20 else "fail", + "detail": f"content_length={len(content)}", + }, + { + "key": "no_unresolved_placeholder", + "label": "Prompt has no TODO/FIXME placeholders", + "required": True, + "status": "fail" if self._contains_any(normalized, ["todo", "fixme"]) else "pass", + "detail": "checked common unfinished markers", + }, + { + "key": "has_output_contract", + "label": "Prompt describes the expected output", + "required": False, + "status": "pass" + if self._contains_any( + normalized, + ["output", "return", "json", "html", "schema", "generate", "返回", "输出", "生成", "结果"], + ) + else "warn", + "detail": "looks for output or return instructions", + }, + { + "key": "has_legal_grounding_instruction", + "label": "Prompt asks for legal basis or source grounding", + "required": False, + "status": "pass" + if self._contains_any(normalized, ["legal", "law", "article", "source", "citation", "法条", "法律", "依据", "来源"]) + else "warn", + "detail": "looks for law/source grounding terms", + }, + ] + if version.variables_json: + variable_names = [str(item.get("name") or "").lower() for item in version.variables_json if isinstance(item, dict)] + mentioned_variables = [name for name in variable_names if name and name in normalized] + checks.append( + { + "key": "declared_variables_referenced", + "label": "Declared variables are referenced by the prompt", + "required": False, + "status": "pass" if mentioned_variables or "{{" in content else "warn", + "detail": f"referenced={len(mentioned_variables)} declared={len(variable_names)}", + } + ) + if asset.prompt_type in {"contract_review", "clause_recommendation", "wording_rewrite"}: + checks.append( + { + "key": "risk_boundary_instruction", + "label": "Prompt asks for risk boundaries or negotiation guidance", + "required": False, + "status": "pass" + if self._contains_any(normalized, ["risk", "boundary", "negotiate", "风险", "边界", "建议", "谈判"]) + else "warn", + "detail": "looks for risk and boundary instructions", + } + ) + return checks + + def _contains_any(self, value: str, candidates: list[str]) -> bool: + return any(candidate.lower() in value for candidate in candidates) + def _normalize_prompt_key(self, value: str) -> str: normalized = self._normalize_required_text(value, "prompt_key").lower() normalized = "".join(char if char.isalnum() else "_" for char in normalized) diff --git a/backend/tests/test_prompt_asset_service.py b/backend/tests/test_prompt_asset_service.py index faec49e0..7744b36f 100644 --- a/backend/tests/test_prompt_asset_service.py +++ b/backend/tests/test_prompt_asset_service.py @@ -136,6 +136,17 @@ class PromptAssetServiceTestCase(unittest.TestCase): self.assertEqual(context.exception.status_code, 409) + def test_run_evaluation_writes_report_to_current_version(self): + created = self._create_asset(evaluation=False) + + evaluation = self.service.evaluate_current_prompt_asset(self.admin_user, created["id"]) + published = self.service.publish_prompt_asset(self.admin_user, created["id"]) + + self.assertEqual(evaluation["asset"]["latest_evaluation_status"], "passed") + self.assertEqual(evaluation["version"]["evaluation_status"], "passed") + self.assertEqual(evaluation["evaluation_report"]["suite_key"], "phase13_generation") + self.assertEqual(published["status"], "published") + def test_publish_with_report_and_create_next_version(self): created = self._create_asset(evaluation=False) published = self.service.publish_prompt_asset( diff --git a/backend/tests/test_prompt_assets_api.py b/backend/tests/test_prompt_assets_api.py index 5e45ead8..be1008ec 100644 --- a/backend/tests/test_prompt_assets_api.py +++ b/backend/tests/test_prompt_assets_api.py @@ -102,6 +102,38 @@ class StubPromptAssetService: self._require_admin(current_user) return self._item(id=asset_id, current_version_label="v1.1", version_count=2) + def evaluate_current_prompt_asset(self, current_user, asset_id): + self._require_admin(current_user) + return { + "asset": self._item(id=asset_id, latest_evaluation_status="passed", latest_evaluation_score=0.95), + "version": { + "id": "promptver-123", + "tenant_id": "tenant-demo-001", + "prompt_asset_id": asset_id, + "prompt_key": "contract_generation_default", + "prompt_type": "contract_generation", + "version_label": "v1.0", + "prompt_content": "Generate a contract.", + "system_prompt": None, + "variables": [], + "output_schema": None, + "model_config": None, + "evaluation_status": "passed", + "evaluation_score": 0.95, + "evaluation_report": {"suite_key": "phase13_generation", "passed": True}, + "change_summary": "Initial prompt version", + "rollback_from_version_id": None, + "is_current": True, + "is_published": False, + "created_by": "user-admin", + "created_by_name": "Admin", + "created_at": "2026-05-21T00:00:00+00:00", + "published_at": None, + "published_by": None, + }, + "evaluation_report": {"suite_key": "phase13_generation", "passed": True, "quality_score": 0.95}, + } + def publish_prompt_asset(self, current_user, asset_id, **kwargs): self._require_admin(current_user) return self._item(id=asset_id, status="published", published_version_id="promptver-123", published_by=current_user["id"], published_at="2026-05-21T01:00:00+00:00") @@ -176,6 +208,7 @@ class PromptAssetsApiTestCase(unittest.TestCase): }, ) version_response = self.client.post("/api/v1/prompt-assets/prompt-123/versions", data={"prompt_content": "Generate a better contract."}) + evaluation_response = self.client.post("/api/v1/prompt-assets/prompt-123/evaluations/run") publish_response = self.client.post("/api/v1/prompt-assets/prompt-123/publish", data={"evaluation_report_json": '{"sample_count":3,"passed":true}'}) rollback_response = self.client.post("/api/v1/prompt-assets/prompt-123/rollback", data={"target_version_label": "v1.0"}) run_link_response = self.client.post( @@ -185,12 +218,14 @@ class PromptAssetsApiTestCase(unittest.TestCase): self.assertEqual(create_response.status_code, 200) self.assertEqual(version_response.json()["data"]["current_version_label"], "v1.1") + self.assertEqual(evaluation_response.json()["data"]["version"]["evaluation_status"], "passed") self.assertEqual(publish_response.json()["data"]["status"], "published") self.assertEqual(rollback_response.json()["data"]["current_version_label"], "v1.2") self.assertEqual(run_link_response.json()["data"]["prompt_asset_version_id"], "promptver-123") self.assertEqual([event["action"] for event in self.audit_log_service.events], [ "create_prompt_asset", "create_prompt_asset_version", + "run_prompt_asset_evaluation", "publish_prompt_asset", "rollback_prompt_asset", "record_prompt_asset_run_link", diff --git a/frontend/src/app/prompt-assets/page.tsx b/frontend/src/app/prompt-assets/page.tsx index d5c104a3..9f4b9ff8 100644 --- a/frontend/src/app/prompt-assets/page.tsx +++ b/frontend/src/app/prompt-assets/page.tsx @@ -3,6 +3,7 @@ import { CheckCircleOutlined, CloudUploadOutlined, + ExperimentOutlined, HistoryOutlined, LinkOutlined, PlusOutlined, @@ -21,6 +22,7 @@ import { publishPromptAsset, recordPromptRunLink, rollbackPromptAsset, + runPromptAssetEvaluation, unpublishPromptAsset, } from "@/services/prompt-asset"; import { useAuthStore } from "@/stores/auth.store"; @@ -291,6 +293,24 @@ export default function PromptAssetsPage() { } }; + const handleRunEvaluation = async (item: PromptAssetItem) => { + setMutatingId(item.id); + try { + const result = await runPromptAssetEvaluation(item.id); + if (result.code !== 0) { + messageApi.error(result.message || "Prompt 评测失败"); + return; + } + const statusText = result.data.version.evaluation_status === "passed" ? "通过" : "未通过"; + messageApi.success(`Prompt 评测已完成：${statusText}`); + await refreshAfterMutation(item.id); + } catch (error) { + messageApi.error(error instanceof Error ? error.message : "Prompt 评测失败"); + } finally { + setMutatingId(null); + } + }; + const handlePublish = async (item: PromptAssetItem) => { setMutatingId(item.id); try { @@ -450,12 +470,22 @@ export default function PromptAssetsPage() { + {item.status === "published" ? ( ) : ( - )} @@ -643,6 +673,9 @@ export default function PromptAssetsPage() { {currentVersion.version_label} + ) : null } diff --git a/frontend/src/services/prompt-asset.ts b/frontend/src/services/prompt-asset.ts index d62ebb33..9eeb483f 100644 --- a/frontend/src/services/prompt-asset.ts +++ b/frontend/src/services/prompt-asset.ts @@ -3,6 +3,7 @@ import type { ApiResponse, PromptAssetCreatePayload, PromptAssetDetailResponse, + PromptAssetEvaluationRunResponse, PromptAssetItem, PromptAssetListResponse, PromptAssetPublishPayload, @@ -40,6 +41,11 @@ export async function createPromptAssetVersion(assetId: string, payload: PromptA return response.data; } +export async function runPromptAssetEvaluation(assetId: string) { + const response = await http.post>(`/prompt-assets/${assetId}/evaluations/run`); + return response.data; +} + export async function publishPromptAsset(assetId: string, payload: PromptAssetPublishPayload = {}) { const formData = new FormData(); appendOptional(formData, "evaluation_report_json", payload.evaluation_report_json); diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index b4d39a08..56972dc6 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -566,6 +566,12 @@ export type PromptAssetDetailResponse = { run_links: PromptAssetRunLinkItem[]; }; +export type PromptAssetEvaluationRunResponse = { + asset: PromptAssetItem; + version: PromptAssetVersionItem; + evaluation_report: Record; +}; + export type PromptAssetCreatePayload = { prompt_key: string; prompt_type: PromptAssetType; diff --git a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md index 8d57b862..a1f81616 100644 --- a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md +++ b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md @@ -4047,3 +4047,34 @@ - Browser 打开 `http://192.168.3.90:3003/contracts/contract-bbe8d2f3bcc1` 后按未登录状态跳转 `/login`，页面标题为 `AI合同平台`，控制台无前端 error。 - 当前闭环结论： - 合同履约任务、续约提醒、履约节点清单和终止归档证据已完成本地编码、本地测试、Git 推送、远端 Alembic 迁移、前后端容器部署、远端 HTTP 烟测和真实 PostgreSQL API 烟测。 + +## 2026-05-21 Phase15-F Prompt 评测触发与发布门禁产品化闭环 + +- 已按《AI合同起草与审查助手-法律文书能力吸收开发方案-2026-05-20.md》继续推进 Phase15-F。 +- 本轮把之前“脚本生成评测报告、人工填入 Prompt 版本”的方式升级为产品内闭环： + - 后端新增 `POST /api/v1/prompt-assets/{asset_id}/evaluations/run`。 + - 接口会对当前 Prompt 版本执行发布前静态质量门禁，生成 `phase15-prompt-evaluation-report.v1` 报告。 + - 评测结果直接回填 `prompt_asset_versions.evaluation_status / evaluation_score / evaluation_report_json`。 + - 同步更新 `prompt_assets.latest_evaluation_status / latest_evaluation_score / latest_evaluation_report_json`。 + - 评测动作写入审计日志 `run_prompt_asset_evaluation`。 + - 继续复用既有发布门禁：未通过评测或没有评测报告时，`publish` 仍拒绝发布。 +- 前端能力： + - `/prompt-assets` 列表新增“评测”按钮。 + - Prompt 详情卡片新增“运行评测”按钮。 + - 发布按钮在最新评测状态不是 `passed` 时不可点击，避免管理员跳过评测直接发布。 +- 数据库影响： + - 本轮不新增表、不新增 Alembic 迁移。 + - 远程库只要已经到 `20260521_0032_create_prompt_assets.py` 或更高版本即可使用；当前 3.90 已在更高版本 `20260521_0037`。 +- 本轮验证： + - `python -m compileall backend\app\services\prompt_asset_service.py backend\app\api\prompt_assets.py` + - `$env:PYTHONPATH='backend'; python -m unittest backend.tests.test_prompt_asset_service backend.tests.test_prompt_assets_api backend.tests.test_permission_metadata -v` + - `$env:PYTHONPATH='backend'; python -m unittest backend.tests.test_prompt_asset_service backend.tests.test_prompt_assets_api backend.tests.test_prompt_runtime_service backend.tests.test_chat_api backend.tests.test_clause_recommendation_service backend.tests.test_clauses_api backend.tests.test_review_service backend.tests.test_contracts_api backend.tests.test_permission_metadata -v` + - `cd frontend; npx eslint src/services/prompt-asset.ts src/app/prompt-assets/page.tsx src/types/api.ts` + - `cd frontend; npm run build` + - `git diff --check` + - Browser 打开 `http://localhost:3000/prompt-assets`，未登录保护页正常显示，页面标题 `AI合同平台`，控制台错误 0。 +- 当前闭环结论： + - Phase15-F 已从“离线脚本可复跑”进一步形成“页面触发评测 -> 后端生成报告 -> 自动回填当前版本 -> 发布按钮受评测状态约束 -> 审计留痕”的产品化闭环。 +- 剩余边界： + - 本轮没有部署到 3.90；由于无数据库迁移，后续只需部署后端/前端即可在测试服务器使用该按钮。 + - 当前评测接口是发布前静态质量门禁，不等同于调用真实大模型逐条跑完整黄金样本输出；后续如需更强门禁，可继续接入异步任务队列，把 `scripts/run-phase15-prompt-golden-eval.ps1` 或等价服务端评测器的结果自动回填同一字段。