diff --git a/backend/app/services/operation_kpi_service.py b/backend/app/services/operation_kpi_service.py index 9cbcb9c1..c082b972 100644 --- a/backend/app/services/operation_kpi_service.py +++ b/backend/app/services/operation_kpi_service.py @@ -6,6 +6,7 @@ from fastapi import HTTPException, status from sqlalchemy import func from sqlalchemy.orm import Session +from app.models.ai_quality_eval_run import AiQualityEvalRun from app.models.audit_log import AuditLog from app.models.clause_recommendation import ClauseRecommendation from app.models.commercial_order import CommercialOrder @@ -49,6 +50,7 @@ class OperationKpiService: "risk_action_top_items": self._build_risk_action_top_items(tenant_id), "clause_recommendation_funnel": self._build_clause_recommendation_funnel(tenant_id, None), "clause_recommendation_top_items": self._build_clause_recommendation_top_items(tenant_id), + "quality_eval_summary": self._build_quality_eval_summary(tenant_id, None), "legal_citation_summary": self._build_legal_citation_summary(tenant_id, None), "conversion_funnel": self._build_conversion_funnel(), "overage_conversion": self._build_overage_conversion(current_user, billing_summary), @@ -83,6 +85,7 @@ class OperationKpiService: ) clause_actions = self._build_clause_action_summary(tenant_id, start_at) recommendation_funnel = self._build_clause_recommendation_funnel(tenant_id, start_at) + quality_summary = self._build_quality_eval_summary(tenant_id, start_at) legal_citations = self._build_legal_citation_summary(tenant_id, start_at) return { @@ -107,6 +110,12 @@ class OperationKpiService: "clause_recommendation_ignored_count": recommendation_funnel["ignored_count"], "clause_recommendation_adoption_rate_percent": recommendation_funnel["adoption_rate_percent"], "clause_recommendation_action_rate_percent": recommendation_funnel["action_rate_percent"], + "quality_eval_run_count": quality_summary["total_runs"], + "quality_eval_passed_count": quality_summary["passed_runs"], + "quality_eval_failed_count": quality_summary["failed_runs"], + "quality_eval_running_count": quality_summary["running_runs"], + "quality_eval_pass_rate_percent": quality_summary["pass_rate_percent"], + "quality_eval_average_score_percent": quality_summary["average_score_percent"], "legal_reference_count": legal_citations["total_reference_count"], "legal_reference_verified_rate_percent": legal_citations["verified_reference_rate_percent"], }, @@ -252,6 +261,39 @@ class OperationKpiService: for row in rows ] + def _build_quality_eval_summary(self, tenant_id: str, start_at: Optional[datetime]) -> dict[str, Any]: + query = self.db.query(AiQualityEvalRun).filter(AiQualityEvalRun.tenant_id == tenant_id) + if start_at is not None: + query = query.filter(AiQualityEvalRun.created_at >= start_at) + runs = query.all() + + total_runs = len(runs) + passed_runs = sum(1 for item in runs if item.status == "passed") + failed_runs = sum(1 for item in runs if item.status == "failed") + running_runs = sum(1 for item in runs if item.status in {"running", "pending"}) + quality_scores = [float(item.quality_score) for item in runs if item.quality_score is not None] + total_samples = sum(int(item.sample_count or 0) for item in runs) + passed_samples = sum(int(item.passed_samples or 0) for item in runs) + average_score_percent = self._percent(sum(quality_scores), len(quality_scores)) if quality_scores else 0.0 + + latest_run = max(runs, key=lambda item: item.created_at, default=None) + return { + "total_runs": total_runs, + "passed_runs": passed_runs, + "failed_runs": failed_runs, + "running_runs": running_runs, + "pass_rate_percent": self._percent(passed_runs, total_runs), + "average_score_percent": round(average_score_percent, 2), + "total_samples": total_samples, + "passed_samples": passed_samples, + "sample_pass_rate_percent": self._percent(passed_samples, total_samples), + "latest_run_id": latest_run.id if latest_run is not None else None, + "latest_subject_name": latest_run.subject_name if latest_run is not None else None, + "latest_eval_domain": latest_run.eval_domain if latest_run is not None else None, + "latest_status": latest_run.status if latest_run is not None else None, + "latest_finished_at": self._iso(latest_run.finished_at if latest_run is not None else None), + } + def _build_legal_citation_summary(self, tenant_id: str, start_at: Optional[datetime]) -> dict[str, Any]: review_results_query = ( self.db.query(ReviewResult) diff --git a/backend/tests/test_operation_kpi_service.py b/backend/tests/test_operation_kpi_service.py index 9e311388..d132ecff 100644 --- a/backend/tests/test_operation_kpi_service.py +++ b/backend/tests/test_operation_kpi_service.py @@ -8,6 +8,7 @@ from sqlalchemy.orm import sessionmaker import app.models # noqa: F401 from app.db import Base +from app.models.ai_quality_eval_run import AiQualityEvalRun from app.models.audit_log import AuditLog from app.models.clause_recommendation import ClauseRecommendation from app.models.commercial_order import CommercialOrder @@ -362,6 +363,48 @@ class OperationKpiServiceTestCase(unittest.TestCase): ) self.db.add_all( [ + AiQualityEvalRun( + id="qualityeval-1", + tenant_id="tenant-001", + source_kind="prompt_asset_evaluation_run", + source_run_id="prompteval-1", + eval_domain="prompt", + subject_type="prompt_asset", + subject_id="prompt-1", + subject_key="contract_generation_default", + subject_name="合同文书生成默认 Prompt", + suite_key="phase13_generation", + status="passed", + sample_count=8, + passed_samples=8, + failed_samples=0, + pass_rate=1.0, + quality_score=0.96, + created_at=now, + started_at=now, + finished_at=now, + ), + AiQualityEvalRun( + id="qualityeval-2", + tenant_id="tenant-001", + source_kind="prompt_asset_evaluation_run", + source_run_id="prompteval-2", + eval_domain="prompt", + subject_type="prompt_asset", + subject_id="prompt-2", + subject_key="contract_review_default", + subject_name="合同审查默认 Prompt", + suite_key="phase13_review", + status="failed", + sample_count=8, + passed_samples=5, + failed_samples=3, + pass_rate=0.625, + quality_score=0.72, + created_at=now, + started_at=now, + finished_at=now, + ), CommercialOrder( id="order-activated", tenant_id="tenant-001", @@ -421,6 +464,16 @@ class OperationKpiServiceTestCase(unittest.TestCase): self.assertEqual(dashboard["clause_recommendation_funnel"]["adopted_count"], 2) self.assertEqual(dashboard["clause_recommendation_funnel"]["adoption_rate_percent"], 50.0) self.assertEqual(dashboard["clause_recommendation_funnel"]["action_rate_percent"], 75.0) + self.assertEqual(monthly["metrics"]["quality_eval_run_count"], 2) + self.assertEqual(monthly["metrics"]["quality_eval_passed_count"], 1) + self.assertEqual(monthly["metrics"]["quality_eval_failed_count"], 1) + self.assertEqual(monthly["metrics"]["quality_eval_running_count"], 0) + self.assertEqual(monthly["metrics"]["quality_eval_pass_rate_percent"], 50.0) + self.assertEqual(monthly["metrics"]["quality_eval_average_score_percent"], 84.0) + self.assertEqual(dashboard["quality_eval_summary"]["total_runs"], 2) + self.assertEqual(dashboard["quality_eval_summary"]["passed_runs"], 1) + self.assertEqual(dashboard["quality_eval_summary"]["failed_runs"], 1) + self.assertEqual(dashboard["quality_eval_summary"]["sample_pass_rate_percent"], 81.25) self.assertEqual(dashboard["clause_recommendation_top_items"][0]["action_count"], 1) self.assertEqual(dashboard["legal_citation_summary"]["total_reference_count"], 3) self.assertEqual(dashboard["legal_citation_summary"]["verified_reference_rate_percent"], 100.0) diff --git a/backend/tests/test_operation_kpis_api.py b/backend/tests/test_operation_kpis_api.py index b4bc9082..47f03a6d 100644 --- a/backend/tests/test_operation_kpis_api.py +++ b/backend/tests/test_operation_kpis_api.py @@ -42,6 +42,12 @@ class StubOperationKpiService: "clause_recommendation_ignored_count": 1, "clause_recommendation_adoption_rate_percent": 50.0, "clause_recommendation_action_rate_percent": 75.0, + "quality_eval_run_count": 3, + "quality_eval_passed_count": 2, + "quality_eval_failed_count": 1, + "quality_eval_running_count": 0, + "quality_eval_pass_rate_percent": 66.67, + "quality_eval_average_score_percent": 91.5, "legal_reference_count": 3, "legal_reference_verified_rate_percent": 100, }, @@ -70,6 +76,22 @@ class StubOperationKpiService: "action_count": 1, } ], + "quality_eval_summary": { + "total_runs": 3, + "passed_runs": 2, + "failed_runs": 1, + "running_runs": 0, + "pass_rate_percent": 66.67, + "average_score_percent": 91.5, + "total_samples": 24, + "passed_samples": 21, + "sample_pass_rate_percent": 87.5, + "latest_run_id": "qualityeval-123", + "latest_subject_name": "合同文书生成默认 Prompt", + "latest_eval_domain": "prompt", + "latest_status": "passed", + "latest_finished_at": "2026-05-21T00:00:01+00:00", + }, "legal_citation_summary": {"total_reference_count": 3, "verified_reference_rate_percent": 100}, "conversion_funnel": {"trial_or_free_tenants": 1, "active_paid_tenants": 1, "plan_distribution": []}, "overage_conversion": {"current_exceeded_count": 2, "pending_overage_order_count": 1}, @@ -95,6 +117,7 @@ class OperationKpisApiTestCase(unittest.TestCase): self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["data"]["periods"][0]["metrics"]["ai_usage_count"], 7) + self.assertEqual(response.json()["data"]["quality_eval_summary"]["total_runs"], 3) self.assertEqual(response.json()["data"]["top_templates"][0]["template_id"], "tpl-1") def test_member_cannot_get_operation_kpi_dashboard(self): diff --git a/frontend/src/app/operation-kpis/page.tsx b/frontend/src/app/operation-kpis/page.tsx index 815481e9..b7d448a4 100644 --- a/frontend/src/app/operation-kpis/page.tsx +++ b/frontend/src/app/operation-kpis/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { BarChartOutlined, PayCircleOutlined, ReloadOutlined, RiseOutlined, SafetyCertificateOutlined } from "@ant-design/icons"; +import { BarChartOutlined, ExperimentOutlined, PayCircleOutlined, ReloadOutlined, RiseOutlined, SafetyCertificateOutlined } from "@ant-design/icons"; import { Alert, Button, Card, Empty, Progress, Segmented, Space, Statistic, Table, Tag, Typography, message } from "antd"; import type { ColumnsType } from "antd/es/table"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -254,6 +254,7 @@ export default function OperationKpisPage() { const overage = dashboard?.overage_conversion; const clauseSummary = dashboard?.clause_action_summary; const recommendationFunnel = dashboard?.clause_recommendation_funnel; + const qualitySummary = dashboard?.quality_eval_summary; return (
@@ -318,11 +319,46 @@ export default function OperationKpisPage() { prefix={} /> + + } /> + + + } /> + + + +
+ + +
+ + + + + + + + + + + + +
+ + + 最近样本：{qualitySummary?.latest_subject_name || "-"} / {qualitySummary?.latest_eval_domain || "-"} / {qualitySummary?.latest_status || "-"} /{" "} + {formatDateTime(qualitySummary?.latest_finished_at)} + + + +
diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 6262b9a7..15486a8c 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -810,6 +810,12 @@ export type OperationKpiPeriodMetrics = { clause_recommendation_ignored_count?: number; clause_recommendation_adoption_rate_percent?: number; clause_recommendation_action_rate_percent?: number; + quality_eval_run_count?: number; + quality_eval_passed_count?: number; + quality_eval_failed_count?: number; + quality_eval_running_count?: number; + quality_eval_pass_rate_percent?: number; + quality_eval_average_score_percent?: number; legal_reference_count: number; legal_reference_verified_rate_percent: number; }; @@ -926,6 +932,23 @@ export type OperationOverageConversion = { note: string; }; +export type OperationQualityEvalSummary = { + total_runs: number; + passed_runs: number; + failed_runs: number; + running_runs: number; + pass_rate_percent: number; + average_score_percent: number; + total_samples: number; + passed_samples: number; + sample_pass_rate_percent: number; + latest_run_id: string | null; + latest_subject_name: string | null; + latest_eval_domain: string | null; + latest_status: string | null; + latest_finished_at: string | null; +}; + export type OperationKpiDashboard = { scope: { tenant_id: string; @@ -940,6 +963,7 @@ export type OperationKpiDashboard = { risk_action_top_items: OperationRiskActionTopItem[]; clause_recommendation_funnel?: OperationClauseRecommendationFunnel; clause_recommendation_top_items?: OperationClauseRecommendationTopItem[]; + quality_eval_summary: OperationQualityEvalSummary; legal_citation_summary: OperationLegalCitationSummary; conversion_funnel: OperationConversionFunnel; overage_conversion: OperationOverageConversion; diff --git a/project-docs/AI合同起草与审查助手-法律文书能力吸收开发方案-代码对齐检查-2026-05-21.md b/project-docs/AI合同起草与审查助手-法律文书能力吸收开发方案-代码对齐检查-2026-05-21.md index cb19779a..5ef67a27 100644 --- a/project-docs/AI合同起草与审查助手-法律文书能力吸收开发方案-代码对齐检查-2026-05-21.md +++ b/project-docs/AI合同起草与审查助手-法律文书能力吸收开发方案-代码对齐检查-2026-05-21.md @@ -43,7 +43,7 @@ | Phase15-E | Prompt 资产治理 | `/prompt-assets` 支持 Prompt 类型、版本、评测、发布、下线、回滚和运行证据 | 已闭环 | | Phase15-F | 黄金样本与质量评测 | 已有 Prompt 评测运行记录、逐条黄金样本报告、发布前门禁、`/prompt-assets` 治理入口和 `/quality-evals` 独立质量评测中心 | 已闭环 | | Phase15-G | 法规变更影响分析 | 已有法条变更入队、结构化和语义影响扫描、影响项、页面、后台 worker 和 Docker 服务 | 已闭环 | -| Phase15-H | 运营 KPI 与商业化增强 | `/operation-kpis` 已展示用量、文书生成、审查、Chat、模板、条款采纳、试用和转化指标；计费、发票状态、销售跟进已有业务闭环 | 主线已闭环 | +| Phase15-H | 运营 KPI 与商业化增强 | `/operation-kpis` 已展示用量、文书生成、审查、Chat、模板、条款采纳、试用、转化和质量评测指标；计费、发票状态、销售跟进已有业务闭环 | 主线已闭环 | ## 四、代码证据 @@ -157,7 +157,7 @@ Phase15-C、Phase15-F、Phase15-G 已完成提交、推送和 `192.168.3.90` 真 - 数据来源：`prompt_assets`、`prompt_asset_versions`、`prompt_asset_evaluation_runs`。 - 执行动作：复用 Prompt 资产评测接口重新运行黄金样本评测。 -当前仍未单独新增通用 `ai_quality_eval_runs/items` 表。这个表属于后续“跨 Prompt、跨业务对象的统一质量评测域”增强，不影响 Phase15-F 的产品入口闭环。 +当前已单独新增通用 `ai_quality_eval_runs/items` 表。`/quality-evals` 已从通用评测域读取数据，Prompt 评测运行也会旁路沉淀到统一质量域。 ### 6.3 专业级富文本增强 @@ -252,7 +252,7 @@ Generating static pages (40/40) 建议下一步按生产增强优先级推进： 1. 统一文档命名，把法规影响分析的接口名称固定为 `/legal-change-impacts`，避免后续误认为还缺 `/regulation-changes`。 -2. 如未来要做更宽的统一质量平台，再补通用 `ai_quality_eval_runs/items` 和非 Prompt 质量评测对象。 +2. 如未来要做更宽的统一质量平台，再继续补审查、文书生成、法律问答等非 Prompt 质量评测对象。 3. 如进入生产体验增强，再做专业富文本库、字段级 diff、全量法规向量化、真实微信或支付宝支付、电子签章和小程序等独立阶段。 按当前代码状态判断，《法律文书能力吸收开发方案》的主线开发没有跑偏，已经基本完成。`/quality-evals` 独立入口补齐后，真正剩下的是若干生产级增强，而不是 Phase15 主线功能缺口。