From 25b42f1abd70dd9e1ce8a025be2e98cda0a4bac4 Mon Sep 17 00:00:00 2001 From: yuqei Date: Tue, 23 Jun 2026 00:56:50 +0800 Subject: [PATCH] feat: add ai run quality evaluation --- ...V2-ai-quality-evaluation-mvp-2026-06-23.md | 59 ++++++ .../packages/ai-openapi/openapi.yaml | 29 +++ ...20260623_0007_add_ai_run_quality_fields.py | 31 +++ .../src/yuqei_ai_platform_api/main.py | 186 +++++++++++++++++- .../src/yuqei_ai_platform_api/models.py | 4 + .../src/yuqei_ai_platform_api/repository.py | 41 +++- .../tests/test_platform_data.py | 25 +++ .../tests/test_provider_flow.py | 8 + yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py | 2 + .../src/yuqei_sdk/ai_platform.py | 11 ++ yuqei-ai-sdk-python/tests/test_ai_platform.py | 15 ++ .../src/app/admin/ai-traces/page.tsx | 112 ++++++++++- .../apps/contract-web/src/app/globals.css | 24 ++- 13 files changed, 533 insertions(+), 14 deletions(-) create mode 100644 project-docs/V2-ai-quality-evaluation-mvp-2026-06-23.md create mode 100644 yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0007_add_ai_run_quality_fields.py diff --git a/project-docs/V2-ai-quality-evaluation-mvp-2026-06-23.md b/project-docs/V2-ai-quality-evaluation-mvp-2026-06-23.md new file mode 100644 index 00000000..084d8fac --- /dev/null +++ b/project-docs/V2-ai-quality-evaluation-mvp-2026-06-23.md @@ -0,0 +1,59 @@ +# V2 AI 质量评测 MVP 闭环记录 + +日期：2026-06-23 + +## 本轮目标 + +给每次 AI 调用增加可复盘的质量标签，让管理员在 AI Trace 页直接看到本次回答是否引用依据、是否走 fallback、token/耗时是否异常、知识来源是否不足。 + +## 已落地内容 + +1. AI run 审计表新增质量字段。 + - `quality_status` + - `quality_score` + - `quality_labels_json` + - `quality_summary` + - 新增迁移 `20260623_0007_add_ai_run_quality_fields.py` + +2. 法律问答主链路增加质量评测。 + - 有引用依据：`citation_present` + - 缺少引用依据：`missing_citation` + - 知识来源不足：`source_insufficient` + - 知识来源偏少：`source_limited` + - 知识来源覆盖：`source_covered` + - 使用 fallback：`fallback_used` + - provider 失败：`provider_failed` + - token 缺失或过高：`token_missing` / `token_high` + - 耗时偏高或严重：`latency_high` / `latency_critical` + +3. 质量判定输出。 + - `passed`：没有 error/warning。 + - `needs_review`：存在 warning，需要人工复核。 + - `failed`：存在 error，需要优先处理。 + - `quality_score` MVP 规则：100 分起，error 每项扣 30 分，warning 每项扣 10 分。 + +4. 契约和 SDK 同步。 + - OpenAPI `AiRun` 新增质量字段。 + - Python SDK `AiRunAudit` 新增质量字段。 + - 新增 `AiQualityLabel` SDK 模型。 + +5. AI Trace 后台展示。 + - `/admin/ai-traces` 新增“质量待复核”指标。 + - 调用列表新增质量状态和质量分。 + - 支持按质量状态筛选：全部、质量通过、需要复核、质量失败、未评测。 + - 展开详情新增“质量评测”区块，展示质量摘要和标签明细。 + +## 验收重点 + +- 执行一次法律问答后，`GET /api/v1/audit/ai-runs` 返回质量字段。 +- fallback 回答应标记 `needs_review`，并包含 `fallback_used`。 +- 缺少引用依据时应标记 `failed`。 +- AI Trace 页能按质量状态筛选，并能展开查看每个质量标签的原因。 + +## 建议下一步 + +进入 AI 质量评测规则后台化： + +- 把 token 阈值、耗时阈值、来源数量阈值做成后台可配置。 +- 给质量标签增加人工确认状态，例如“已复核”“误报”“已处理”。 +- 给管理首页增加 AI 质量异常趋势，形成运营级监控。 diff --git a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml index 84fa5ed7..49995be9 100644 --- a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml +++ b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml @@ -1148,6 +1148,23 @@ components: type: array items: type: string + AiQualityLabel: + type: object + required: + - code + - title + - detail + properties: + code: + type: string + severity: + type: string + enum: [info, warning, error] + default: info + title: + type: string + detail: + type: string AiRun: type: object required: @@ -1214,6 +1231,18 @@ components: format: float latency_ms: type: integer + quality_status: + type: string + enum: [unknown, passed, needs_review, failed] + quality_score: + type: integer + quality_labels: + type: array + items: + $ref: "#/components/schemas/AiQualityLabel" + quality_summary: + type: string + nullable: true status: type: string enum: [pending, running, succeeded, fallback, skipped, failed, cancelled] diff --git a/yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0007_add_ai_run_quality_fields.py b/yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0007_add_ai_run_quality_fields.py new file mode 100644 index 00000000..d0f60613 --- /dev/null +++ b/yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0007_add_ai_run_quality_fields.py @@ -0,0 +1,31 @@ +"""add ai run quality fields + +Revision ID: 20260623_0007 +Revises: 20260623_0006 +Create Date: 2026-06-23 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260623_0007" +down_revision: Union[str, None] = "20260623_0006" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("ai_run_audits", sa.Column("quality_status", sa.String(length=32), nullable=False, server_default="unknown")) + op.add_column("ai_run_audits", sa.Column("quality_score", sa.Integer(), nullable=False, server_default="0")) + op.add_column("ai_run_audits", sa.Column("quality_labels_json", sa.Text(), nullable=False, server_default="[]")) + op.add_column("ai_run_audits", sa.Column("quality_summary", sa.Text(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("ai_run_audits", "quality_summary") + op.drop_column("ai_run_audits", "quality_labels_json") + op.drop_column("ai_run_audits", "quality_score") + op.drop_column("ai_run_audits", "quality_status") diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py index 09c7c08b..f0d5dbc3 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py @@ -13,6 +13,7 @@ from yuqei_ai_platform_api.prompts import render_legal_qa_prompt from yuqei_ai_platform_api.providers import ProviderRequest, build_provider_adapter from yuqei_ai_platform_api.repository import ( AiPlatformRepository, + AiQualityLabel, AiRunAudit, KnowledgeChunk, KnowledgeDocument, @@ -249,6 +250,16 @@ def create_app( confidence=0.0, ) audit_status = "failed" + latency_ms = max(0, int((perf_counter() - started_at) * 1000)) + quality_status, quality_score, quality_labels, quality_summary = _evaluate_ai_run_quality( + status=audit_status, + provider_status=provider_response.status, + provider_error=provider_response.error_message, + citations_count=len(response.citations), + retrieved_sources_count=len(retrieval_sources), + total_tokens=provider_response.total_tokens, + latency_ms=latency_ms, + ) store.add_ai_run_audit( AiRunAudit( id=f"run-{uuid4().hex[:12]}", @@ -271,7 +282,11 @@ def create_app( completion_tokens=provider_response.completion_tokens, total_tokens=provider_response.total_tokens, estimated_cost=provider_response.estimated_cost, - latency_ms=max(0, int((perf_counter() - started_at) * 1000)), + latency_ms=latency_ms, + quality_status=quality_status, + quality_score=quality_score, + quality_labels=quality_labels, + quality_summary=quality_summary, status=audit_status, ) ) @@ -289,6 +304,175 @@ def _preview_text(value: str, *, limit: int = 4000) -> str: return f"{value[:limit]}..." +def _evaluate_ai_run_quality( + *, + status: str, + provider_status: str | None, + provider_error: str | None, + citations_count: int, + retrieved_sources_count: int, + total_tokens: int, + latency_ms: int, +) -> tuple[str, int, list[AiQualityLabel], str]: + labels: list[AiQualityLabel] = [] + + if citations_count <= 0: + labels.append( + AiQualityLabel( + code="missing_citation", + severity="error", + title="Missing citation", + detail="The answer did not return any legal citation or evidence source.", + ) + ) + else: + labels.append( + AiQualityLabel( + code="citation_present", + severity="info", + title="Citation present", + detail=f"The answer returned {citations_count} citation(s).", + ) + ) + + if retrieved_sources_count <= 0: + labels.append( + AiQualityLabel( + code="source_insufficient", + severity="warning", + title="Source insufficient", + detail="No retrieved knowledge chunk source was recorded for this run.", + ) + ) + elif retrieved_sources_count < 2: + labels.append( + AiQualityLabel( + code="source_limited", + severity="warning", + title="Source limited", + detail="Only one retrieved source was recorded; reviewer may need to verify coverage.", + ) + ) + else: + labels.append( + AiQualityLabel( + code="source_covered", + severity="info", + title="Source covered", + detail=f"{retrieved_sources_count} retrieved sources were recorded.", + ) + ) + + if status == "failed": + labels.append( + AiQualityLabel( + code="run_failed", + severity="error", + title="Run failed", + detail="The AI run did not complete successfully.", + ) + ) + elif status == "fallback": + labels.append( + AiQualityLabel( + code="fallback_used", + severity="warning", + title="Fallback used", + detail="The answer used the local fallback path instead of a successful provider response.", + ) + ) + + if provider_status == "failed": + labels.append( + AiQualityLabel( + code="provider_failed", + severity="warning", + title="Provider failed", + detail=provider_error or "The model provider returned a failure status.", + ) + ) + elif provider_status == "succeeded": + labels.append( + AiQualityLabel( + code="provider_succeeded", + severity="info", + title="Provider succeeded", + detail="The model provider returned a successful response.", + ) + ) + + if total_tokens <= 0: + labels.append( + AiQualityLabel( + code="token_missing", + severity="warning", + title="Token usage missing", + detail="Token usage was not reported for this run.", + ) + ) + elif total_tokens > 8000: + labels.append( + AiQualityLabel( + code="token_high", + severity="warning", + title="Token usage high", + detail=f"The run used {total_tokens} tokens, which is above the MVP review threshold.", + ) + ) + else: + labels.append( + AiQualityLabel( + code="token_normal", + severity="info", + title="Token usage normal", + detail=f"The run used {total_tokens} tokens.", + ) + ) + + if latency_ms >= 30000: + labels.append( + AiQualityLabel( + code="latency_critical", + severity="error", + title="Latency critical", + detail=f"The run took {latency_ms} ms.", + ) + ) + elif latency_ms >= 15000: + labels.append( + AiQualityLabel( + code="latency_high", + severity="warning", + title="Latency high", + detail=f"The run took {latency_ms} ms.", + ) + ) + else: + labels.append( + AiQualityLabel( + code="latency_normal", + severity="info", + title="Latency normal", + detail=f"The run took {latency_ms} ms.", + ) + ) + + error_count = sum(1 for label in labels if label.severity == "error") + warning_count = sum(1 for label in labels if label.severity == "warning") + score = max(0, 100 - error_count * 30 - warning_count * 10) + if error_count: + quality_status = "failed" + elif warning_count: + quality_status = "needs_review" + else: + quality_status = "passed" + summary = ( + f"{quality_status}: {error_count} error(s), " + f"{warning_count} warning(s), score {score}." + ) + return quality_status, score, labels, summary + + def _search_citation_documents( repository: AiPlatformRepository, request: LegalQaRequest, diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py index 83070b5f..7b24501b 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py @@ -124,6 +124,10 @@ class AiRunAuditModel(Base): total_tokens: Mapped[int] = mapped_column(Integer, default=0) estimated_cost: Mapped[float] = mapped_column(Float, default=0.0) latency_ms: Mapped[int] = mapped_column(Integer, default=0) + quality_status: Mapped[str] = mapped_column(String(32), default="unknown") + quality_score: Mapped[int] = mapped_column(Integer, default=0) + quality_labels_json: Mapped[str] = mapped_column(Text, default="[]") + quality_summary: Mapped[str | None] = mapped_column(Text, nullable=True) status: Mapped[str] = mapped_column(String(32), default="succeeded") created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py index 5740ed22..3afbdb19 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py @@ -190,6 +190,13 @@ class RetrievalSource(BaseModel): matched_terms: list[str] = Field(default_factory=list) +class AiQualityLabel(BaseModel): + code: str + severity: str = "info" + title: str + detail: str + + class AiRunAudit(BaseModel): id: str operation: str @@ -212,6 +219,10 @@ class AiRunAudit(BaseModel): total_tokens: int = 0 estimated_cost: float = 0.0 latency_ms: int = 0 + quality_status: str = "unknown" + quality_score: int = 0 + quality_labels: list[AiQualityLabel] = Field(default_factory=list) + quality_summary: str | None = None status: str = "succeeded" created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) @@ -962,6 +973,12 @@ class SqlAlchemyAiPlatformRepository: total_tokens=audit.total_tokens, estimated_cost=audit.estimated_cost, latency_ms=audit.latency_ms, + quality_status=audit.quality_status, + quality_score=audit.quality_score, + quality_labels_json=_encode_json_list( + [label.model_dump() for label in audit.quality_labels] + ), + quality_summary=audit.quality_summary, status=audit.status, created_at=audit.created_at, ) @@ -1170,23 +1187,31 @@ def _ai_run_audit_from_model(row: AiRunAuditModel) -> AiRunAudit: operation=row.operation, provider_config_id=row.provider_config_id, prompt_template_id=row.prompt_template_id, - provider=row.provider, - model=row.model, - provider_status=row.provider_status, - provider_error=row.provider_error, - prompt_preview=row.prompt_preview, - question=row.question, - answer=row.answer, + provider=row.provider, + model=row.model, + provider_status=row.provider_status, + provider_error=row.provider_error, + prompt_preview=row.prompt_preview, + question=row.question, + answer=row.answer, citations_count=row.citations_count, retrieval_query=row.retrieval_query, retrieval_strategy=row.retrieval_strategy, retrieved_chunk_ids=_decode_json_list(row.retrieved_chunk_ids_json), - retrieved_sources=[RetrievalSource.model_validate(item) for item in _decode_json_list(row.retrieved_sources_json)], + retrieved_sources=[ + RetrievalSource.model_validate(item) for item in _decode_json_list(row.retrieved_sources_json) + ], prompt_tokens=row.prompt_tokens, completion_tokens=row.completion_tokens, total_tokens=row.total_tokens, estimated_cost=row.estimated_cost, latency_ms=row.latency_ms, + quality_status=row.quality_status, + quality_score=row.quality_score, + quality_labels=[ + AiQualityLabel.model_validate(item) for item in _decode_json_list(row.quality_labels_json) + ], + quality_summary=row.quality_summary, status=row.status, created_at=_ensure_utc(row.created_at), ) diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py index 99f1dbe8..92037b99 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py @@ -327,6 +327,10 @@ def test_legal_qa_uses_chunk_search_and_records_audit() -> None: assert audits[0]["retrieved_sources"][0]["reference"] == "Lease-001" assert audits[0]["retrieved_sources"][0]["score"] > 0 assert audits[0]["status"] == "fallback" + assert audits[0]["quality_status"] == "needs_review" + assert audits[0]["quality_score"] == 70 + quality_codes = {label["code"] for label in audits[0]["quality_labels"]} + assert {"citation_present", "source_limited", "fallback_used", "token_missing"} <= quality_codes def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: @@ -407,6 +411,23 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: total_tokens=20, estimated_cost=0.00028, latency_ms=12, + quality_status="passed", + quality_score=100, + quality_labels=[ + { + "code": "citation_present", + "severity": "info", + "title": "Citation present", + "detail": "The answer returned 1 citation.", + }, + { + "code": "source_covered", + "severity": "info", + "title": "Source covered", + "detail": "Sufficient retrieved sources were recorded.", + }, + ], + quality_summary="passed: 0 error(s), 0 warning(s), score 100.", status="succeeded", ) ) @@ -438,3 +459,7 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: assert audit.estimated_cost == 0.00028 assert audit.retrieved_chunk_ids == ["chunk-test-001"] assert audit.retrieved_sources[0].reference == "Article 703" + assert audit.quality_status == "passed" + assert audit.quality_score == 100 + assert audit.quality_labels[0].code == "citation_present" + assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100." diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_provider_flow.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_provider_flow.py index f5f69f69..2001573a 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_provider_flow.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_provider_flow.py @@ -78,6 +78,10 @@ def test_legal_qa_uses_provider_answer_when_enabled(monkeypatch) -> None: assert audit["completion_tokens"] == 50 assert audit["total_tokens"] == 150 assert audit["estimated_cost"] == 0.0002 + assert audit["quality_status"] == "needs_review" + assert audit["quality_score"] == 90 + quality_codes = {label["code"] for label in audit["quality_labels"]} + assert {"citation_present", "source_limited", "provider_succeeded", "token_normal", "latency_normal"} <= quality_codes def test_provider_failure_falls_back_to_rule_answer_and_audit_status() -> None: @@ -116,6 +120,10 @@ def test_provider_failure_falls_back_to_rule_answer_and_audit_status() -> None: assert audit["provider_error"] == "Missing provider API key." assert "试用期如何约定" in audit["prompt_preview"] assert audit["total_tokens"] == 0 + assert audit["quality_status"] == "needs_review" + assert audit["quality_score"] == 60 + quality_codes = {label["code"] for label in audit["quality_labels"]} + assert {"citation_present", "fallback_used", "provider_failed", "token_missing"} <= quality_codes def test_openai_compatible_provider_extracts_chat_answer() -> None: diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py index 5c97e0a5..32842712 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py @@ -1,5 +1,6 @@ from yuqei_sdk.ai_platform import ( AiPlatformClient, + AiQualityLabel, AiRunAudit, KnowledgeChunk, KnowledgeDocument, @@ -24,6 +25,7 @@ from yuqei_sdk.context import RequestContext __all__ = [ "AiPlatformClient", + "AiQualityLabel", "AiRunAudit", "KnowledgeChunk", "KnowledgeDocument", diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py index 0ef3f384..e8284b20 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py @@ -188,6 +188,13 @@ class RetrievalSource(BaseModel): matched_terms: list[str] = Field(default_factory=list) +class AiQualityLabel(BaseModel): + code: str + severity: str = "info" + title: str + detail: str + + class AiRunAudit(BaseModel): id: str operation: str @@ -210,6 +217,10 @@ class AiRunAudit(BaseModel): total_tokens: int = 0 estimated_cost: float = 0.0 latency_ms: int = 0 + quality_status: str = "unknown" + quality_score: int = 0 + quality_labels: list[AiQualityLabel] = Field(default_factory=list) + quality_summary: str | None = None status: str created_at: str diff --git a/yuqei-ai-sdk-python/tests/test_ai_platform.py b/yuqei-ai-sdk-python/tests/test_ai_platform.py index e89ab08d..923161f1 100644 --- a/yuqei-ai-sdk-python/tests/test_ai_platform.py +++ b/yuqei-ai-sdk-python/tests/test_ai_platform.py @@ -382,6 +382,17 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: "total_tokens": 20, "estimated_cost": 0.00028, "latency_ms": 12, + "quality_status": "passed", + "quality_score": 100, + "quality_labels": [ + { + "code": "citation_present", + "severity": "info", + "title": "Citation present", + "detail": "The answer returned 1 citation.", + } + ], + "quality_summary": "passed: 0 error(s), 0 warning(s), score 100.", "status": "succeeded", "created_at": "2026-06-22T00:00:00Z", } @@ -452,4 +463,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: assert audit.estimated_cost == 0.00028 assert audit.retrieved_chunk_ids == ["chunk-1"] assert audit.retrieved_sources[0].reference == "Article 703" + assert audit.quality_status == "passed" + assert audit.quality_score == 100 + assert audit.quality_labels[0].code == "citation_present" + assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100." assert "/api/v1/audit/ai-runs" in seen_paths diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx index 15bdb94b..f0c6e035 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx @@ -10,6 +10,7 @@ import { FileText, RefreshCcw, Search, + ShieldCheck, Sigma } from "lucide-react"; @@ -29,6 +30,13 @@ type RetrievalSource = { matched_terms?: string[]; }; +type AiQualityLabel = { + code: string; + severity?: string; + title: string; + detail: string; +}; + type AiRunAudit = { id: string; operation: string; @@ -51,6 +59,10 @@ type AiRunAudit = { total_tokens?: number; estimated_cost?: number; latency_ms?: number; + quality_status?: string; + quality_score?: number; + quality_labels?: AiQualityLabel[]; + quality_summary?: string | null; status: string; created_at: string; }; @@ -80,9 +92,48 @@ const statusTone: Record = { cancelled: "tag-danger" }; +const qualityStatusLabels: Record = { + passed: "质量通过", + needs_review: "需要复核", + failed: "质量失败", + unknown: "未评测" +}; + +const qualityStatusTone: Record = { + passed: "tag-success", + needs_review: "tag-warning", + failed: "tag-danger", + unknown: "" +}; + +const qualityLabelText: Record = { + citation_present: { title: "已引用依据", detail: "回答已返回法条或证据来源。" }, + missing_citation: { title: "缺少依据", detail: "回答未返回法条或证据来源，需要复核。" }, + source_covered: { title: "来源覆盖", detail: "本次回答记录了多个知识命中来源。" }, + source_limited: { title: "来源偏少", detail: "本次回答仅记录一个知识命中来源，建议复核覆盖面。" }, + source_insufficient: { title: "来源不足", detail: "本次回答没有记录知识命中来源。" }, + fallback_used: { title: "使用回退", detail: "本次回答走了本地回退链路。" }, + run_failed: { title: "调用失败", detail: "本次 AI 调用未成功完成。" }, + provider_succeeded: { title: "模型成功", detail: "模型服务返回成功响应。" }, + provider_failed: { title: "模型失败", detail: "模型服务返回失败状态。" }, + token_normal: { title: "Token 正常", detail: "本次调用 token 用量在阈值内。" }, + token_missing: { title: "Token 缺失", detail: "本次调用没有返回 token 用量。" }, + token_high: { title: "Token 偏高", detail: "本次调用 token 用量超过 MVP 复核阈值。" }, + latency_normal: { title: "耗时正常", detail: "本次调用耗时在阈值内。" }, + latency_high: { title: "耗时偏高", detail: "本次调用耗时超过 15 秒。" }, + latency_critical: { title: "耗时严重", detail: "本次调用耗时超过 30 秒。" } +}; + +const severityTone: Record = { + info: "tag-primary", + warning: "tag-warning", + error: "tag-danger" +}; + export default function AiTracesPage() { const [state, setState] = useState({ status: "loading" }); const [statusFilter, setStatusFilter] = useState("all"); + const [qualityFilter, setQualityFilter] = useState("all"); const [keyword, setKeyword] = useState(""); async function loadRuns() { @@ -111,6 +162,8 @@ export default function AiTracesPage() { const normalizedKeyword = keyword.trim().toLowerCase(); return runs.filter((run) => { const matchesStatus = statusFilter === "all" || run.status === statusFilter; + const qualityStatus = run.quality_status ?? "unknown"; + const matchesQuality = qualityFilter === "all" || qualityStatus === qualityFilter; const haystack = [ run.id, run.operation, @@ -120,20 +173,27 @@ export default function AiTracesPage() { run.answer, run.provider_error, run.retrieval_query, + run.quality_status, + run.quality_summary, + ...(run.quality_labels ?? []).map((label) => `${label.code} ${label.title} ${label.detail}`), ...(run.retrieved_sources ?? []).map((source) => `${source.title} ${source.reference}`) ].join("\n").toLowerCase(); - return matchesStatus && (!normalizedKeyword || haystack.includes(normalizedKeyword)); + return matchesStatus && matchesQuality && (!normalizedKeyword || haystack.includes(normalizedKeyword)); }); - }, [keyword, runs, statusFilter]); + }, [keyword, qualityFilter, runs, statusFilter]); const metrics = useMemo(() => { const totalTokens = runs.reduce((sum, run) => sum + (run.total_tokens ?? 0), 0); const estimatedCost = runs.reduce((sum, run) => sum + (run.estimated_cost ?? 0), 0); const failedCount = runs.filter((run) => run.status === "failed" || run.provider_status === "failed").length; + const qualityReviewCount = runs.filter((run) => { + const qualityStatus = run.quality_status ?? "unknown"; + return qualityStatus === "needs_review" || qualityStatus === "failed"; + }).length; const averageLatency = runs.length ? Math.round(runs.reduce((sum, run) => sum + (run.latency_ms ?? 0), 0) / runs.length) : 0; - return { totalTokens, estimatedCost, failedCount, averageLatency }; + return { totalTokens, estimatedCost, failedCount, qualityReviewCount, averageLatency }; }, [runs]); return ( @@ -153,6 +213,7 @@ export default function AiTracesPage() { } label="最近调用" value={String(runs.length)} /> } label="总 token" value={formatInteger(metrics.totalTokens)} /> } label="平均耗时" value={`${metrics.averageLatency} ms`} /> + } label="质量待复核" value={String(metrics.qualityReviewCount)} /> } label="异常调用" value={String(metrics.failedCount)} /> @@ -182,6 +243,17 @@ export default function AiTracesPage() { + {state.status === "loading" &&
正在加载 AI 调用审计...
} @@ -220,6 +292,9 @@ function Metric({ icon, label, value }: { icon: ReactNode; label: string; value: function TraceItem({ run }: { run: AiRunAudit }) { const statusClass = statusTone[run.status] ?? ""; const providerStatusClass = statusTone[run.provider_status ?? ""] ?? ""; + const qualityStatus = run.quality_status ?? "unknown"; + const qualityClass = qualityStatusTone[qualityStatus] ?? ""; + const qualityLabels = run.quality_labels ?? []; return (
@@ -227,7 +302,12 @@ function TraceItem({ run }: { run: AiRunAudit }) {
{run.question || run.operation} - {statusLabels[run.status] ?? run.status} +
+ + {qualityStatusLabels[qualityStatus] ?? qualityStatus} · {run.quality_score ?? 0} + + {statusLabels[run.status] ?? run.status} +
{formatDate(run.created_at)} @@ -271,6 +351,30 @@ function TraceItem({ run }: { run: AiRunAudit }) {
)} +
+

质量评测

+
+ 质量状态{qualityStatusLabels[qualityStatus] ?? qualityStatus} + 质量分{run.quality_score ?? 0} + 摘要{run.quality_summary || "暂无质量摘要"} +
+ {qualityLabels.length === 0 ? ( +

暂无质量标签。

+ ) : ( +
+ {qualityLabels.map((label) => { + const text = qualityLabelText[label.code]; + const severityClass = severityTone[label.severity ?? "info"] ?? ""; + return ( +
+ {text?.title ?? label.title} + {text?.detail ?? label.detail} +
+ ); + })} +
+ )} +

运行状态

diff --git a/yuqei-contract-platform/apps/contract-web/src/app/globals.css b/yuqei-contract-platform/apps/contract-web/src/app/globals.css index 019bc6b9..ae524e9a 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/globals.css +++ b/yuqei-contract-platform/apps/contract-web/src/app/globals.css @@ -237,7 +237,7 @@ select { .metric-row { display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; margin-bottom: 16px; } @@ -534,6 +534,13 @@ select { gap: 8px; } +.trace-summary-badges { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 6px; +} + .trace-meta { display: flex; flex-wrap: wrap; @@ -603,6 +610,21 @@ select { overflow-wrap: anywhere; } +.quality-labels { + display: grid; + gap: 8px; +} + +.quality-label { + display: grid; + grid-template-columns: max-content minmax(0, 1fr); + gap: 8px 10px; + align-items: center; + color: var(--muted); + font-size: 13px; + line-height: 1.5; +} + .compact-list-item { padding: 12px; }