feat: add ai run quality evaluation

2026-06-23 00:56:50 +08:00
parent 81977ead8f
commit 25b42f1abd
13 changed files with 533 additions and 14 deletions
@@ -0,0 +1,59 @@
+# V2 AI 质量评测 MVP 闭环记录
+
+日期：2026-06-23
+
+## 本轮目标
+
+给每次 AI 调用增加可复盘的质量标签，让管理员在 AI Trace 页直接看到本次回答是否引用依据、是否走 fallback、token/耗时是否异常、知识来源是否不足。
+
+## 已落地内容
+
+1. AI run 审计表新增质量字段。
+   - `quality_status`
+   - `quality_score`
+   - `quality_labels_json`
+   - `quality_summary`
+   - 新增迁移 `20260623_0007_add_ai_run_quality_fields.py`
+
+2. 法律问答主链路增加质量评测。
+   - 有引用依据：`citation_present`
+   - 缺少引用依据：`missing_citation`
+   - 知识来源不足：`source_insufficient`
+   - 知识来源偏少：`source_limited`
+   - 知识来源覆盖：`source_covered`
+   - 使用 fallback：`fallback_used`
+   - provider 失败：`provider_failed`
+   - token 缺失或过高：`token_missing` / `token_high`
+   - 耗时偏高或严重：`latency_high` / `latency_critical`
+
+3. 质量判定输出。
+   - `passed`：没有 error/warning。
+   - `needs_review`：存在 warning，需要人工复核。
+   - `failed`：存在 error，需要优先处理。
+   - `quality_score` MVP 规则：100 分起，error 每项扣 30 分，warning 每项扣 10 分。
+
+4. 契约和 SDK 同步。
+   - OpenAPI `AiRun` 新增质量字段。
+   - Python SDK `AiRunAudit` 新增质量字段。
+   - 新增 `AiQualityLabel` SDK 模型。
+
+5. AI Trace 后台展示。
+   - `/admin/ai-traces` 新增“质量待复核”指标。
+   - 调用列表新增质量状态和质量分。
+   - 支持按质量状态筛选：全部、质量通过、需要复核、质量失败、未评测。
+   - 展开详情新增“质量评测”区块，展示质量摘要和标签明细。
+
+## 验收重点
+
+- 执行一次法律问答后，`GET /api/v1/audit/ai-runs` 返回质量字段。
+- fallback 回答应标记 `needs_review`，并包含 `fallback_used`。
+- 缺少引用依据时应标记 `failed`。
+- AI Trace 页能按质量状态筛选，并能展开查看每个质量标签的原因。
+
+## 建议下一步
+
+进入 AI 质量评测规则后台化：
+
+- 把 token 阈值、耗时阈值、来源数量阈值做成后台可配置。
+- 给质量标签增加人工确认状态，例如“已复核”“误报”“已处理”。
+- 给管理首页增加 AI 质量异常趋势，形成运营级监控。
@@ -1148,6 +1148,23 @@ components:
          type: array
          items:
            type: string
+    AiQualityLabel:
+      type: object
+      required:
+        - code
+        - title
+        - detail
+      properties:
+        code:
+          type: string
+        severity:
+          type: string
+          enum: [info, warning, error]
+          default: info
+        title:
+          type: string
+        detail:
+          type: string
    AiRun:
      type: object
      required:
@@ -1214,6 +1231,18 @@ components:
          format: float
        latency_ms:
          type: integer
+        quality_status:
+          type: string
+          enum: [unknown, passed, needs_review, failed]
+        quality_score:
+          type: integer
+        quality_labels:
+          type: array
+          items:
+            $ref: "#/components/schemas/AiQualityLabel"
+        quality_summary:
+          type: string
+          nullable: true
        status:
          type: string
          enum: [pending, running, succeeded, fallback, skipped, failed, cancelled]
@@ -0,0 +1,31 @@
+"""add ai run quality fields
+
+Revision ID: 20260623_0007
+Revises: 20260623_0006
+Create Date: 2026-06-23
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision: str = "20260623_0007"
+down_revision: Union[str, None] = "20260623_0006"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column("ai_run_audits", sa.Column("quality_status", sa.String(length=32), nullable=False, server_default="unknown"))
+    op.add_column("ai_run_audits", sa.Column("quality_score", sa.Integer(), nullable=False, server_default="0"))
+    op.add_column("ai_run_audits", sa.Column("quality_labels_json", sa.Text(), nullable=False, server_default="[]"))
+    op.add_column("ai_run_audits", sa.Column("quality_summary", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column("ai_run_audits", "quality_summary")
+    op.drop_column("ai_run_audits", "quality_labels_json")
+    op.drop_column("ai_run_audits", "quality_score")
+    op.drop_column("ai_run_audits", "quality_status")
@@ -13,6 +13,7 @@ from yuqei_ai_platform_api.prompts import render_legal_qa_prompt
 from yuqei_ai_platform_api.providers import ProviderRequest, build_provider_adapter
 from yuqei_ai_platform_api.repository import (
    AiPlatformRepository,
+    AiQualityLabel,
    AiRunAudit,
    KnowledgeChunk,
    KnowledgeDocument,
@@ -249,6 +250,16 @@ def create_app(
                confidence=0.0,
            )
            audit_status = "failed"
+        latency_ms = max(0, int((perf_counter() - started_at) * 1000))
+        quality_status, quality_score, quality_labels, quality_summary = _evaluate_ai_run_quality(
+            status=audit_status,
+            provider_status=provider_response.status,
+            provider_error=provider_response.error_message,
+            citations_count=len(response.citations),
+            retrieved_sources_count=len(retrieval_sources),
+            total_tokens=provider_response.total_tokens,
+            latency_ms=latency_ms,
+        )
        store.add_ai_run_audit(
            AiRunAudit(
                id=f"run-{uuid4().hex[:12]}",
@@ -271,7 +282,11 @@ def create_app(
                completion_tokens=provider_response.completion_tokens,
                total_tokens=provider_response.total_tokens,
                estimated_cost=provider_response.estimated_cost,
-                latency_ms=max(0, int((perf_counter() - started_at) * 1000)),
+                latency_ms=latency_ms,
+                quality_status=quality_status,
+                quality_score=quality_score,
+                quality_labels=quality_labels,
+                quality_summary=quality_summary,
                status=audit_status,
            )
        )
@@ -289,6 +304,175 @@ def _preview_text(value: str, *, limit: int = 4000) -> str:
    return f"{value[:limit]}..."


+def _evaluate_ai_run_quality(
+    *,
+    status: str,
+    provider_status: str | None,
+    provider_error: str | None,
+    citations_count: int,
+    retrieved_sources_count: int,
+    total_tokens: int,
+    latency_ms: int,
+) -> tuple[str, int, list[AiQualityLabel], str]:
+    labels: list[AiQualityLabel] = []
+
+    if citations_count <= 0:
+        labels.append(
+            AiQualityLabel(
+                code="missing_citation",
+                severity="error",
+                title="Missing citation",
+                detail="The answer did not return any legal citation or evidence source.",
+            )
+        )
+    else:
+        labels.append(
+            AiQualityLabel(
+                code="citation_present",
+                severity="info",
+                title="Citation present",
+                detail=f"The answer returned {citations_count} citation(s).",
+            )
+        )
+
+    if retrieved_sources_count <= 0:
+        labels.append(
+            AiQualityLabel(
+                code="source_insufficient",
+                severity="warning",
+                title="Source insufficient",
+                detail="No retrieved knowledge chunk source was recorded for this run.",
+            )
+        )
+    elif retrieved_sources_count < 2:
+        labels.append(
+            AiQualityLabel(
+                code="source_limited",
+                severity="warning",
+                title="Source limited",
+                detail="Only one retrieved source was recorded; reviewer may need to verify coverage.",
+            )
+        )
+    else:
+        labels.append(
+            AiQualityLabel(
+                code="source_covered",
+                severity="info",
+                title="Source covered",
+                detail=f"{retrieved_sources_count} retrieved sources were recorded.",
+            )
+        )
+
+    if status == "failed":
+        labels.append(
+            AiQualityLabel(
+                code="run_failed",
+                severity="error",
+                title="Run failed",
+                detail="The AI run did not complete successfully.",
+            )
+        )
+    elif status == "fallback":
+        labels.append(
+            AiQualityLabel(
+                code="fallback_used",
+                severity="warning",
+                title="Fallback used",
+                detail="The answer used the local fallback path instead of a successful provider response.",
+            )
+        )
+
+    if provider_status == "failed":
+        labels.append(
+            AiQualityLabel(
+                code="provider_failed",
+                severity="warning",
+                title="Provider failed",
+                detail=provider_error or "The model provider returned a failure status.",
+            )
+        )
+    elif provider_status == "succeeded":
+        labels.append(
+            AiQualityLabel(
+                code="provider_succeeded",
+                severity="info",
+                title="Provider succeeded",
+                detail="The model provider returned a successful response.",
+            )
+        )
+
+    if total_tokens <= 0:
+        labels.append(
+            AiQualityLabel(
+                code="token_missing",
+                severity="warning",
+                title="Token usage missing",
+                detail="Token usage was not reported for this run.",
+            )
+        )
+    elif total_tokens > 8000:
+        labels.append(
+            AiQualityLabel(
+                code="token_high",
+                severity="warning",
+                title="Token usage high",
+                detail=f"The run used {total_tokens} tokens, which is above the MVP review threshold.",
+            )
+        )
+    else:
+        labels.append(
+            AiQualityLabel(
+                code="token_normal",
+                severity="info",
+                title="Token usage normal",
+                detail=f"The run used {total_tokens} tokens.",
+            )
+        )
+
+    if latency_ms >= 30000:
+        labels.append(
+            AiQualityLabel(
+                code="latency_critical",
+                severity="error",
+                title="Latency critical",
+                detail=f"The run took {latency_ms} ms.",
+            )
+        )
+    elif latency_ms >= 15000:
+        labels.append(
+            AiQualityLabel(
+                code="latency_high",
+                severity="warning",
+                title="Latency high",
+                detail=f"The run took {latency_ms} ms.",
+            )
+        )
+    else:
+        labels.append(
+            AiQualityLabel(
+                code="latency_normal",
+                severity="info",
+                title="Latency normal",
+                detail=f"The run took {latency_ms} ms.",
+            )
+        )
+
+    error_count = sum(1 for label in labels if label.severity == "error")
+    warning_count = sum(1 for label in labels if label.severity == "warning")
+    score = max(0, 100 - error_count * 30 - warning_count * 10)
+    if error_count:
+        quality_status = "failed"
+    elif warning_count:
+        quality_status = "needs_review"
+    else:
+        quality_status = "passed"
+    summary = (
+        f"{quality_status}: {error_count} error(s), "
+        f"{warning_count} warning(s), score {score}."
+    )
+    return quality_status, score, labels, summary
+
+
 def _search_citation_documents(
    repository: AiPlatformRepository,
    request: LegalQaRequest,
@@ -124,6 +124,10 @@ class AiRunAuditModel(Base):
    total_tokens: Mapped[int] = mapped_column(Integer, default=0)
    estimated_cost: Mapped[float] = mapped_column(Float, default=0.0)
    latency_ms: Mapped[int] = mapped_column(Integer, default=0)
+    quality_status: Mapped[str] = mapped_column(String(32), default="unknown")
+    quality_score: Mapped[int] = mapped_column(Integer, default=0)
+    quality_labels_json: Mapped[str] = mapped_column(Text, default="[]")
+    quality_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
    status: Mapped[str] = mapped_column(String(32), default="succeeded")
    created_at: Mapped[datetime] = mapped_column(
        DateTime(timezone=True),
@@ -190,6 +190,13 @@ class RetrievalSource(BaseModel):
    matched_terms: list[str] = Field(default_factory=list)


+class AiQualityLabel(BaseModel):
+    code: str
+    severity: str = "info"
+    title: str
+    detail: str
+
+
 class AiRunAudit(BaseModel):
    id: str
    operation: str
@@ -212,6 +219,10 @@ class AiRunAudit(BaseModel):
    total_tokens: int = 0
    estimated_cost: float = 0.0
    latency_ms: int = 0
+    quality_status: str = "unknown"
+    quality_score: int = 0
+    quality_labels: list[AiQualityLabel] = Field(default_factory=list)
+    quality_summary: str | None = None
    status: str = "succeeded"
    created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))

@@ -962,6 +973,12 @@ class SqlAlchemyAiPlatformRepository:
            total_tokens=audit.total_tokens,
            estimated_cost=audit.estimated_cost,
            latency_ms=audit.latency_ms,
+            quality_status=audit.quality_status,
+            quality_score=audit.quality_score,
+            quality_labels_json=_encode_json_list(
+                [label.model_dump() for label in audit.quality_labels]
+            ),
+            quality_summary=audit.quality_summary,
            status=audit.status,
            created_at=audit.created_at,
        )
@@ -1170,23 +1187,31 @@ def _ai_run_audit_from_model(row: AiRunAuditModel) -> AiRunAudit:
        operation=row.operation,
        provider_config_id=row.provider_config_id,
        prompt_template_id=row.prompt_template_id,
-            provider=row.provider,
-            model=row.model,
-            provider_status=row.provider_status,
-            provider_error=row.provider_error,
-            prompt_preview=row.prompt_preview,
-            question=row.question,
-            answer=row.answer,
+        provider=row.provider,
+        model=row.model,
+        provider_status=row.provider_status,
+        provider_error=row.provider_error,
+        prompt_preview=row.prompt_preview,
+        question=row.question,
+        answer=row.answer,
        citations_count=row.citations_count,
        retrieval_query=row.retrieval_query,
        retrieval_strategy=row.retrieval_strategy,
        retrieved_chunk_ids=_decode_json_list(row.retrieved_chunk_ids_json),
-        retrieved_sources=[RetrievalSource.model_validate(item) for item in _decode_json_list(row.retrieved_sources_json)],
+        retrieved_sources=[
+            RetrievalSource.model_validate(item) for item in _decode_json_list(row.retrieved_sources_json)
+        ],
        prompt_tokens=row.prompt_tokens,
        completion_tokens=row.completion_tokens,
        total_tokens=row.total_tokens,
        estimated_cost=row.estimated_cost,
        latency_ms=row.latency_ms,
+        quality_status=row.quality_status,
+        quality_score=row.quality_score,
+        quality_labels=[
+            AiQualityLabel.model_validate(item) for item in _decode_json_list(row.quality_labels_json)
+        ],
+        quality_summary=row.quality_summary,
        status=row.status,
        created_at=_ensure_utc(row.created_at),
    )
@@ -327,6 +327,10 @@ def test_legal_qa_uses_chunk_search_and_records_audit() -> None:
    assert audits[0]["retrieved_sources"][0]["reference"] == "Lease-001"
    assert audits[0]["retrieved_sources"][0]["score"] > 0
    assert audits[0]["status"] == "fallback"
+    assert audits[0]["quality_status"] == "needs_review"
+    assert audits[0]["quality_score"] == 70
+    quality_codes = {label["code"] for label in audits[0]["quality_labels"]}
+    assert {"citation_present", "source_limited", "fallback_used", "token_missing"} <= quality_codes


 def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None:
@@ -407,6 +411,23 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None:
            total_tokens=20,
            estimated_cost=0.00028,
            latency_ms=12,
+            quality_status="passed",
+            quality_score=100,
+            quality_labels=[
+                {
+                    "code": "citation_present",
+                    "severity": "info",
+                    "title": "Citation present",
+                    "detail": "The answer returned 1 citation.",
+                },
+                {
+                    "code": "source_covered",
+                    "severity": "info",
+                    "title": "Source covered",
+                    "detail": "Sufficient retrieved sources were recorded.",
+                },
+            ],
+            quality_summary="passed: 0 error(s), 0 warning(s), score 100.",
            status="succeeded",
        )
    )
@@ -438,3 +459,7 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None:
    assert audit.estimated_cost == 0.00028
    assert audit.retrieved_chunk_ids == ["chunk-test-001"]
    assert audit.retrieved_sources[0].reference == "Article 703"
+    assert audit.quality_status == "passed"
+    assert audit.quality_score == 100
+    assert audit.quality_labels[0].code == "citation_present"
+    assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100."
@@ -78,6 +78,10 @@ def test_legal_qa_uses_provider_answer_when_enabled(monkeypatch) -> None:
    assert audit["completion_tokens"] == 50
    assert audit["total_tokens"] == 150
    assert audit["estimated_cost"] == 0.0002
+    assert audit["quality_status"] == "needs_review"
+    assert audit["quality_score"] == 90
+    quality_codes = {label["code"] for label in audit["quality_labels"]}
+    assert {"citation_present", "source_limited", "provider_succeeded", "token_normal", "latency_normal"} <= quality_codes


 def test_provider_failure_falls_back_to_rule_answer_and_audit_status() -> None:
@@ -116,6 +120,10 @@ def test_provider_failure_falls_back_to_rule_answer_and_audit_status() -> None:
    assert audit["provider_error"] == "Missing provider API key."
    assert "试用期如何约定" in audit["prompt_preview"]
    assert audit["total_tokens"] == 0
+    assert audit["quality_status"] == "needs_review"
+    assert audit["quality_score"] == 60
+    quality_codes = {label["code"] for label in audit["quality_labels"]}
+    assert {"citation_present", "fallback_used", "provider_failed", "token_missing"} <= quality_codes


 def test_openai_compatible_provider_extracts_chat_answer() -> None:
@@ -1,5 +1,6 @@
 from yuqei_sdk.ai_platform import (
    AiPlatformClient,
+    AiQualityLabel,
    AiRunAudit,
    KnowledgeChunk,
    KnowledgeDocument,
@@ -24,6 +25,7 @@ from yuqei_sdk.context import RequestContext

 __all__ = [
    "AiPlatformClient",
+    "AiQualityLabel",
    "AiRunAudit",
    "KnowledgeChunk",
    "KnowledgeDocument",
@@ -188,6 +188,13 @@ class RetrievalSource(BaseModel):
    matched_terms: list[str] = Field(default_factory=list)


+class AiQualityLabel(BaseModel):
+    code: str
+    severity: str = "info"
+    title: str
+    detail: str
+
+
 class AiRunAudit(BaseModel):
    id: str
    operation: str
@@ -210,6 +217,10 @@ class AiRunAudit(BaseModel):
    total_tokens: int = 0
    estimated_cost: float = 0.0
    latency_ms: int = 0
+    quality_status: str = "unknown"
+    quality_score: int = 0
+    quality_labels: list[AiQualityLabel] = Field(default_factory=list)
+    quality_summary: str | None = None
    status: str
    created_at: str

@@ -382,6 +382,17 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
                        "total_tokens": 20,
                        "estimated_cost": 0.00028,
                        "latency_ms": 12,
+                        "quality_status": "passed",
+                        "quality_score": 100,
+                        "quality_labels": [
+                            {
+                                "code": "citation_present",
+                                "severity": "info",
+                                "title": "Citation present",
+                                "detail": "The answer returned 1 citation.",
+                            }
+                        ],
+                        "quality_summary": "passed: 0 error(s), 0 warning(s), score 100.",
                        "status": "succeeded",
                        "created_at": "2026-06-22T00:00:00Z",
                    }
@@ -452,4 +463,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
    assert audit.estimated_cost == 0.00028
    assert audit.retrieved_chunk_ids == ["chunk-1"]
    assert audit.retrieved_sources[0].reference == "Article 703"
+    assert audit.quality_status == "passed"
+    assert audit.quality_score == 100
+    assert audit.quality_labels[0].code == "citation_present"
+    assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100."
    assert "/api/v1/audit/ai-runs" in seen_paths
@@ -10,6 +10,7 @@ import {
  FileText,
  RefreshCcw,
  Search,
+  ShieldCheck,
  Sigma
 } from "lucide-react";

@@ -29,6 +30,13 @@ type RetrievalSource = {
  matched_terms?: string[];
 };

+type AiQualityLabel = {
+  code: string;
+  severity?: string;
+  title: string;
+  detail: string;
+};
+
 type AiRunAudit = {
  id: string;
  operation: string;
@@ -51,6 +59,10 @@ type AiRunAudit = {
  total_tokens?: number;
  estimated_cost?: number;
  latency_ms?: number;
+  quality_status?: string;
+  quality_score?: number;
+  quality_labels?: AiQualityLabel[];
+  quality_summary?: string | null;
  status: string;
  created_at: string;
 };
@@ -80,9 +92,48 @@ const statusTone: Record<string, string> = {
  cancelled: "tag-danger"
 };

+const qualityStatusLabels: Record<string, string> = {
+  passed: "质量通过",
+  needs_review: "需要复核",
+  failed: "质量失败",
+  unknown: "未评测"
+};
+
+const qualityStatusTone: Record<string, string> = {
+  passed: "tag-success",
+  needs_review: "tag-warning",
+  failed: "tag-danger",
+  unknown: ""
+};
+
+const qualityLabelText: Record<string, { title: string; detail: string }> = {
+  citation_present: { title: "已引用依据", detail: "回答已返回法条或证据来源。" },
+  missing_citation: { title: "缺少依据", detail: "回答未返回法条或证据来源，需要复核。" },
+  source_covered: { title: "来源覆盖", detail: "本次回答记录了多个知识命中来源。" },
+  source_limited: { title: "来源偏少", detail: "本次回答仅记录一个知识命中来源，建议复核覆盖面。" },
+  source_insufficient: { title: "来源不足", detail: "本次回答没有记录知识命中来源。" },
+  fallback_used: { title: "使用回退", detail: "本次回答走了本地回退链路。" },
+  run_failed: { title: "调用失败", detail: "本次 AI 调用未成功完成。" },
+  provider_succeeded: { title: "模型成功", detail: "模型服务返回成功响应。" },
+  provider_failed: { title: "模型失败", detail: "模型服务返回失败状态。" },
+  token_normal: { title: "Token 正常", detail: "本次调用 token 用量在阈值内。" },
+  token_missing: { title: "Token 缺失", detail: "本次调用没有返回 token 用量。" },
+  token_high: { title: "Token 偏高", detail: "本次调用 token 用量超过 MVP 复核阈值。" },
+  latency_normal: { title: "耗时正常", detail: "本次调用耗时在阈值内。" },
+  latency_high: { title: "耗时偏高", detail: "本次调用耗时超过 15 秒。" },
+  latency_critical: { title: "耗时严重", detail: "本次调用耗时超过 30 秒。" }
+};
+
+const severityTone: Record<string, string> = {
+  info: "tag-primary",
+  warning: "tag-warning",
+  error: "tag-danger"
+};
+
 export default function AiTracesPage() {
  const [state, setState] = useState<LoadState>({ status: "loading" });
  const [statusFilter, setStatusFilter] = useState("all");
+  const [qualityFilter, setQualityFilter] = useState("all");
  const [keyword, setKeyword] = useState("");

  async function loadRuns() {
@@ -111,6 +162,8 @@ export default function AiTracesPage() {
    const normalizedKeyword = keyword.trim().toLowerCase();
    return runs.filter((run) => {
      const matchesStatus = statusFilter === "all" || run.status === statusFilter;
+      const qualityStatus = run.quality_status ?? "unknown";
+      const matchesQuality = qualityFilter === "all" || qualityStatus === qualityFilter;
      const haystack = [
        run.id,
        run.operation,
@@ -120,20 +173,27 @@ export default function AiTracesPage() {
        run.answer,
        run.provider_error,
        run.retrieval_query,
+        run.quality_status,
+        run.quality_summary,
+        ...(run.quality_labels ?? []).map((label) => `${label.code} ${label.title} ${label.detail}`),
        ...(run.retrieved_sources ?? []).map((source) => `${source.title} ${source.reference}`)
      ].join("\n").toLowerCase();
-      return matchesStatus && (!normalizedKeyword || haystack.includes(normalizedKeyword));
+      return matchesStatus && matchesQuality && (!normalizedKeyword || haystack.includes(normalizedKeyword));
    });
-  }, [keyword, runs, statusFilter]);
+  }, [keyword, qualityFilter, runs, statusFilter]);

  const metrics = useMemo(() => {
    const totalTokens = runs.reduce((sum, run) => sum + (run.total_tokens ?? 0), 0);
    const estimatedCost = runs.reduce((sum, run) => sum + (run.estimated_cost ?? 0), 0);
    const failedCount = runs.filter((run) => run.status === "failed" || run.provider_status === "failed").length;
+    const qualityReviewCount = runs.filter((run) => {
+      const qualityStatus = run.quality_status ?? "unknown";
+      return qualityStatus === "needs_review" || qualityStatus === "failed";
+    }).length;
    const averageLatency = runs.length
      ? Math.round(runs.reduce((sum, run) => sum + (run.latency_ms ?? 0), 0) / runs.length)
      : 0;
-    return { totalTokens, estimatedCost, failedCount, averageLatency };
+    return { totalTokens, estimatedCost, failedCount, qualityReviewCount, averageLatency };
  }, [runs]);

  return (
@@ -153,6 +213,7 @@ export default function AiTracesPage() {
        <Metric icon={<Bot size={18} />} label="最近调用" value={String(runs.length)} />
        <Metric icon={<Sigma size={18} />} label="总 token" value={formatInteger(metrics.totalTokens)} />
        <Metric icon={<Clock3 size={18} />} label="平均耗时" value={`${metrics.averageLatency} ms`} />
+        <Metric icon={<ShieldCheck size={18} />} label="质量待复核" value={String(metrics.qualityReviewCount)} />
        <Metric icon={<AlertTriangle size={18} />} label="异常调用" value={String(metrics.failedCount)} />
      </div>

@@ -182,6 +243,17 @@ export default function AiTracesPage() {
              <option value="fallback">已回退</option>
              <option value="failed">失败</option>
            </select>
+            <select
+              className="search-input select-input"
+              value={qualityFilter}
+              onChange={(event) => setQualityFilter(event.target.value)}
+            >
+              <option value="all">全部质量</option>
+              <option value="passed">质量通过</option>
+              <option value="needs_review">需要复核</option>
+              <option value="failed">质量失败</option>
+              <option value="unknown">未评测</option>
+            </select>
          </div>

          {state.status === "loading" && <div className="empty-state">正在加载 AI 调用审计...</div>}
@@ -220,6 +292,9 @@ function Metric({ icon, label, value }: { icon: ReactNode; label: string; value:
 function TraceItem({ run }: { run: AiRunAudit }) {
  const statusClass = statusTone[run.status] ?? "";
  const providerStatusClass = statusTone[run.provider_status ?? ""] ?? "";
+  const qualityStatus = run.quality_status ?? "unknown";
+  const qualityClass = qualityStatusTone[qualityStatus] ?? "";
+  const qualityLabels = run.quality_labels ?? [];

  return (
    <details className="trace-item">
@@ -227,7 +302,12 @@ function TraceItem({ run }: { run: AiRunAudit }) {
        <div className="trace-summary-main">
          <div className="list-item-header">
            <strong>{run.question || run.operation}</strong>
-            <span className={`tag ${statusClass}`}>{statusLabels[run.status] ?? run.status}</span>
+            <div className="trace-summary-badges">
+              <span className={`tag ${qualityClass}`}>
+                {qualityStatusLabels[qualityStatus] ?? qualityStatus} · {run.quality_score ?? 0}
+              </span>
+              <span className={`tag ${statusClass}`}>{statusLabels[run.status] ?? run.status}</span>
+            </div>
          </div>
          <div className="trace-meta">
            <span>{formatDate(run.created_at)}</span>
@@ -271,6 +351,30 @@ function TraceItem({ run }: { run: AiRunAudit }) {
            </div>
          )}
        </section>
+        <section className="trace-block">
+          <h3><ShieldCheck size={16} /> 质量评测</h3>
+          <div className="trace-facts">
+            <span>质量状态</span><strong className={`tag ${qualityClass}`}>{qualityStatusLabels[qualityStatus] ?? qualityStatus}</strong>
+            <span>质量分</span><strong>{run.quality_score ?? 0}</strong>
+            <span>摘要</span><strong>{run.quality_summary || "暂无质量摘要"}</strong>
+          </div>
+          {qualityLabels.length === 0 ? (
+            <p className="muted">暂无质量标签。</p>
+          ) : (
+            <div className="quality-labels">
+              {qualityLabels.map((label) => {
+                const text = qualityLabelText[label.code];
+                const severityClass = severityTone[label.severity ?? "info"] ?? "";
+                return (
+                  <div className="quality-label" key={label.code}>
+                    <span className={`tag ${severityClass}`}>{text?.title ?? label.title}</span>
+                    <span>{text?.detail ?? label.detail}</span>
+                  </div>
+                );
+              })}
+            </div>
+          )}
+        </section>
        <section className="trace-block">
          <h3><AlertTriangle size={16} /> 运行状态</h3>
          <div className="trace-facts">
@@ -237,7 +237,7 @@ select {

 .metric-row {
  display: grid;
-  grid-template-columns: repeat(4, minmax(0, 1fr));
+  grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
  gap: 12px;
  margin-bottom: 16px;
 }
@@ -534,6 +534,13 @@ select {
  gap: 8px;
 }

+.trace-summary-badges {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: flex-end;
+  gap: 6px;
+}
+
 .trace-meta {
  display: flex;
  flex-wrap: wrap;
@@ -603,6 +610,21 @@ select {
  overflow-wrap: anywhere;
 }

+.quality-labels {
+  display: grid;
+  gap: 8px;
+}
+
+.quality-label {
+  display: grid;
+  grid-template-columns: max-content minmax(0, 1fr);
+  gap: 8px 10px;
+  align-items: center;
+  color: var(--muted);
+  font-size: 13px;
+  line-height: 1.5;
+}
+
 .compact-list-item {
  padding: 12px;
 }