diff --git a/project-docs/V2-ai-quality-ops-dashboard-2026-06-23.md b/project-docs/V2-ai-quality-ops-dashboard-2026-06-23.md new file mode 100644 index 00000000..495e31c2 --- /dev/null +++ b/project-docs/V2-ai-quality-ops-dashboard-2026-06-23.md @@ -0,0 +1,57 @@ +# V2 AI 质量运营看板闭环记录 + +日期：2026-06-23 + +## 本轮目标 + +把单次 AI Trace 的质量标签上升为运营视角，让管理员按日期查看质量通过率、fallback 率、来源不足率、耗时异常率，并形成待处理质量标签队列。 + +## 已落地内容 + +1. AI Platform 新增质量运营聚合接口。 + - `GET /api/v1/audit/quality-dashboard` + - 参数： + - `days`：统计最近 N 天，范围 1-90。 + - `pending_limit`：待处理标签数量，范围 1-100。 + +2. 后端聚合指标。 + - 总调用数 + - 质量通过数和通过率 + - `needs_review` 数量 + - 质量失败数量 + - fallback 数量和 fallback 率 + - 来源不足数量和来源不足率 + - token 异常数量 + - 耗时异常数量和耗时异常率 + - 待处理 warning/error 标签数量 + +3. 日期趋势。 + - 最近 N 天自动补齐空日期。 + - 每天返回调用数、通过率、fallback 率、来源不足率、耗时异常率和待处理标签数量。 + +4. 待处理质量标签队列。 + - 只聚合 `review_status=pending` 且 severity 为 warning/error 的标签。 + - 返回 run id、标签 code、问题、模型、质量分、创建时间和标签原因。 + - 管理员可以从看板进入 AI Trace 继续处理标签。 + +5. 契约、SDK 和前端同步。 + - OpenAPI 新增 `AiQualityDashboard` 相关 schema。 + - Python SDK 新增 `get_quality_dashboard()`。 + - 合同 Web 新增同源代理：`/api/ai-platform/audit/quality-dashboard`。 + - 管理区新增页面：`/admin/ai-quality-dashboard`。 + - 管理首页新增“AI 质量运营”入口。 + +## 验收重点 + +- 访问 `/admin/ai-quality-dashboard` 能看到质量通过率、fallback 率、来源不足率、耗时异常率。 +- 最近 N 天切换后，趋势列表随之变化。 +- 有待处理 warning/error 标签时，队列中能看到对应问题、标签原因、模型和质量分。 +- 在 AI Trace 页处理标签后，刷新看板时待处理数量会减少。 + +## 下一步建议 + +进入 AI 质量问题归因： + +- 将异常按模型、Prompt 模板、知识库、标签类型聚合。 +- 找出最常见的失败来源，例如某个模型 fallback 高、某类知识库来源不足、某个 Prompt token 过高。 +- 给看板增加“Top 问题”和“改进建议”，帮助管理员决定先调模型、补知识库还是改 Prompt。 diff --git a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml index 2aa67373..9636edb6 100644 --- a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml +++ b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml @@ -441,6 +441,34 @@ paths: type: array items: $ref: "#/components/schemas/AiRun" + /api/v1/audit/quality-dashboard: + get: + operationId: getQualityDashboard + summary: Get AI quality operation dashboard + parameters: + - name: days + in: query + required: false + schema: + type: integer + default: 14 + minimum: 1 + maximum: 90 + - name: pending_limit + in: query + required: false + schema: + type: integer + default: 20 + minimum: 1 + maximum: 100 + responses: + "200": + description: AI quality dashboard + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityDashboard" /api/v1/audit/ai-runs/{run_id}/quality-labels/{label_code}: patch: operationId: updateAiRunQualityLabel @@ -1299,6 +1327,93 @@ components: updated_at: type: string format: date-time + AiQualityDailyMetric: + type: object + properties: + date: + type: string + total_runs: + type: integer + passed_runs: + type: integer + needs_review_runs: + type: integer + failed_quality_runs: + type: integer + fallback_runs: + type: integer + source_issue_runs: + type: integer + token_issue_runs: + type: integer + latency_issue_runs: + type: integer + pending_labels: + type: integer + pass_rate: + type: number + format: float + fallback_rate: + type: number + format: float + source_issue_rate: + type: number + format: float + latency_issue_rate: + type: number + format: float + AiQualityPendingLabel: + type: object + properties: + run_id: + type: string + label_code: + type: string + severity: + type: string + title: + type: string + detail: + type: string + review_status: + type: string + enum: [pending, reviewed, false_positive, resolved] + created_at: + type: string + format: date-time + question: + type: string + nullable: true + provider: + type: string + nullable: true + model: + type: string + nullable: true + quality_status: + type: string + quality_score: + type: integer + AiQualityDashboardSummary: + allOf: + - $ref: "#/components/schemas/AiQualityDailyMetric" + - type: object + properties: + days: + type: integer + AiQualityDashboard: + type: object + properties: + summary: + $ref: "#/components/schemas/AiQualityDashboardSummary" + daily_metrics: + type: array + items: + $ref: "#/components/schemas/AiQualityDailyMetric" + pending_labels: + type: array + items: + $ref: "#/components/schemas/AiQualityPendingLabel" AiRun: type: object required: diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py index 9e26e6fc..b48ec1a1 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py @@ -15,6 +15,7 @@ from yuqei_ai_platform_api.repository import ( AiPlatformRepository, AiQualityLabel, AiQualityLabelReviewUpdate, + AiQualityDashboard, AiQualityRuleConfig, AiQualityRuleConfigCreate, AiRunAudit, @@ -234,6 +235,17 @@ def create_app( def list_ai_runs(limit: int = Query(20, ge=1, le=100)) -> list[AiRunAudit]: return store.list_ai_run_audits(limit=limit) + @app.get( + f"{resolved_settings.api_prefix}/audit/quality-dashboard", + response_model=AiQualityDashboard, + tags=["audit"], + ) + def get_quality_dashboard( + days: int = Query(14, ge=1, le=90), + pending_limit: int = Query(20, ge=1, le=100), + ) -> AiQualityDashboard: + return store.get_quality_dashboard(days=days, pending_limit=pending_limit) + @app.patch( f"{resolved_settings.api_prefix}/audit/ai-runs/{{run_id}}/quality-labels/{{label_code}}", response_model=AiRunAudit, diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py index c095ccae..e1c3d3ae 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py @@ -3,7 +3,7 @@ from __future__ import annotations import json import math from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from threading import Lock from typing import Protocol from uuid import uuid4 @@ -253,6 +253,61 @@ class AiRunAudit(BaseModel): created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) +class AiQualityDailyMetric(BaseModel): + date: str + total_runs: int = 0 + passed_runs: int = 0 + needs_review_runs: int = 0 + failed_quality_runs: int = 0 + fallback_runs: int = 0 + source_issue_runs: int = 0 + token_issue_runs: int = 0 + latency_issue_runs: int = 0 + pending_labels: int = 0 + pass_rate: float = 0.0 + fallback_rate: float = 0.0 + source_issue_rate: float = 0.0 + latency_issue_rate: float = 0.0 + + +class AiQualityPendingLabel(BaseModel): + run_id: str + label_code: str + severity: str + title: str + detail: str + review_status: str = "pending" + created_at: datetime + question: str | None = None + provider: str | None = None + model: str | None = None + quality_status: str = "unknown" + quality_score: int = 0 + + +class AiQualityDashboardSummary(BaseModel): + days: int + total_runs: int = 0 + passed_runs: int = 0 + needs_review_runs: int = 0 + failed_quality_runs: int = 0 + fallback_runs: int = 0 + source_issue_runs: int = 0 + token_issue_runs: int = 0 + latency_issue_runs: int = 0 + pending_labels: int = 0 + pass_rate: float = 0.0 + fallback_rate: float = 0.0 + source_issue_rate: float = 0.0 + latency_issue_rate: float = 0.0 + + +class AiQualityDashboard(BaseModel): + summary: AiQualityDashboardSummary + daily_metrics: list[AiQualityDailyMetric] = Field(default_factory=list) + pending_labels: list[AiQualityPendingLabel] = Field(default_factory=list) + + class AiPlatformRepository(Protocol): def upsert_provider_config(self, payload: ProviderConfigCreate) -> ProviderConfig: ... @@ -316,6 +371,8 @@ class AiPlatformRepository(Protocol): def list_ai_run_audits(self, *, limit: int = 20) -> list[AiRunAudit]: ... + def get_quality_dashboard(self, *, days: int = 14, pending_limit: int = 20) -> AiQualityDashboard: ... + def update_ai_run_quality_label( self, run_id: str, @@ -595,6 +652,16 @@ class InMemoryAiPlatformRepository: with self._lock: return self.ai_run_audits[:limit] + def get_quality_dashboard(self, *, days: int = 14, pending_limit: int = 20) -> AiQualityDashboard: + cutoff = datetime.now(UTC) - timedelta(days=max(days - 1, 0)) + with self._lock: + runs = [ + audit + for audit in self.ai_run_audits + if _ensure_utc(audit.created_at) >= cutoff.replace(hour=0, minute=0, second=0, microsecond=0) + ] + return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit) + def update_ai_run_quality_label( self, run_id: str, @@ -1093,6 +1160,18 @@ class SqlAlchemyAiPlatformRepository: ) return [_ai_run_audit_from_model(row) for row in rows] + def get_quality_dashboard(self, *, days: int = 14, pending_limit: int = 20) -> AiQualityDashboard: + cutoff = datetime.now(UTC) - timedelta(days=max(days - 1, 0)) + cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0) + with self._session_factory() as session: + rows = session.scalars( + select(AiRunAuditModel) + .where(AiRunAuditModel.created_at >= cutoff) + .order_by(AiRunAuditModel.created_at.desc()) + ).all() + runs = [_ai_run_audit_from_model(row) for row in rows] + return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit) + def update_ai_run_quality_label( self, run_id: str, @@ -1357,6 +1436,125 @@ def _ensure_utc(value: datetime) -> datetime: return value +def _build_quality_dashboard( + runs: list[AiRunAudit], + *, + days: int, + pending_limit: int, +) -> AiQualityDashboard: + normalized_days = max(days, 1) + today = datetime.now(UTC).date() + dates = [ + (today - timedelta(days=offset)).isoformat() + for offset in range(normalized_days - 1, -1, -1) + ] + daily: dict[str, AiQualityDailyMetric] = { + date: AiQualityDailyMetric(date=date) + for date in dates + } + summary = AiQualityDashboardSummary(days=normalized_days) + pending_labels: list[AiQualityPendingLabel] = [] + + for run in runs: + created_at = _ensure_utc(run.created_at) + date_key = created_at.date().isoformat() + metric = daily.setdefault(date_key, AiQualityDailyMetric(date=date_key)) + labels = run.quality_labels + label_codes = {label.code for label in labels} + has_source_issue = bool(label_codes & {"source_insufficient", "source_limited"}) + has_token_issue = bool(label_codes & {"token_missing", "token_high"}) + has_latency_issue = bool(label_codes & {"latency_high", "latency_critical"}) + pending_for_run = [ + label + for label in labels + if label.review_status == "pending" and label.severity in {"warning", "error"} + ] + + _add_quality_counts( + metric, + run=run, + has_source_issue=has_source_issue, + has_token_issue=has_token_issue, + has_latency_issue=has_latency_issue, + pending_count=len(pending_for_run), + ) + _add_quality_counts( + summary, + run=run, + has_source_issue=has_source_issue, + has_token_issue=has_token_issue, + has_latency_issue=has_latency_issue, + pending_count=len(pending_for_run), + ) + + for label in pending_for_run: + pending_labels.append( + AiQualityPendingLabel( + run_id=run.id, + label_code=label.code, + severity=label.severity, + title=label.title, + detail=label.detail, + review_status=label.review_status, + created_at=created_at, + question=run.question, + provider=run.provider, + model=run.model, + quality_status=run.quality_status, + quality_score=run.quality_score, + ) + ) + + for metric in daily.values(): + _finalize_quality_rates(metric) + _finalize_quality_rates(summary) + pending_labels = sorted(pending_labels, key=lambda item: item.created_at, reverse=True)[:pending_limit] + ordered_daily = [daily[date] for date in dates if date in daily] + extra_dates = sorted((date for date in daily if date not in dates), reverse=False) + ordered_daily.extend(daily[date] for date in extra_dates) + return AiQualityDashboard( + summary=summary, + daily_metrics=ordered_daily, + pending_labels=pending_labels, + ) + + +def _add_quality_counts( + metric: AiQualityDailyMetric | AiQualityDashboardSummary, + *, + run: AiRunAudit, + has_source_issue: bool, + has_token_issue: bool, + has_latency_issue: bool, + pending_count: int, +) -> None: + metric.total_runs += 1 + if run.quality_status == "passed": + metric.passed_runs += 1 + elif run.quality_status == "failed": + metric.failed_quality_runs += 1 + elif run.quality_status == "needs_review": + metric.needs_review_runs += 1 + if run.status == "fallback" or any(label.code == "fallback_used" for label in run.quality_labels): + metric.fallback_runs += 1 + if has_source_issue: + metric.source_issue_runs += 1 + if has_token_issue: + metric.token_issue_runs += 1 + if has_latency_issue: + metric.latency_issue_runs += 1 + metric.pending_labels += pending_count + + +def _finalize_quality_rates(metric: AiQualityDailyMetric | AiQualityDashboardSummary) -> None: + if metric.total_runs <= 0: + return + metric.pass_rate = round(metric.passed_runs / metric.total_runs, 4) + metric.fallback_rate = round(metric.fallback_runs / metric.total_runs, 4) + metric.source_issue_rate = round(metric.source_issue_runs / metric.total_runs, 4) + metric.latency_issue_rate = round(metric.latency_issue_runs / metric.total_runs, 4) + + def _update_quality_label_list( labels: list[AiQualityLabel], label_code: str, diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py index 63d07a91..dcd15981 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py @@ -166,6 +166,35 @@ def test_ai_run_quality_label_review_status_can_be_updated() -> None: assert label["reviewed_at"] +def test_quality_dashboard_summarizes_daily_metrics_and_pending_labels() -> None: + client = make_client() + + response = client.post("/api/v1/legal/qa", json={"question": "试用期如何约定？"}) + assert response.status_code == 200 + + dashboard_response = client.get("/api/v1/audit/quality-dashboard", params={"days": 7, "pending_limit": 10}) + assert dashboard_response.status_code == 200 + dashboard = dashboard_response.json() + assert dashboard["summary"]["days"] == 7 + assert dashboard["summary"]["total_runs"] == 1 + assert dashboard["summary"]["fallback_runs"] == 1 + assert dashboard["summary"]["pending_labels"] >= 1 + assert dashboard["summary"]["fallback_rate"] == 1.0 + assert len(dashboard["daily_metrics"]) == 7 + assert dashboard["daily_metrics"][-1]["total_runs"] == 1 + assert dashboard["pending_labels"] + + first_pending = dashboard["pending_labels"][0] + update_response = client.patch( + f"/api/v1/audit/ai-runs/{first_pending['run_id']}/quality-labels/{first_pending['label_code']}", + json={"review_status": "resolved", "review_note": "已处理。", "reviewed_by": "admin"}, + ) + assert update_response.status_code == 200 + + updated_dashboard = client.get("/api/v1/audit/quality-dashboard", params={"days": 7, "pending_limit": 10}).json() + assert updated_dashboard["summary"]["pending_labels"] == dashboard["summary"]["pending_labels"] - 1 + + def test_knowledge_document_can_be_added_and_searched() -> None: client = make_client() diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py index e59fe53b..11d4f1dc 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py @@ -1,7 +1,11 @@ from yuqei_sdk.ai_platform import ( AiPlatformClient, + AiQualityDashboard, + AiQualityDashboardSummary, + AiQualityDailyMetric, AiQualityLabel, AiQualityLabelReviewUpdate, + AiQualityPendingLabel, AiQualityRuleConfig, AiQualityRuleConfigUpdate, AiRunAudit, @@ -28,8 +32,12 @@ from yuqei_sdk.context import RequestContext __all__ = [ "AiPlatformClient", + "AiQualityDashboard", + "AiQualityDashboardSummary", + "AiQualityDailyMetric", "AiQualityLabel", "AiQualityLabelReviewUpdate", + "AiQualityPendingLabel", "AiQualityRuleConfig", "AiQualityRuleConfigUpdate", "AiRunAudit", diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py index ceb39c7a..d19f2b43 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py @@ -250,6 +250,61 @@ class AiRunAudit(BaseModel): created_at: str +class AiQualityDailyMetric(BaseModel): + date: str + total_runs: int = 0 + passed_runs: int = 0 + needs_review_runs: int = 0 + failed_quality_runs: int = 0 + fallback_runs: int = 0 + source_issue_runs: int = 0 + token_issue_runs: int = 0 + latency_issue_runs: int = 0 + pending_labels: int = 0 + pass_rate: float = 0.0 + fallback_rate: float = 0.0 + source_issue_rate: float = 0.0 + latency_issue_rate: float = 0.0 + + +class AiQualityDashboardSummary(BaseModel): + days: int + total_runs: int = 0 + passed_runs: int = 0 + needs_review_runs: int = 0 + failed_quality_runs: int = 0 + fallback_runs: int = 0 + source_issue_runs: int = 0 + token_issue_runs: int = 0 + latency_issue_runs: int = 0 + pending_labels: int = 0 + pass_rate: float = 0.0 + fallback_rate: float = 0.0 + source_issue_rate: float = 0.0 + latency_issue_rate: float = 0.0 + + +class AiQualityPendingLabel(BaseModel): + run_id: str + label_code: str + severity: str + title: str + detail: str + review_status: str = "pending" + created_at: str + question: str | None = None + provider: str | None = None + model: str | None = None + quality_status: str = "unknown" + quality_score: int = 0 + + +class AiQualityDashboard(BaseModel): + summary: AiQualityDashboardSummary + daily_metrics: list[AiQualityDailyMetric] = Field(default_factory=list) + pending_labels: list[AiQualityPendingLabel] = Field(default_factory=list) + + class AiPlatformClient: def __init__( self, @@ -512,6 +567,21 @@ class AiPlatformClient: response.raise_for_status() return [AiRunAudit.model_validate(item) for item in response.json()] + def get_quality_dashboard( + self, + *, + days: int = 14, + pending_limit: int = 20, + context: RequestContext | None = None, + ) -> AiQualityDashboard: + response = self._client.get( + f"{self._api_prefix}/audit/quality-dashboard", + params={"days": days, "pending_limit": pending_limit}, + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return AiQualityDashboard.model_validate(response.json()) + def update_ai_run_quality_label( self, run_id: str, diff --git a/yuqei-ai-sdk-python/tests/test_ai_platform.py b/yuqei-ai-sdk-python/tests/test_ai_platform.py index 6fefa048..7cd5a99d 100644 --- a/yuqei-ai-sdk-python/tests/test_ai_platform.py +++ b/yuqei-ai-sdk-python/tests/test_ai_platform.py @@ -436,6 +436,65 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: } ], ) + if request.url.path == "/api/v1/audit/quality-dashboard": + query = parse_qs(request.url.query.decode()) + assert query["days"] == ["7"] + assert query["pending_limit"] == ["5"] + return httpx.Response( + 200, + json={ + "summary": { + "days": 7, + "total_runs": 1, + "passed_runs": 1, + "needs_review_runs": 0, + "failed_quality_runs": 0, + "fallback_runs": 0, + "source_issue_runs": 0, + "token_issue_runs": 0, + "latency_issue_runs": 0, + "pending_labels": 1, + "pass_rate": 1.0, + "fallback_rate": 0.0, + "source_issue_rate": 0.0, + "latency_issue_rate": 0.0, + }, + "daily_metrics": [ + { + "date": "2026-06-22", + "total_runs": 1, + "passed_runs": 1, + "needs_review_runs": 0, + "failed_quality_runs": 0, + "fallback_runs": 0, + "source_issue_runs": 0, + "token_issue_runs": 0, + "latency_issue_runs": 0, + "pending_labels": 1, + "pass_rate": 1.0, + "fallback_rate": 0.0, + "source_issue_rate": 0.0, + "latency_issue_rate": 0.0, + } + ], + "pending_labels": [ + { + "run_id": "run-1", + "label_code": "citation_present", + "severity": "info", + "title": "Citation present", + "detail": "The answer returned 1 citation.", + "review_status": "pending", + "created_at": "2026-06-22T00:00:00Z", + "question": "How should lease deposit refund be handled?", + "provider": "deepseek", + "model": "deepseek-chat", + "quality_status": "passed", + "quality_score": 100, + } + ], + }, + ) if request.url.path == "/api/v1/audit/ai-runs/run-1/quality-labels/citation_present": payload = json.loads(request.read().decode("utf-8")) assert request.method == "PATCH" @@ -565,6 +624,10 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: assert audit.quality_labels[0].code == "citation_present" assert audit.quality_labels[0].review_status == "pending" assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100." + dashboard = client.get_quality_dashboard(days=7, pending_limit=5) + assert dashboard.summary.pass_rate == 1.0 + assert dashboard.daily_metrics[0].date == "2026-06-22" + assert dashboard.pending_labels[0].run_id == "run-1" updated_audit = client.update_ai_run_quality_label( "run-1", "citation_present", @@ -577,4 +640,5 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: assert updated_audit.quality_labels[0].review_status == "resolved" assert updated_audit.quality_labels[0].reviewed_at == "2026-06-22T00:01:00Z" assert "/api/v1/audit/ai-runs" in seen_paths + assert "/api/v1/audit/quality-dashboard" in seen_paths assert "/api/v1/quality-rules/default" in seen_paths diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx new file mode 100644 index 00000000..ac6dd064 --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx @@ -0,0 +1,239 @@ +"use client"; + +import Link from "next/link"; +import type { ReactNode } from "react"; +import { useEffect, useState } from "react"; +import { AlertTriangle, Clock3, Database, RefreshCcw, ShieldCheck, Undo2 } from "lucide-react"; + +type QualityDailyMetric = { + date: string; + total_runs: number; + passed_runs: number; + needs_review_runs: number; + failed_quality_runs: number; + fallback_runs: number; + source_issue_runs: number; + token_issue_runs: number; + latency_issue_runs: number; + pending_labels: number; + pass_rate: number; + fallback_rate: number; + source_issue_rate: number; + latency_issue_rate: number; +}; + +type QualityPendingLabel = { + run_id: string; + label_code: string; + severity: string; + title: string; + detail: string; + review_status: string; + created_at: string; + question?: string | null; + provider?: string | null; + model?: string | null; + quality_status: string; + quality_score: number; +}; + +type QualityDashboard = { + summary: QualityDailyMetric & { days: number }; + daily_metrics: QualityDailyMetric[]; + pending_labels: QualityPendingLabel[]; +}; + +type LoadState = + | { status: "loading" } + | { status: "ready"; dashboard: QualityDashboard } + | { status: "error"; message: string }; + +const severityTone: Record = { + info: "tag-primary", + warning: "tag-warning", + error: "tag-danger" +}; + +export default function AiQualityDashboardPage() { + const [state, setState] = useState({ status: "loading" }); + const [days, setDays] = useState(14); + + async function loadDashboard(nextDays = days) { + setState({ status: "loading" }); + try { + const response = await fetch( + `/api/ai-platform/audit/quality-dashboard?days=${nextDays}&pending_limit=30`, + { cache: "no-store" } + ); + const payload = await response.json(); + if (!response.ok) { + throw new Error(payload?.message ?? `HTTP ${response.status}`); + } + setState({ status: "ready", dashboard: payload }); + } catch (error) { + setState({ + status: "error", + message: error instanceof Error ? error.message : "AI 质量运营看板加载失败" + }); + } + } + + useEffect(() => { + void loadDashboard(days); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + const dashboard = state.status === "ready" ? state.dashboard : null; + function changeDays(value: string) { + const nextDays = Number(value); + setDays(nextDays); + void loadDashboard(nextDays); + } + + return ( + <> +
+
+

AI 质量运营

+

按日期查看质量通过率、fallback、来源不足、耗时异常和待处理标签。

+
+
+ + +
+
+ + {state.status === "loading" &&
正在加载 AI 质量运营数据...
} + {state.status === "error" && ( +
+ 无法连接 AI Platform 质量运营接口 + {state.message} +
+ )} + {dashboard && ( + <> +
+ } label="质量通过率" value={formatPercent(dashboard.summary.pass_rate)} /> + } label="Fallback 率" value={formatPercent(dashboard.summary.fallback_rate)} /> + } label="来源不足率" value={formatPercent(dashboard.summary.source_issue_rate)} /> + } label="耗时异常率" value={formatPercent(dashboard.summary.latency_issue_rate)} /> + } label="待处理标签" value={String(dashboard.summary.pending_labels)} /> +
+ +
+
+
+

趋势

+ 总调用 {dashboard.summary.total_runs} +
+
+ {dashboard.daily_metrics.map((metric) => ( +
+ {formatShortDate(metric.date)} +
+ + + +
+ {metric.total_runs} 次 +
+ ))} +
+ 通过 + Fallback + 来源/耗时异常 +
+
+
+ +
+
+

待处理质量标签

+ + 打开 Trace + +
+
+ {dashboard.pending_labels.length === 0 ? ( +
当前没有待处理质量标签。
+ ) : ( +
+ {dashboard.pending_labels.map((label) => ( +
+
+ {label.question || label.run_id} + {label.title} +
+ {label.detail} + + {formatDateTime(label.created_at)} · {label.provider || "unknown"} / {label.model || "unknown"} · 质量分 {label.quality_score} + +
+ ))} +
+ )} +
+
+
+ + )} + + ); +} + +function Metric({ icon, label, value }: { icon: ReactNode; label: string; value: string }) { + return ( +
+
{icon}
+
{value}
+
{label}
+
+ ); +} + +function formatPercent(value: number) { + return `${Math.round(value * 100)}%`; +} + +function formatShortDate(value: string) { + const date = new Date(`${value}T00:00:00`); + if (Number.isNaN(date.getTime())) { + return value; + } + return new Intl.DateTimeFormat("zh-CN", { month: "2-digit", day: "2-digit" }).format(date); +} + +function formatDateTime(value: string) { + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return value; + } + return new Intl.DateTimeFormat("zh-CN", { + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit" + }).format(date); +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx index 65b95692..b29502d3 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx @@ -1,6 +1,6 @@ import Link from "next/link"; import type { LucideIcon } from "lucide-react"; -import { Bot, Settings, ShieldCheck, SlidersHorizontal } from "lucide-react"; +import { Activity, Bot, Settings, ShieldCheck, SlidersHorizontal } from "lucide-react"; type AdminSection = { title: string; @@ -21,6 +21,12 @@ const adminSections: AdminSection[] = [ icon: SlidersHorizontal, href: "/admin/ai-quality" }, + { + title: "AI 质量运营", + description: "质量通过率、fallback、异常趋势和待处理标签。", + icon: Activity, + href: "/admin/ai-quality-dashboard" + }, { title: "AI 调用 Trace", description: "查看模型、Prompt、知识命中、token、费用、错误和耗时。", diff --git a/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-dashboard/route.ts b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-dashboard/route.ts new file mode 100644 index 00000000..b9ced065 --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-dashboard/route.ts @@ -0,0 +1,39 @@ +import { NextResponse } from "next/server"; + +const aiPlatformBaseUrl = + process.env.AI_PLATFORM_API_BASE_URL ?? + process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ?? + "http://localhost:8101"; + +export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const days = searchParams.get("days") ?? "14"; + const pendingLimit = searchParams.get("pending_limit") ?? "20"; + const targetUrl = new URL("/api/v1/audit/quality-dashboard", aiPlatformBaseUrl); + targetUrl.searchParams.set("days", days); + targetUrl.searchParams.set("pending_limit", pendingLimit); + + try { + const response = await fetch(targetUrl, { cache: "no-store" }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform quality dashboard service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform quality dashboard service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/globals.css b/yuqei-contract-platform/apps/contract-web/src/app/globals.css index 52186bc1..04f27fda 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/globals.css +++ b/yuqei-contract-platform/apps/contract-web/src/app/globals.css @@ -689,6 +689,95 @@ select { background: var(--danger-soft); } +.dashboard-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(320px, 0.8fr); + gap: 16px; +} + +.trend-list { + display: grid; + gap: 10px; +} + +.trend-row { + display: grid; + grid-template-columns: 58px minmax(0, 1fr) 54px; + gap: 10px; + align-items: center; + color: var(--muted); + font-size: 13px; +} + +.trend-date, +.trend-count { + white-space: nowrap; +} + +.trend-count { + text-align: right; +} + +.trend-bars { + position: relative; + display: grid; + gap: 3px; + min-height: 32px; + padding: 5px 0; +} + +.trend-bar { + display: block; + min-height: 6px; + max-width: 100%; + border-radius: 999px; +} + +.trend-bar-pass { + background: var(--success); +} + +.trend-bar-fallback { + background: var(--warning); +} + +.trend-bar-issue { + background: var(--danger); +} + +.trend-legend { + display: flex; + flex-wrap: wrap; + gap: 10px 14px; + padding-top: 8px; + color: var(--muted); + font-size: 12px; +} + +.trend-legend span { + display: inline-flex; + align-items: center; + gap: 6px; +} + +.trend-dot { + width: 8px; + height: 8px; + border-radius: 999px; +} + +.trend-dot-pass { + background: var(--success); +} + +.trend-dot-fallback { + background: var(--warning); +} + +.trend-dot-issue { + background: var(--danger); +} + .compact-list-item { padding: 12px; } @@ -719,6 +808,7 @@ details.panel summary::-webkit-details-marker { .split-layout, .detail-grid, .grid-two, + .dashboard-grid, .trace-detail-grid, .quality-label { grid-template-columns: 1fr;