yuqei/ai-contract-platform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: close ai quality improvement loop

Browse Source
This commit is contained in:
yuqei
2026-06-23 02:34:35 +08:00
parent 00f4ebb652
commit 87bb3c2b75
13 changed files with 1538 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
project-docs/V2-ai-quality-improvement-action-loop-2026-06-23.md
+46
View File
@@ -0,0 +1,46 @@
# V2 AI 质量改进动作闭环记录 - 2026-06-23
## 本轮目标
在已有 AI 质量看板和问题归因基础上，补齐“异常归因 -> 待处理改进项 -> 人工处理状态 -> 处理后异常率回看”的运营闭环。
## 已完成能力
1. AI Platform 后端新增 `ai_quality_improvement_actions` 持久化模型。
2. 新增改进项创建、列表、状态更新接口：
- `GET /api/v1/audit/quality-improvement-actions`
- `POST /api/v1/audit/quality-improvement-actions`
- `PATCH /api/v1/audit/quality-improvement-actions/{action_id}`
3. 质量看板响应新增 `improvement_actions`，可直接展示待推进改进队列。
4. 改进项记录维度包括模型、Prompt、知识库、标签类型，并保存来源标签、根因提示、负责人、优先级、处理说明。
5. 改进项自动记录基线数据，并在看板中计算当前影响调用数、待处理标签数、质量分和异常率变化。
6. Python SDK 补齐对应 DTO 和客户端方法，合同系统后续可通过正式 SDK 调用 AI 中台能力。
7. OpenAPI 契约同步新增改进项接口和 schema。
8. 合同系统后台 AI 质量运营页新增：
- 归因卡片一键“转改进项”。
- 改进动作队列。
- 开始处理、标记完成、误报关闭。
- 基线/当前异常率、待处理标签、质量分对比。
9. 合同系统新增 Next API 代理，浏览器端仍通过合同系统访问 AI Platform。
## 闭环使用方式
1. 管理员进入 `AI 质量运营` 页面。
2. 查看模型、Prompt、知识库、标签归因。
3. 对高频异常点击“转改进项”。
4. 在改进动作队列中推进状态：待处理、处理中、已完成、误报关闭。
5. 后续 AI 调用产生新数据后，看板会持续回看当前异常率是否低于创建时基线。
## 验证命令
```powershell
python -m pytest yuqei-ai-platform/services/ai-platform-api/tests yuqei-ai-sdk-python/tests -q
python -c "import yaml; yaml.safe_load(open('yuqei-ai-platform/packages/ai-openapi/openapi.yaml', encoding='utf-8')); print('openapi ok')"
npm run web:typecheck
npm run web:build
git diff --check
```
## 后续建议
下一步可以继续做 AI 质量改进项的运营增强：支持按负责人、优先级、状态筛选；支持关联具体 Prompt 版本、知识库导入批次、模型配置变更记录；支持改进完成后自动生成一份效果复盘。
yuqei-ai-platform/packages/ai-openapi/openapi.yaml
+201
View File
@@ -469,6 +469,73 @@ paths:
application/json:
schema:
$ref: "#/components/schemas/AiQualityDashboard"
/api/v1/audit/quality-improvement-actions:
get:
operationId: listQualityImprovementActions
summary: List AI quality improvement actions
parameters:
- name: status
in: query
required: false
schema:
type: string
enum: [open, in_progress, resolved, false_positive, closed]
- name: limit
in: query
required: false
schema:
type: integer
default: 20
minimum: 1
maximum: 100
responses:
"200":
description: AI quality improvement actions
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/AiQualityImprovementAction"
post:
operationId: createQualityImprovementAction
summary: Create an AI quality improvement action from attribution
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/AiQualityImprovementActionCreate"
responses:
"200":
description: Created AI quality improvement action
content:
application/json:
schema:
$ref: "#/components/schemas/AiQualityImprovementAction"
/api/v1/audit/quality-improvement-actions/{action_id}:
patch:
operationId: updateQualityImprovementAction
summary: Update an AI quality improvement action status
parameters:
- name: action_id
in: path
required: true
schema:
type: string
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/AiQualityImprovementActionUpdate"
responses:
"200":
description: Updated AI quality improvement action
content:
application/json:
schema:
$ref: "#/components/schemas/AiQualityImprovementAction"
/api/v1/audit/ai-runs/{run_id}/quality-labels/{label_code}:
patch:
operationId: updateAiRunQualityLabel
@@ -1452,6 +1519,136 @@ components:
type: array
items:
$ref: "#/components/schemas/AiQualityAttributionItem"
AiQualityImprovementActionCreate:
type: object
required:
- dimension
- key
- label
properties:
dimension:
type: string
enum: [model, prompt_template, knowledge_base, label_code]
key:
type: string
minLength: 1
maxLength: 256
label:
type: string
minLength: 1
maxLength: 256
action_type:
type: string
enum: [investigate, tune_prompt, improve_knowledge, tune_model, mark_false_positive]
default: investigate
title:
type: string
nullable: true
maxLength: 256
description:
type: string
nullable: true
owner:
type: string
nullable: true
maxLength: 128
priority:
type: string
enum: [low, medium, high, critical]
default: medium
source_label_codes:
type: array
items:
type: string
source_run_ids:
type: array
items:
type: string
root_cause_hint:
type: string
nullable: true
evaluation_days:
type: integer
minimum: 1
maximum: 90
default: 14
AiQualityImprovementActionUpdate:
type: object
required:
- status
properties:
status:
type: string
enum: [open, in_progress, resolved, false_positive, closed]
owner:
type: string
nullable: true
maxLength: 128
priority:
type: string
enum: [low, medium, high, critical]
nullable: true
resolution_note:
type: string
nullable: true
AiQualityImprovementAction:
allOf:
- $ref: "#/components/schemas/AiQualityImprovementActionCreate"
- type: object
required:
- id
- title
- status
- created_at
- updated_at
properties:
id:
type: string
title:
type: string
status:
type: string
enum: [open, in_progress, resolved, false_positive, closed]
default: open
baseline_total_runs:
type: integer
baseline_affected_runs:
type: integer
baseline_pending_labels:
type: integer
baseline_quality_score:
type: number
format: float
current_total_runs:
type: integer
current_affected_runs:
type: integer
current_pending_labels:
type: integer
current_quality_score:
type: number
format: float
affected_rate_delta:
type: number
format: float
pending_labels_delta:
type: integer
quality_score_delta:
type: number
format: float
resolution_note:
type: string
nullable: true
created_at:
type: string
format: date-time
updated_at:
type: string
format: date-time
resolved_at:
type: string
format: date-time
nullable: true
AiQualityDashboard:
type: object
properties:
@@ -1467,6 +1664,10 @@ components:
$ref: "#/components/schemas/AiQualityPendingLabel"
attribution:
$ref: "#/components/schemas/AiQualityAttribution"
improvement_actions:
type: array
items:
$ref: "#/components/schemas/AiQualityImprovementAction"
AiRun:
type: object
required:
yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py
+38
View File
@@ -16,6 +16,9 @@ from yuqei_ai_platform_api.repository import (
AiQualityLabel,
AiQualityLabelReviewUpdate,
AiQualityDashboard,
AiQualityImprovementAction,
AiQualityImprovementActionCreate,
AiQualityImprovementActionUpdate,
AiQualityRuleConfig,
AiQualityRuleConfigCreate,
AiRunAudit,
@@ -246,6 +249,41 @@ def create_app(
) -> AiQualityDashboard:
return store.get_quality_dashboard(days=days, pending_limit=pending_limit)
@app.get(
f"{resolved_settings.api_prefix}/audit/quality-improvement-actions",
response_model=list[AiQualityImprovementAction],
tags=["audit"],
)
def list_quality_improvement_actions(
status: str | None = Query(None, pattern="^(open|in_progress|resolved|false_positive|closed)$"),
limit: int = Query(20, ge=1, le=100),
) -> list[AiQualityImprovementAction]:
return store.list_quality_improvement_actions(status=status, limit=limit)
@app.post(
f"{resolved_settings.api_prefix}/audit/quality-improvement-actions",
response_model=AiQualityImprovementAction,
tags=["audit"],
)
def create_quality_improvement_action(
payload: AiQualityImprovementActionCreate,
) -> AiQualityImprovementAction:
return store.create_quality_improvement_action(payload)
@app.patch(
f"{resolved_settings.api_prefix}/audit/quality-improvement-actions/{{action_id}}",
response_model=AiQualityImprovementAction,
tags=["audit"],
)
def update_quality_improvement_action(
action_id: str,
payload: AiQualityImprovementActionUpdate,
) -> AiQualityImprovementAction:
try:
return store.update_quality_improvement_action(action_id, payload)
except KeyError as error:
raise HTTPException(status_code=404, detail="AI quality improvement action not found.") from error
@app.patch(
f"{resolved_settings.api_prefix}/audit/ai-runs/{{run_id}}/quality-labels/{{label_code}}",
response_model=AiRunAudit,
yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py
+35
View File
@@ -151,3 +151,38 @@ class AiRunAuditModel(Base):
default=lambda: datetime.now(UTC),
index=True,
)
class AiQualityImprovementActionModel(Base):
__tablename__ = "ai_quality_improvement_actions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
dimension: Mapped[str] = mapped_column(String(64), index=True)
key: Mapped[str] = mapped_column(String(256), index=True)
label: Mapped[str] = mapped_column(String(256))
action_type: Mapped[str] = mapped_column(String(64), index=True)
title: Mapped[str] = mapped_column(String(256))
description: Mapped[str | None] = mapped_column(Text, nullable=True)
owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
priority: Mapped[str] = mapped_column(String(32), default="medium", index=True)
status: Mapped[str] = mapped_column(String(32), default="open", index=True)
source_label_codes_json: Mapped[str] = mapped_column(Text, default="[]")
source_run_ids_json: Mapped[str] = mapped_column(Text, default="[]")
root_cause_hint: Mapped[str | None] = mapped_column(Text, nullable=True)
evaluation_days: Mapped[int] = mapped_column(Integer, default=14)
baseline_total_runs: Mapped[int] = mapped_column(Integer, default=0)
baseline_affected_runs: Mapped[int] = mapped_column(Integer, default=0)
baseline_pending_labels: Mapped[int] = mapped_column(Integer, default=0)
baseline_quality_score: Mapped[float] = mapped_column(Float, default=0.0)
resolution_note: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(UTC),
index=True,
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True),
default=lambda: datetime.now(UTC),
index=True,
)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py
+419 -4
View File
@@ -5,7 +5,7 @@ import math
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from threading import Lock
from typing import Protocol
from typing import Iterable, Protocol
from uuid import uuid4
from pydantic import BaseModel, Field
@@ -24,6 +24,7 @@ from yuqei_ai_platform_api.embeddings import (
from yuqei_ai_platform_api.legal_qa import LegalCitation
from yuqei_ai_platform_api.models import (
AiProviderConfigModel,
AiQualityImprovementActionModel,
AiQualityRuleConfigModel,
AiRunAuditModel,
Base,
@@ -307,6 +308,52 @@ class AiQualityAttribution(BaseModel):
by_label_code: list[AiQualityAttributionItem] = Field(default_factory=list)
class AiQualityImprovementActionCreate(BaseModel):
dimension: str = Field(pattern="^(model|prompt_template|knowledge_base|label_code)$")
key: str = Field(min_length=1, max_length=256)
label: str = Field(min_length=1, max_length=256)
action_type: str = Field(
default="investigate",
pattern="^(investigate|tune_prompt|improve_knowledge|tune_model|mark_false_positive)$",
)
title: str | None = Field(default=None, max_length=256)
description: str | None = None
owner: str | None = Field(default=None, max_length=128)
priority: str = Field(default="medium", pattern="^(low|medium|high|critical)$")
source_label_codes: list[str] = Field(default_factory=list)
source_run_ids: list[str] = Field(default_factory=list)
root_cause_hint: str | None = None
evaluation_days: int = Field(default=14, ge=1, le=90)
class AiQualityImprovementActionUpdate(BaseModel):
status: str = Field(pattern="^(open|in_progress|resolved|false_positive|closed)$")
owner: str | None = Field(default=None, max_length=128)
priority: str | None = Field(default=None, pattern="^(low|medium|high|critical)$")
resolution_note: str | None = None
class AiQualityImprovementAction(AiQualityImprovementActionCreate):
id: str
title: str
status: str = "open"
baseline_total_runs: int = 0
baseline_affected_runs: int = 0
baseline_pending_labels: int = 0
baseline_quality_score: float = 0.0
current_total_runs: int = 0
current_affected_runs: int = 0
current_pending_labels: int = 0
current_quality_score: float = 0.0
affected_rate_delta: float = 0.0
pending_labels_delta: int = 0
quality_score_delta: float = 0.0
resolution_note: str | None = None
created_at: datetime
updated_at: datetime
resolved_at: datetime | None = None
class AiQualityDashboardSummary(BaseModel):
days: int
total_runs: int = 0
@@ -329,6 +376,7 @@ class AiQualityDashboard(BaseModel):
daily_metrics: list[AiQualityDailyMetric] = Field(default_factory=list)
pending_labels: list[AiQualityPendingLabel] = Field(default_factory=list)
attribution: AiQualityAttribution = Field(default_factory=AiQualityAttribution)
improvement_actions: list[AiQualityImprovementAction] = Field(default_factory=list)
class AiPlatformRepository(Protocol):
@@ -396,6 +444,24 @@ class AiPlatformRepository(Protocol):
def get_quality_dashboard(self, *, days: int = 14, pending_limit: int = 20) -> AiQualityDashboard: ...
def list_quality_improvement_actions(
self,
*,
status: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]: ...
def create_quality_improvement_action(
self,
payload: AiQualityImprovementActionCreate,
) -> AiQualityImprovementAction: ...
def update_quality_improvement_action(
self,
action_id: str,
payload: AiQualityImprovementActionUpdate,
) -> AiQualityImprovementAction: ...
def update_ai_run_quality_label(
self,
run_id: str,
@@ -414,6 +480,7 @@ class InMemoryAiPlatformRepository:
knowledge_documents: dict[str, KnowledgeDocument] = field(default_factory=dict)
knowledge_chunks: dict[str, KnowledgeChunk] = field(default_factory=dict)
ai_run_audits: list[AiRunAudit] = field(default_factory=list)
quality_improvement_actions: dict[str, AiQualityImprovementAction] = field(default_factory=dict)
quality_rule_config: AiQualityRuleConfig = field(
default_factory=lambda: AiQualityRuleConfig(
id="default",
@@ -683,7 +750,75 @@ class InMemoryAiPlatformRepository:
for audit in self.ai_run_audits
if _ensure_utc(audit.created_at) >= cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
]
return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit)
actions = _materialize_improvement_actions(
self.quality_improvement_actions.values(),
self.ai_run_audits,
)
return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit, improvement_actions=actions)
def list_quality_improvement_actions(
self,
*,
status: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]:
with self._lock:
actions = _materialize_improvement_actions(
self.quality_improvement_actions.values(),
self.ai_run_audits,
)
return _filter_improvement_actions(actions, status=status, limit=limit)
def create_quality_improvement_action(
self,
payload: AiQualityImprovementActionCreate,
) -> AiQualityImprovementAction:
now = datetime.now(UTC)
with self._lock:
baseline = _calculate_improvement_action_stats(payload, self.ai_run_audits)
action = AiQualityImprovementAction(
id=f"qa-action-{uuid4().hex[:12]}",
title=payload.title or _default_improvement_action_title(payload),
status="open",
baseline_total_runs=baseline["total_runs"],
baseline_affected_runs=baseline["affected_runs"],
baseline_pending_labels=baseline["pending_labels"],
baseline_quality_score=baseline["quality_score"],
current_total_runs=baseline["total_runs"],
current_affected_runs=baseline["affected_runs"],
current_pending_labels=baseline["pending_labels"],
current_quality_score=baseline["quality_score"],
created_at=now,
updated_at=now,
**payload.model_dump(exclude={"title"}),
)
self.quality_improvement_actions[action.id] = action
return action
def update_quality_improvement_action(
self,
action_id: str,
payload: AiQualityImprovementActionUpdate,
) -> AiQualityImprovementAction:
now = datetime.now(UTC)
with self._lock:
action = self.quality_improvement_actions.get(action_id)
if action is None:
raise KeyError(action_id)
update: dict[str, object] = {
"status": payload.status,
"updated_at": now,
"resolved_at": now if payload.status in {"resolved", "false_positive", "closed"} else None,
}
if payload.owner is not None:
update["owner"] = payload.owner
if payload.priority is not None:
update["priority"] = payload.priority
if payload.resolution_note is not None:
update["resolution_note"] = payload.resolution_note
updated = action.model_copy(update=update)
self.quality_improvement_actions[action_id] = updated
return _materialize_improvement_action(updated, self.ai_run_audits)
def update_ai_run_quality_label(
self,
@@ -1187,13 +1322,133 @@ class SqlAlchemyAiPlatformRepository:
cutoff = datetime.now(UTC) - timedelta(days=max(days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
with self._session_factory() as session:
action_rows = session.scalars(
select(AiQualityImprovementActionModel)
.order_by(AiQualityImprovementActionModel.updated_at.desc())
.limit(20)
).all()
action_days = [row.evaluation_days for row in action_rows]
stats_cutoff = datetime.now(UTC) - timedelta(days=max([days, *action_days], default=days) - 1)
stats_cutoff = stats_cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= stats_cutoff)
.order_by(AiRunAuditModel.created_at.desc())
).all()
all_runs = [_ai_run_audit_from_model(row) for row in rows]
dashboard_runs = [
run for run in all_runs if _ensure_utc(run.created_at) >= cutoff
]
actions = _materialize_improvement_actions(
[_quality_improvement_action_from_model(row) for row in action_rows],
all_runs,
)
return _build_quality_dashboard(
dashboard_runs,
days=days,
pending_limit=pending_limit,
improvement_actions=_filter_improvement_actions(actions, limit=20),
)
def list_quality_improvement_actions(
self,
*,
status: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]:
with self._session_factory() as session:
statement = select(AiQualityImprovementActionModel).order_by(
AiQualityImprovementActionModel.updated_at.desc()
)
if status:
statement = statement.where(AiQualityImprovementActionModel.status == status)
rows = session.scalars(statement.limit(limit)).all()
max_days = max((row.evaluation_days for row in rows), default=14)
cutoff = datetime.now(UTC) - timedelta(days=max(max_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
audit_rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= cutoff)
.order_by(AiRunAuditModel.created_at.desc())
).all()
runs = [_ai_run_audit_from_model(row) for row in rows]
return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit)
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
actions = _materialize_improvement_actions(
[_quality_improvement_action_from_model(row) for row in rows],
runs,
)
return _filter_improvement_actions(actions, status=status, limit=limit)
def create_quality_improvement_action(
self,
payload: AiQualityImprovementActionCreate,
) -> AiQualityImprovementAction:
now = datetime.now(UTC)
cutoff = now - timedelta(days=max(payload.evaluation_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
with self._session_factory.begin() as session:
audit_rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= cutoff)
.order_by(AiRunAuditModel.created_at.desc())
).all()
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
baseline = _calculate_improvement_action_stats(payload, runs)
row = AiQualityImprovementActionModel(
id=f"qa-action-{uuid4().hex[:12]}",
dimension=payload.dimension,
key=payload.key,
label=payload.label,
action_type=payload.action_type,
title=payload.title or _default_improvement_action_title(payload),
description=payload.description,
owner=payload.owner,
priority=payload.priority,
status="open",
source_label_codes_json=_encode_json_list(payload.source_label_codes),
source_run_ids_json=_encode_json_list(payload.source_run_ids),
root_cause_hint=payload.root_cause_hint,
evaluation_days=payload.evaluation_days,
baseline_total_runs=baseline["total_runs"],
baseline_affected_runs=baseline["affected_runs"],
baseline_pending_labels=baseline["pending_labels"],
baseline_quality_score=baseline["quality_score"],
created_at=now,
updated_at=now,
)
session.add(row)
session.flush()
return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs)
def update_quality_improvement_action(
self,
action_id: str,
payload: AiQualityImprovementActionUpdate,
) -> AiQualityImprovementAction:
now = datetime.now(UTC)
with self._session_factory.begin() as session:
row = session.get(AiQualityImprovementActionModel, action_id)
if row is None:
raise KeyError(action_id)
row.status = payload.status
if payload.owner is not None:
row.owner = payload.owner
if payload.priority is not None:
row.priority = payload.priority
if payload.resolution_note is not None:
row.resolution_note = payload.resolution_note
row.updated_at = now
row.resolved_at = now if payload.status in {"resolved", "false_positive", "closed"} else None
cutoff = now - timedelta(days=max(row.evaluation_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
audit_rows = session.scalars(
select(AiRunAuditModel)
.where(AiRunAuditModel.created_at >= cutoff)
.order_by(AiRunAuditModel.created_at.desc())
).all()
runs = [_ai_run_audit_from_model(row) for row in audit_rows]
session.flush()
return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs)
def update_ai_run_quality_label(
self,
@@ -1453,6 +1708,33 @@ def _ai_run_audit_from_model(row: AiRunAuditModel) -> AiRunAudit:
)
def _quality_improvement_action_from_model(row: AiQualityImprovementActionModel) -> AiQualityImprovementAction:
return AiQualityImprovementAction(
id=row.id,
dimension=row.dimension,
key=row.key,
label=row.label,
action_type=row.action_type,
title=row.title,
description=row.description,
owner=row.owner,
priority=row.priority,
status=row.status,
source_label_codes=[str(item) for item in _decode_json_list(row.source_label_codes_json)],
source_run_ids=[str(item) for item in _decode_json_list(row.source_run_ids_json)],
root_cause_hint=row.root_cause_hint,
evaluation_days=row.evaluation_days,
baseline_total_runs=row.baseline_total_runs,
baseline_affected_runs=row.baseline_affected_runs,
baseline_pending_labels=row.baseline_pending_labels,
baseline_quality_score=row.baseline_quality_score,
resolution_note=row.resolution_note,
created_at=_ensure_utc(row.created_at),
updated_at=_ensure_utc(row.updated_at),
resolved_at=_ensure_utc(row.resolved_at) if row.resolved_at else None,
)
def _ensure_utc(value: datetime) -> datetime:
if value.tzinfo is None:
return value.replace(tzinfo=UTC)
@@ -1464,6 +1746,7 @@ def _build_quality_dashboard(
*,
days: int,
pending_limit: int,
improvement_actions: list[AiQualityImprovementAction] | None = None,
) -> AiQualityDashboard:
normalized_days = max(days, 1)
today = datetime.now(UTC).date()
@@ -1541,6 +1824,7 @@ def _build_quality_dashboard(
daily_metrics=ordered_daily,
pending_labels=pending_labels,
attribution=attribution,
improvement_actions=improvement_actions or [],
)
@@ -1754,6 +2038,137 @@ def _finalize_attribution_groups(groups: dict[str, dict[str, object]]) -> list[A
)[:5]
def _materialize_improvement_actions(
actions: Iterable[AiQualityImprovementAction],
runs: list[AiRunAudit],
) -> list[AiQualityImprovementAction]:
return [_materialize_improvement_action(action, runs) for action in actions]
def _materialize_improvement_action(
action: AiQualityImprovementAction,
runs: list[AiRunAudit],
) -> AiQualityImprovementAction:
current = _calculate_improvement_action_stats(action, runs)
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
current_rate = _safe_rate(current["affected_runs"], current["total_runs"])
return action.model_copy(
update={
"current_total_runs": current["total_runs"],
"current_affected_runs": current["affected_runs"],
"current_pending_labels": current["pending_labels"],
"current_quality_score": current["quality_score"],
"affected_rate_delta": round(current_rate - baseline_rate, 4),
"pending_labels_delta": current["pending_labels"] - action.baseline_pending_labels,
"quality_score_delta": round(current["quality_score"] - action.baseline_quality_score, 2),
}
)
def _filter_improvement_actions(
actions: list[AiQualityImprovementAction],
*,
status: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]:
filtered = [action for action in actions if status is None or action.status == status]
status_order = {"open": 0, "in_progress": 1, "resolved": 2, "false_positive": 3, "closed": 4}
priority_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
return sorted(
filtered,
key=lambda action: (
status_order.get(action.status, 9),
priority_order.get(action.priority, 9),
-_ensure_utc(action.updated_at).timestamp(),
),
)[: max(limit, 0)]
def _calculate_improvement_action_stats(
action: AiQualityImprovementActionCreate | AiQualityImprovementAction,
runs: list[AiRunAudit],
) -> dict[str, int | float]:
cutoff = datetime.now(UTC) - timedelta(days=max(action.evaluation_days - 1, 0))
cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0)
matched_runs: list[AiRunAudit] = []
affected_runs: set[str] = set()
pending_labels = 0
score_sum = 0
for run in runs:
if _ensure_utc(run.created_at) < cutoff:
continue
if not _run_matches_improvement_dimension(run, action):
continue
matched_runs.append(run)
score_sum += run.quality_score
issue_labels = _matching_improvement_issue_labels(run, action)
if issue_labels:
affected_runs.add(run.id)
pending_labels += sum(1 for label in issue_labels if label.review_status == "pending")
total_runs = len(matched_runs)
return {
"total_runs": total_runs,
"affected_runs": len(affected_runs),
"pending_labels": pending_labels,
"quality_score": round(score_sum / total_runs, 2) if total_runs else 0.0,
}
def _run_matches_improvement_dimension(
run: AiRunAudit,
action: AiQualityImprovementActionCreate | AiQualityImprovementAction,
) -> bool:
if action.dimension == "model":
return f"{run.provider or 'unknown'} / {run.model or 'unknown'}" == action.key
if action.dimension == "prompt_template":
return (run.prompt_template_id or "unknown") == action.key
if action.dimension == "knowledge_base":
knowledge_base_ids = {
source.knowledge_base_id for source in run.retrieved_sources if source.knowledge_base_id
}
if action.key in knowledge_base_ids:
return True
return action.key == "no_knowledge_source" and not knowledge_base_ids and _has_source_issue(run)
if action.dimension == "label_code":
return any(label.code == action.key for label in run.quality_labels)
return False
def _matching_improvement_issue_labels(
run: AiRunAudit,
action: AiQualityImprovementActionCreate | AiQualityImprovementAction,
) -> list[AiQualityLabel]:
issue_labels = [label for label in run.quality_labels if label.severity in {"warning", "error"}]
if action.dimension == "label_code":
return [label for label in issue_labels if label.code == action.key]
source_label_codes = set(action.source_label_codes)
if source_label_codes:
return [label for label in issue_labels if label.code in source_label_codes]
return issue_labels
def _has_source_issue(run: AiRunAudit) -> bool:
return any(label.code in {"source_insufficient", "source_limited"} for label in run.quality_labels)
def _safe_rate(numerator: int | float, denominator: int | float) -> float:
return round(float(numerator) / float(denominator), 4) if denominator else 0.0
def _default_improvement_action_title(action: AiQualityImprovementActionCreate) -> str:
action_names = {
"investigate": "Investigate",
"tune_prompt": "Tune prompt",
"improve_knowledge": "Improve knowledge base",
"tune_model": "Tune model config",
"mark_false_positive": "Review false positive",
}
prefix = action_names.get(action.action_type, "Improve")
return f"{prefix}: {action.label}"
def _label_title(label: AiQualityLabel) -> str:
return label.title or label.code
yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py
+88
View File
@@ -4,6 +4,8 @@ from yuqei_ai_platform_api.config import AiPlatformSettings
from yuqei_ai_platform_api.embeddings import EmbeddingVector
from yuqei_ai_platform_api.main import create_app
from yuqei_ai_platform_api.repository import (
AiQualityImprovementActionCreate,
AiQualityImprovementActionUpdate,
AiQualityLabelReviewUpdate,
AiQualityRuleConfigCreate,
AiRunAudit,
@@ -199,6 +201,67 @@ def test_quality_dashboard_summarizes_daily_metrics_and_pending_labels() -> None
assert updated_dashboard["summary"]["pending_labels"] == dashboard["summary"]["pending_labels"] - 1
def test_quality_attribution_can_create_and_resolve_improvement_action() -> None:
client = make_client()
response = client.post("/api/v1/legal/qa", json={"question": "How should probation be agreed?"})
assert response.status_code == 200
dashboard_response = client.get("/api/v1/audit/quality-dashboard", params={"days": 7, "pending_limit": 10})
assert dashboard_response.status_code == 200
dashboard = dashboard_response.json()
model_attribution = dashboard["attribution"]["by_model"][0]
create_response = client.post(
"/api/v1/audit/quality-improvement-actions",
json={
"dimension": model_attribution["dimension"],
"key": model_attribution["key"],
"label": model_attribution["label"],
"action_type": "tune_model",
"description": "Check provider timeout and model configuration.",
"owner": "admin",
"priority": "high",
"source_label_codes": model_attribution["top_label_codes"],
"root_cause_hint": model_attribution["root_cause_hint"],
"evaluation_days": 7,
},
)
assert create_response.status_code == 200
created = create_response.json()
assert created["status"] == "open"
assert created["baseline_total_runs"] == 1
assert created["baseline_affected_runs"] == 1
assert created["current_affected_runs"] == 1
list_response = client.get(
"/api/v1/audit/quality-improvement-actions",
params={"status": "open", "limit": 10},
)
assert list_response.status_code == 200
assert list_response.json()[0]["id"] == created["id"]
progress_response = client.patch(
f"/api/v1/audit/quality-improvement-actions/{created['id']}",
json={"status": "in_progress", "owner": "ops", "priority": "critical"},
)
assert progress_response.status_code == 200
assert progress_response.json()["status"] == "in_progress"
assert progress_response.json()["owner"] == "ops"
resolved_response = client.patch(
f"/api/v1/audit/quality-improvement-actions/{created['id']}",
json={"status": "resolved", "resolution_note": "Provider config was tuned."},
)
assert resolved_response.status_code == 200
resolved = resolved_response.json()
assert resolved["status"] == "resolved"
assert resolved["resolved_at"]
updated_dashboard = client.get("/api/v1/audit/quality-dashboard", params={"days": 7}).json()
assert any(action["id"] == created["id"] for action in updated_dashboard["improvement_actions"])
def test_knowledge_document_can_be_added_and_searched() -> None:
client = make_client()
@@ -545,6 +608,12 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None:
"title": "Source covered",
"detail": "Sufficient retrieved sources were recorded.",
},
{
"code": "source_limited",
"severity": "warning",
"title": "Source limited",
"detail": "Only one source was used.",
},
],
quality_summary="passed: 0 error(s), 0 warning(s), score 100.",
status="succeeded",
@@ -596,3 +665,22 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None:
assert updated_audit.quality_labels[0].review_status == "resolved"
assert updated_audit.quality_labels[0].review_note == "已确认依据有效。"
assert updated_audit.quality_labels[0].reviewed_at is not None
action = reloaded.create_quality_improvement_action(
AiQualityImprovementActionCreate(
dimension="model",
key="deepseek / deepseek-chat",
label="deepseek / deepseek-chat",
action_type="tune_model",
source_label_codes=["source_limited"],
evaluation_days=14,
)
)
assert action.baseline_total_runs == 1
assert action.baseline_affected_runs == 1
assert reloaded.list_quality_improvement_actions()[0].id == action.id
updated_action = reloaded.update_quality_improvement_action(
action.id,
AiQualityImprovementActionUpdate(status="resolved", resolution_note="done"),
)
assert updated_action.status == "resolved"
assert updated_action.resolved_at is not None
yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py
+6
View File
@@ -5,6 +5,9 @@ from yuqei_sdk.ai_platform import (
AiQualityDashboard,
AiQualityDashboardSummary,
AiQualityDailyMetric,
AiQualityImprovementAction,
AiQualityImprovementActionCreate,
AiQualityImprovementActionUpdate,
AiQualityLabel,
AiQualityLabelReviewUpdate,
AiQualityPendingLabel,
@@ -39,6 +42,9 @@ __all__ = [
"AiQualityDashboard",
"AiQualityDashboardSummary",
"AiQualityDailyMetric",
"AiQualityImprovementAction",
"AiQualityImprovementActionCreate",
"AiQualityImprovementActionUpdate",
"AiQualityLabel",
"AiQualityLabelReviewUpdate",
"AiQualityPendingLabel",
yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py
+91
View File
@@ -321,11 +321,55 @@ class AiQualityAttribution(BaseModel):
by_label_code: list[AiQualityAttributionItem] = Field(default_factory=list)
class AiQualityImprovementActionCreate(BaseModel):
dimension: str
key: str = Field(min_length=1, max_length=256)
label: str = Field(min_length=1, max_length=256)
action_type: str = "investigate"
title: str | None = Field(default=None, max_length=256)
description: str | None = None
owner: str | None = Field(default=None, max_length=128)
priority: str = "medium"
source_label_codes: list[str] = Field(default_factory=list)
source_run_ids: list[str] = Field(default_factory=list)
root_cause_hint: str | None = None
evaluation_days: int = 14
class AiQualityImprovementActionUpdate(BaseModel):
status: str
owner: str | None = Field(default=None, max_length=128)
priority: str | None = None
resolution_note: str | None = None
class AiQualityImprovementAction(AiQualityImprovementActionCreate):
id: str
title: str
status: str = "open"
baseline_total_runs: int = 0
baseline_affected_runs: int = 0
baseline_pending_labels: int = 0
baseline_quality_score: float = 0.0
current_total_runs: int = 0
current_affected_runs: int = 0
current_pending_labels: int = 0
current_quality_score: float = 0.0
affected_rate_delta: float = 0.0
pending_labels_delta: int = 0
quality_score_delta: float = 0.0
resolution_note: str | None = None
created_at: str
updated_at: str
resolved_at: str | None = None
class AiQualityDashboard(BaseModel):
summary: AiQualityDashboardSummary
daily_metrics: list[AiQualityDailyMetric] = Field(default_factory=list)
pending_labels: list[AiQualityPendingLabel] = Field(default_factory=list)
attribution: AiQualityAttribution = Field(default_factory=AiQualityAttribution)
improvement_actions: list[AiQualityImprovementAction] = Field(default_factory=list)
class AiPlatformClient:
@@ -605,6 +649,53 @@ class AiPlatformClient:
response.raise_for_status()
return AiQualityDashboard.model_validate(response.json())
def list_quality_improvement_actions(
self,
*,
status: str | None = None,
limit: int = 20,
context: RequestContext | None = None,
) -> list[AiQualityImprovementAction]:
params: dict[str, Any] = {"limit": limit}
if status:
params["status"] = status
response = self._client.get(
f"{self._api_prefix}/audit/quality-improvement-actions",
params=params,
headers=(context or RequestContext()).to_headers(),
)
response.raise_for_status()
return [AiQualityImprovementAction.model_validate(item) for item in response.json()]
def create_quality_improvement_action(
self,
action: AiQualityImprovementActionCreate,
*,
context: RequestContext | None = None,
) -> AiQualityImprovementAction:
response = self._client.post(
f"{self._api_prefix}/audit/quality-improvement-actions",
json=action.model_dump(exclude_none=True),
headers=(context or RequestContext()).to_headers(),
)
response.raise_for_status()
return AiQualityImprovementAction.model_validate(response.json())
def update_quality_improvement_action(
self,
action_id: str,
update: AiQualityImprovementActionUpdate,
*,
context: RequestContext | None = None,
) -> AiQualityImprovementAction:
response = self._client.patch(
f"{self._api_prefix}/audit/quality-improvement-actions/{action_id}",
json=update.model_dump(exclude_none=True),
headers=(context or RequestContext()).to_headers(),
)
response.raise_for_status()
return AiQualityImprovementAction.model_validate(response.json())
def update_ai_run_quality_label(
self,
run_id: str,
yuqei-ai-sdk-python/tests/test_ai_platform.py
+94
View File
@@ -5,6 +5,8 @@ import httpx
from yuqei_sdk import (
AiPlatformClient,
AiQualityImprovementActionCreate,
AiQualityImprovementActionUpdate,
AiQualityLabelReviewUpdate,
AiQualityRuleConfigUpdate,
KnowledgeDocumentCreate,
@@ -91,6 +93,44 @@ def test_ai_platform_client_posts_legal_qa_with_context_headers() -> None:
def test_ai_platform_client_covers_admin_data_endpoints() -> None:
seen_paths: list[str] = []
def quality_action_payload(
*,
status: str = "open",
resolution_note: str | None = None,
resolved_at: str | None = None,
) -> dict[str, object]:
return {
"id": "qa-action-1",
"dimension": "model",
"key": "deepseek / deepseek-chat",
"label": "deepseek / deepseek-chat",
"action_type": "tune_model",
"title": "Tune model config: deepseek / deepseek-chat",
"description": "Check provider timeout and model configuration.",
"owner": "admin",
"priority": "high",
"status": status,
"source_label_codes": ["source_limited"],
"source_run_ids": ["run-1"],
"root_cause_hint": "Model quality issues are concentrated.",
"evaluation_days": 7,
"baseline_total_runs": 1,
"baseline_affected_runs": 1,
"baseline_pending_labels": 1,
"baseline_quality_score": 90.0,
"current_total_runs": 1,
"current_affected_runs": 1,
"current_pending_labels": 1,
"current_quality_score": 90.0,
"affected_rate_delta": 0.0,
"pending_labels_delta": 0,
"quality_score_delta": 0.0,
"resolution_note": resolution_note,
"created_at": "2026-06-22T00:00:00Z",
"updated_at": "2026-06-22T00:02:00Z",
"resolved_at": resolved_at,
}
def handler(request: httpx.Request) -> httpx.Response:
seen_paths.append(request.url.path)
if request.url.path == "/api/v1/model-configs" and request.method == "POST":
@@ -529,8 +569,33 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
}
],
},
"improvement_actions": [quality_action_payload()],
},
)
if request.url.path == "/api/v1/audit/quality-improvement-actions" and request.method == "POST":
payload = json.loads(request.read().decode("utf-8"))
assert payload["dimension"] == "model"
assert payload["action_type"] == "tune_model"
assert payload["source_label_codes"] == ["source_limited"]
return httpx.Response(200, json=quality_action_payload())
if request.url.path == "/api/v1/audit/quality-improvement-actions":
query = parse_qs(request.url.query.decode())
assert query["status"] == ["open"]
assert query["limit"] == ["5"]
return httpx.Response(200, json=[quality_action_payload()])
if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1":
payload = json.loads(request.read().decode("utf-8"))
assert request.method == "PATCH"
assert payload["status"] == "resolved"
assert payload["resolution_note"] == "Provider config was tuned."
return httpx.Response(
200,
json=quality_action_payload(
status="resolved",
resolution_note="Provider config was tuned.",
resolved_at="2026-06-22T00:05:00Z",
),
)
if request.url.path == "/api/v1/audit/ai-runs/run-1/quality-labels/citation_present":
payload = json.loads(request.read().decode("utf-8"))
assert request.method == "PATCH"
@@ -667,6 +732,34 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
assert dashboard.attribution.by_model[0].key == "deepseek / deepseek-chat"
assert dashboard.attribution.by_label_code[0].top_label_codes == ["source_limited"]
assert dashboard.attribution.by_label_code[0].root_cause_hint is not None
assert dashboard.improvement_actions[0].id == "qa-action-1"
created_action = client.create_quality_improvement_action(
AiQualityImprovementActionCreate(
dimension="model",
key="deepseek / deepseek-chat",
label="deepseek / deepseek-chat",
action_type="tune_model",
description="Check provider timeout and model configuration.",
owner="admin",
priority="high",
source_label_codes=["source_limited"],
source_run_ids=["run-1"],
evaluation_days=7,
)
)
assert created_action.status == "open"
assert created_action.baseline_affected_runs == 1
listed_actions = client.list_quality_improvement_actions(status="open", limit=5)
assert listed_actions[0].id == "qa-action-1"
updated_action = client.update_quality_improvement_action(
"qa-action-1",
AiQualityImprovementActionUpdate(
status="resolved",
resolution_note="Provider config was tuned.",
),
)
assert updated_action.status == "resolved"
assert updated_action.resolved_at == "2026-06-22T00:05:00Z"
updated_audit = client.update_ai_run_quality_label(
"run-1",
"citation_present",
@@ -680,4 +773,5 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
assert updated_audit.quality_labels[0].reviewed_at == "2026-06-22T00:01:00Z"
assert "/api/v1/audit/ai-runs" in seen_paths
assert "/api/v1/audit/quality-dashboard" in seen_paths
assert "/api/v1/audit/quality-improvement-actions" in seen_paths
assert "/api/v1/quality-rules/default" in seen_paths
yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx
+352 -25
View File
@@ -3,7 +3,18 @@
import Link from "next/link";
import type { ReactNode } from "react";
import { useEffect, useState } from "react";
import { AlertTriangle, Clock3, Database, RefreshCcw, ShieldCheck, Undo2 } from "lucide-react";
import {
AlertTriangle,
CheckCircle2,
Clock3,
Database,
PlayCircle,
RefreshCcw,
ShieldCheck,
Undo2,
Wrench,
XCircle
} from "lucide-react";
type QualityDailyMetric = {
date: string;
@@ -59,11 +70,44 @@ type QualityAttribution = {
by_label_code: QualityAttributionItem[];
};
type QualityImprovementAction = {
id: string;
dimension: string;
key: string;
label: string;
action_type: string;
title: string;
description?: string | null;
owner?: string | null;
priority: string;
status: string;
source_label_codes: string[];
source_run_ids: string[];
root_cause_hint?: string | null;
evaluation_days: number;
baseline_total_runs: number;
baseline_affected_runs: number;
baseline_pending_labels: number;
baseline_quality_score: number;
current_total_runs: number;
current_affected_runs: number;
current_pending_labels: number;
current_quality_score: number;
affected_rate_delta: number;
pending_labels_delta: number;
quality_score_delta: number;
resolution_note?: string | null;
created_at: string;
updated_at: string;
resolved_at?: string | null;
};
type QualityDashboard = {
summary: QualityDailyMetric & { days: number };
daily_metrics: QualityDailyMetric[];
pending_labels: QualityPendingLabel[];
attribution?: QualityAttribution;
improvement_actions?: QualityImprovementAction[];
};
type LoadState =
@@ -77,9 +121,42 @@ const severityTone: Record<string, string> = {
error: "tag-danger"
};
const statusTone: Record<string, string> = {
open: "tag-warning",
in_progress: "tag-primary",
resolved: "tag-success",
false_positive: "",
closed: ""
};
const statusLabels: Record<string, string> = {
open: "待处理",
in_progress: "处理中",
resolved: "已完成",
false_positive: "误报关闭",
closed: "已关闭"
};
const actionTypeLabels: Record<string, string> = {
investigate: "排查原因",
tune_prompt: "调整 Prompt",
improve_knowledge: "补知识库",
tune_model: "调模型参数",
mark_false_positive: "处理误报"
};
const dimensionLabels: Record<string, string> = {
model: "模型",
prompt_template: "Prompt",
knowledge_base: "知识库",
label_code: "标签"
};
export default function AiQualityDashboardPage() {
const [state, setState] = useState<LoadState>({ status: "loading" });
const [days, setDays] = useState(14);
const [actionMessage, setActionMessage] = useState<string | null>(null);
const [actionBusyKey, setActionBusyKey] = useState<string | null>(null);
async function loadDashboard(nextDays = days) {
setState({ status: "loading" });
@@ -107,18 +184,83 @@ export default function AiQualityDashboardPage() {
}, []);
const dashboard = state.status === "ready" ? state.dashboard : null;
const improvementActions = dashboard?.improvement_actions ?? [];
const openActionCount = improvementActions.filter((action) => ["open", "in_progress"].includes(action.status)).length;
function changeDays(value: string) {
const nextDays = Number(value);
setDays(nextDays);
setActionMessage(null);
void loadDashboard(nextDays);
}
async function createActionFromAttribution(item: QualityAttributionItem) {
const busyKey = attributionBusyKey(item);
setActionBusyKey(busyKey);
setActionMessage(null);
try {
const response = await fetch("/api/ai-platform/audit/quality-improvement-actions", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
dimension: item.dimension,
key: item.key,
label: item.label,
action_type: inferActionType(item),
description: item.root_cause_hint || `来自 ${dimensionLabels[item.dimension] ?? item.dimension} 归因的质量改进项。`,
owner: "admin",
priority: inferPriority(item),
source_label_codes: item.top_label_codes,
root_cause_hint: item.root_cause_hint,
evaluation_days: days
})
});
const payload = await response.json().catch(() => null);
if (!response.ok) {
throw new Error(payload?.message ?? `HTTP ${response.status}`);
}
setActionMessage("已生成改进项，可以在下方队列继续处理。");
await loadDashboard(days);
} catch (error) {
setActionMessage(error instanceof Error ? error.message : "生成改进项失败");
} finally {
setActionBusyKey(null);
}
}
async function updateAction(action: QualityImprovementAction, status: string) {
const busyKey = actionBusyKeyFor(action, status);
setActionBusyKey(busyKey);
setActionMessage(null);
try {
const response = await fetch(`/api/ai-platform/audit/quality-improvement-actions/${encodeURIComponent(action.id)}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
status,
owner: "admin",
resolution_note: defaultResolutionNote(status)
})
});
const payload = await response.json().catch(() => null);
if (!response.ok) {
throw new Error(payload?.message ?? `HTTP ${response.status}`);
}
setActionMessage("改进项状态已更新，指标会随新的 AI 调用继续回看。");
await loadDashboard(days);
} catch (error) {
setActionMessage(error instanceof Error ? error.message : "更新改进项失败");
} finally {
setActionBusyKey(null);
}
}
return (
<>
<div className="page-heading">
<div>
<h1>AI 质量运营</h1>
<p>按日期查看质量通过率、fallback、来源不足、耗时异常和待处理标签。</p>
<p>按日期跟踪质量通过率、fallback、来源不足、耗时异常，并把高频问题转成可处理的改进项。</p>
</div>
<div className="toolbar toolbar-inline">
<select className="search-input select-input" value={days} onChange={(event) => changeDays(event.target.value)}>
@@ -127,7 +269,7 @@ export default function AiQualityDashboardPage() {
<option value={30}>最近 30 天</option>
<option value={90}>最近 90 天</option>
</select>
<button className="button button-soft" onClick={() => void loadDashboard()}>
<button className="button button-soft" onClick={() => void loadDashboard()} type="button">
<RefreshCcw size={16} />
刷新
</button>
@@ -151,13 +293,61 @@ export default function AiQualityDashboardPage() {
<Metric icon={<AlertTriangle size={18} />} label="待处理标签" value={String(dashboard.summary.pending_labels)} />
</div>
{actionMessage && <div className="inline-message">{actionMessage}</div>}
<div className="attribution-grid">
<AttributionPanel title="模型归因" items={dashboard.attribution?.by_model ?? []} />
<AttributionPanel title="Prompt 归因" items={dashboard.attribution?.by_prompt_template ?? []} />
<AttributionPanel title="知识库归因" items={dashboard.attribution?.by_knowledge_base ?? []} />
<AttributionPanel title="标签归因" items={dashboard.attribution?.by_label_code ?? []} />
<AttributionPanel
title="模型归因"
items={dashboard.attribution?.by_model ?? []}
busyKey={actionBusyKey}
onCreateAction={createActionFromAttribution}
/>
<AttributionPanel
title="Prompt 归因"
items={dashboard.attribution?.by_prompt_template ?? []}
busyKey={actionBusyKey}
onCreateAction={createActionFromAttribution}
/>
<AttributionPanel
title="知识库归因"
items={dashboard.attribution?.by_knowledge_base ?? []}
busyKey={actionBusyKey}
onCreateAction={createActionFromAttribution}
/>
<AttributionPanel
title="标签归因"
items={dashboard.attribution?.by_label_code ?? []}
busyKey={actionBusyKey}
onCreateAction={createActionFromAttribution}
/>
</div>
<section className="panel improvement-panel">
<div className="panel-header">
<h2 className="panel-title">
<Wrench size={18} />
改进动作闭环
</h2>
<span className="tag tag-primary">待推进 {openActionCount}</span>
</div>
<div className="panel-body">
{improvementActions.length === 0 ? (
<div className="empty-state empty-state-compact">暂无改进项，可先从上方归因卡片生成。</div>
) : (
<div className="quality-action-list">
{improvementActions.map((action) => (
<QualityActionItem
action={action}
busyKey={actionBusyKey}
key={action.id}
onUpdate={updateAction}
/>
))}
</div>
)}
</div>
</section>
<div className="dashboard-grid">
<section className="panel">
<div className="panel-header">
@@ -243,7 +433,17 @@ function Metric({ icon, label, value }: { icon: ReactNode; label: string; value:
);
}
function AttributionPanel({ title, items }: { title: string; items: QualityAttributionItem[] }) {
function AttributionPanel({
title,
items,
busyKey,
onCreateAction
}: {
title: string;
items: QualityAttributionItem[];
busyKey: string | null;
onCreateAction: (item: QualityAttributionItem) => void | Promise<void>;
}) {
return (
<section className="panel">
<div className="panel-header">
@@ -254,34 +454,161 @@ function AttributionPanel({ title, items }: { title: string; items: QualityAttri
{items.length === 0 ? (
<div className="empty-state empty-state-compact">暂无异常归因。</div>
) : (
items.map((item) => (
<div className="attribution-item" key={`${item.dimension}-${item.key}`}>
<div className="list-item-header">
<strong>{item.label}</strong>
<span className="tag tag-warning">{item.affected_runs}/{item.total_runs}</span>
items.map((item) => {
const busy = busyKey === attributionBusyKey(item);
return (
<div className="attribution-item" key={`${item.dimension}-${item.key}`}>
<div className="list-item-header">
<strong>{item.label}</strong>
<span className="tag tag-warning">{item.affected_runs}/{item.total_runs}</span>
</div>
<div className="trace-meta">
<span>待处理 {item.pending_labels}</span>
<span>warning {item.warning_labels}</span>
<span>error {item.error_labels}</span>
<span>均分 {item.average_quality_score}</span>
</div>
{item.top_label_codes.length > 0 && (
<span className="item-meta">常见标签：{item.top_label_codes.join("、")}</span>
)}
{item.root_cause_hint && <span className="item-meta">{item.root_cause_hint}</span>}
<div className="action-row">
<button className="button button-soft" disabled={busy} onClick={() => void onCreateAction(item)} type="button">
<Wrench size={16} />
{busy ? "生成中" : "转改进项"}
</button>
</div>
</div>
<div className="trace-meta">
<span>待处理 {item.pending_labels}</span>
<span>warning {item.warning_labels}</span>
<span>error {item.error_labels}</span>
<span>均分 {item.average_quality_score}</span>
</div>
{item.top_label_codes.length > 0 && (
<span className="item-meta">常见标签：{item.top_label_codes.join("、")}</span>
)}
{item.root_cause_hint && <span className="item-meta">{item.root_cause_hint}</span>}
</div>
))
);
})
)}
</div>
</section>
);
}
function QualityActionItem({
action,
busyKey,
onUpdate
}: {
action: QualityImprovementAction;
busyKey: string | null;
onUpdate: (action: QualityImprovementAction, status: string) => void | Promise<void>;
}) {
const busy = busyKey?.startsWith(`${action.id}:`) ?? false;
const baselineRate = safeRate(action.baseline_affected_runs, action.baseline_total_runs);
const currentRate = safeRate(action.current_affected_runs, action.current_total_runs);
return (
<div className="quality-action-item">
<div className="list-item-header">
<div>
<strong>{action.title}</strong>
<div className="trace-meta">
<span>{dimensionLabels[action.dimension] ?? action.dimension}</span>
<span>{actionTypeLabels[action.action_type] ?? action.action_type}</span>
<span>优先级 {action.priority}</span>
<span>{action.owner || "未分配"}</span>
</div>
</div>
<span className={`tag ${statusTone[action.status] ?? ""}`}>{statusLabels[action.status] ?? action.status}</span>
</div>
{(action.description || action.root_cause_hint) && (
<span className="item-meta">{action.description || action.root_cause_hint}</span>
)}
<div className="quality-action-stats">
<span>异常率 {formatPercent(baselineRate)} → {formatPercent(currentRate)}</span>
<span>待处理 {action.baseline_pending_labels} → {action.current_pending_labels}</span>
<span>质量分 {action.baseline_quality_score} → {action.current_quality_score}</span>
<span>变化 {formatSignedPercent(action.affected_rate_delta)}</span>
</div>
{action.resolution_note && <span className="item-meta">处理说明：{action.resolution_note}</span>}
<div className="action-row">
{action.status === "open" && (
<button className="button button-soft" disabled={busy} onClick={() => void onUpdate(action, "in_progress")} type="button">
<PlayCircle size={16} />
开始处理
</button>
)}
{["open", "in_progress"].includes(action.status) && (
<>
<button className="button button-primary" disabled={busy} onClick={() => void onUpdate(action, "resolved")} type="button">
<CheckCircle2 size={16} />
标记完成
</button>
<button className="button button-soft" disabled={busy} onClick={() => void onUpdate(action, "false_positive")} type="button">
<XCircle size={16} />
误报关闭
</button>
</>
)}
</div>
</div>
);
}
function inferActionType(item: QualityAttributionItem) {
if (item.dimension === "prompt_template") {
return "tune_prompt";
}
if (item.dimension === "knowledge_base") {
return "improve_knowledge";
}
if (item.dimension === "model") {
return "tune_model";
}
const codes = [item.key, ...item.top_label_codes].join(" ");
if (codes.includes("source")) {
return "improve_knowledge";
}
if (codes.includes("fallback") || codes.includes("token") || codes.includes("latency") || codes.includes("failed")) {
return "tune_model";
}
return "investigate";
}
function inferPriority(item: QualityAttributionItem) {
if (item.error_labels > 0 || item.fallback_runs > 0) {
return "high";
}
if (safeRate(item.affected_runs, item.total_runs) >= 0.5 || item.pending_labels >= 10) {
return "high";
}
return "medium";
}
function defaultResolutionNote(status: string) {
if (status === "resolved") {
return "已处理，等待后续 AI 调用数据验证异常率是否下降。";
}
if (status === "false_positive") {
return "已确认为误报，后续会结合质量规则继续校准。";
}
return undefined;
}
function attributionBusyKey(item: QualityAttributionItem) {
return `create:${item.dimension}:${item.key}`;
}
function actionBusyKeyFor(action: QualityImprovementAction, status: string) {
return `${action.id}:${status}`;
}
function safeRate(numerator: number, denominator: number) {
return denominator ? numerator / denominator : 0;
}
function formatPercent(value: number) {
return `${Math.round(value * 100)}%`;
}
function formatSignedPercent(value: number) {
const percent = Math.round(value * 100);
return `${percent > 0 ? "+" : ""}${percent}%`;
}
function formatShortDate(value: string) {
const date = new Date(`${value}T00:00:00`);
if (Number.isNaN(date.getTime())) {
yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/[actionId]/route.ts
+47
View File
@@ -0,0 +1,47 @@
import { NextResponse } from "next/server";
const aiPlatformBaseUrl =
process.env.AI_PLATFORM_API_BASE_URL ??
process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ??
"http://localhost:8101";
export async function PATCH(
request: Request,
context: { params: Promise<{ actionId: string }> }
) {
const { actionId } = await context.params;
const targetUrl = new URL(
`/api/v1/audit/quality-improvement-actions/${encodeURIComponent(actionId)}`,
aiPlatformBaseUrl
);
const body: unknown = await request.json().catch(() => null);
try {
const response = await fetch(targetUrl, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
cache: "no-store"
});
const payload: unknown = await response.json().catch(() => null);
if (!response.ok) {
return NextResponse.json(
{
message: "AI Platform quality improvement action service returned an error.",
status: response.status,
payload
},
{ status: response.status }
);
}
return NextResponse.json(payload);
} catch (error) {
return NextResponse.json(
{
message: error instanceof Error ? error.message : "AI Platform quality improvement action service is unavailable.",
baseUrl: aiPlatformBaseUrl
},
{ status: 502 }
);
}
}
yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/route.ts
+75
View File
@@ -0,0 +1,75 @@
import { NextResponse } from "next/server";
const aiPlatformBaseUrl =
process.env.AI_PLATFORM_API_BASE_URL ??
process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ??
"http://localhost:8101";
export async function GET(request: Request) {
const { searchParams } = new URL(request.url);
const status = searchParams.get("status");
const limit = searchParams.get("limit") ?? "20";
const targetUrl = new URL("/api/v1/audit/quality-improvement-actions", aiPlatformBaseUrl);
targetUrl.searchParams.set("limit", limit);
if (status) {
targetUrl.searchParams.set("status", status);
}
try {
const response = await fetch(targetUrl, { cache: "no-store" });
const payload: unknown = await response.json().catch(() => null);
if (!response.ok) {
return NextResponse.json(
{
message: "AI Platform quality improvement action service returned an error.",
status: response.status,
payload
},
{ status: response.status }
);
}
return NextResponse.json(payload);
} catch (error) {
return NextResponse.json(
{
message: error instanceof Error ? error.message : "AI Platform quality improvement action service is unavailable.",
baseUrl: aiPlatformBaseUrl
},
{ status: 502 }
);
}
}
export async function POST(request: Request) {
const targetUrl = new URL("/api/v1/audit/quality-improvement-actions", aiPlatformBaseUrl);
const body: unknown = await request.json().catch(() => null);
try {
const response = await fetch(targetUrl, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(body),
cache: "no-store"
});
const payload: unknown = await response.json().catch(() => null);
if (!response.ok) {
return NextResponse.json(
{
message: "AI Platform quality improvement action service returned an error.",
status: response.status,
payload
},
{ status: response.status }
);
}
return NextResponse.json(payload);
} catch (error) {
return NextResponse.json(
{
message: error instanceof Error ? error.message : "AI Platform quality improvement action service is unavailable.",
baseUrl: aiPlatformBaseUrl
},
{ status: 502 }
);
}
}
yuqei-contract-platform/apps/contract-web/src/app/globals.css
+46
View File
@@ -722,6 +722,52 @@ select {
background: #fbfcfe;
}
.action-row {
display: flex;
flex-wrap: wrap;
gap: 8px;
align-items: center;
}
.button:disabled {
cursor: not-allowed;
opacity: 0.58;
}
.improvement-panel {
margin-bottom: 16px;
}
.quality-action-list {
display: grid;
gap: 12px;
}
.quality-action-item {
display: grid;
gap: 10px;
min-width: 0;
padding: 14px;
border: 1px solid var(--line);
border-radius: 8px;
background: #fbfcfe;
}
.quality-action-stats {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
gap: 8px;
color: var(--muted);
font-size: 13px;
}
.quality-action-stats span {
min-width: 0;
padding: 8px 10px;
border-radius: 8px;
background: #f1f5f9;
}
.trend-list {
display: grid;
gap: 10px;