diff --git a/project-docs/V2-ai-quality-improvement-action-loop-2026-06-23.md b/project-docs/V2-ai-quality-improvement-action-loop-2026-06-23.md new file mode 100644 index 00000000..33def6a2 --- /dev/null +++ b/project-docs/V2-ai-quality-improvement-action-loop-2026-06-23.md @@ -0,0 +1,46 @@ +# V2 AI 质量改进动作闭环记录 - 2026-06-23 + +## 本轮目标 + +在已有 AI 质量看板和问题归因基础上，补齐“异常归因 -> 待处理改进项 -> 人工处理状态 -> 处理后异常率回看”的运营闭环。 + +## 已完成能力 + +1. AI Platform 后端新增 `ai_quality_improvement_actions` 持久化模型。 +2. 新增改进项创建、列表、状态更新接口： + - `GET /api/v1/audit/quality-improvement-actions` + - `POST /api/v1/audit/quality-improvement-actions` + - `PATCH /api/v1/audit/quality-improvement-actions/{action_id}` +3. 质量看板响应新增 `improvement_actions`，可直接展示待推进改进队列。 +4. 改进项记录维度包括模型、Prompt、知识库、标签类型，并保存来源标签、根因提示、负责人、优先级、处理说明。 +5. 改进项自动记录基线数据，并在看板中计算当前影响调用数、待处理标签数、质量分和异常率变化。 +6. Python SDK 补齐对应 DTO 和客户端方法，合同系统后续可通过正式 SDK 调用 AI 中台能力。 +7. OpenAPI 契约同步新增改进项接口和 schema。 +8. 合同系统后台 AI 质量运营页新增： + - 归因卡片一键“转改进项”。 + - 改进动作队列。 + - 开始处理、标记完成、误报关闭。 + - 基线/当前异常率、待处理标签、质量分对比。 +9. 合同系统新增 Next API 代理，浏览器端仍通过合同系统访问 AI Platform。 + +## 闭环使用方式 + +1. 管理员进入 `AI 质量运营` 页面。 +2. 查看模型、Prompt、知识库、标签归因。 +3. 对高频异常点击“转改进项”。 +4. 在改进动作队列中推进状态：待处理、处理中、已完成、误报关闭。 +5. 后续 AI 调用产生新数据后，看板会持续回看当前异常率是否低于创建时基线。 + +## 验证命令 + +```powershell +python -m pytest yuqei-ai-platform/services/ai-platform-api/tests yuqei-ai-sdk-python/tests -q +python -c "import yaml; yaml.safe_load(open('yuqei-ai-platform/packages/ai-openapi/openapi.yaml', encoding='utf-8')); print('openapi ok')" +npm run web:typecheck +npm run web:build +git diff --check +``` + +## 后续建议 + +下一步可以继续做 AI 质量改进项的运营增强：支持按负责人、优先级、状态筛选；支持关联具体 Prompt 版本、知识库导入批次、模型配置变更记录；支持改进完成后自动生成一份效果复盘。 diff --git a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml index 4048107c..300bc6a3 100644 --- a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml +++ b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml @@ -469,6 +469,73 @@ paths: application/json: schema: $ref: "#/components/schemas/AiQualityDashboard" + /api/v1/audit/quality-improvement-actions: + get: + operationId: listQualityImprovementActions + summary: List AI quality improvement actions + parameters: + - name: status + in: query + required: false + schema: + type: string + enum: [open, in_progress, resolved, false_positive, closed] + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + minimum: 1 + maximum: 100 + responses: + "200": + description: AI quality improvement actions + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AiQualityImprovementAction" + post: + operationId: createQualityImprovementAction + summary: Create an AI quality improvement action from attribution + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityImprovementActionCreate" + responses: + "200": + description: Created AI quality improvement action + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityImprovementAction" + /api/v1/audit/quality-improvement-actions/{action_id}: + patch: + operationId: updateQualityImprovementAction + summary: Update an AI quality improvement action status + parameters: + - name: action_id + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityImprovementActionUpdate" + responses: + "200": + description: Updated AI quality improvement action + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityImprovementAction" /api/v1/audit/ai-runs/{run_id}/quality-labels/{label_code}: patch: operationId: updateAiRunQualityLabel @@ -1452,6 +1519,136 @@ components: type: array items: $ref: "#/components/schemas/AiQualityAttributionItem" + AiQualityImprovementActionCreate: + type: object + required: + - dimension + - key + - label + properties: + dimension: + type: string + enum: [model, prompt_template, knowledge_base, label_code] + key: + type: string + minLength: 1 + maxLength: 256 + label: + type: string + minLength: 1 + maxLength: 256 + action_type: + type: string + enum: [investigate, tune_prompt, improve_knowledge, tune_model, mark_false_positive] + default: investigate + title: + type: string + nullable: true + maxLength: 256 + description: + type: string + nullable: true + owner: + type: string + nullable: true + maxLength: 128 + priority: + type: string + enum: [low, medium, high, critical] + default: medium + source_label_codes: + type: array + items: + type: string + source_run_ids: + type: array + items: + type: string + root_cause_hint: + type: string + nullable: true + evaluation_days: + type: integer + minimum: 1 + maximum: 90 + default: 14 + AiQualityImprovementActionUpdate: + type: object + required: + - status + properties: + status: + type: string + enum: [open, in_progress, resolved, false_positive, closed] + owner: + type: string + nullable: true + maxLength: 128 + priority: + type: string + enum: [low, medium, high, critical] + nullable: true + resolution_note: + type: string + nullable: true + AiQualityImprovementAction: + allOf: + - $ref: "#/components/schemas/AiQualityImprovementActionCreate" + - type: object + required: + - id + - title + - status + - created_at + - updated_at + properties: + id: + type: string + title: + type: string + status: + type: string + enum: [open, in_progress, resolved, false_positive, closed] + default: open + baseline_total_runs: + type: integer + baseline_affected_runs: + type: integer + baseline_pending_labels: + type: integer + baseline_quality_score: + type: number + format: float + current_total_runs: + type: integer + current_affected_runs: + type: integer + current_pending_labels: + type: integer + current_quality_score: + type: number + format: float + affected_rate_delta: + type: number + format: float + pending_labels_delta: + type: integer + quality_score_delta: + type: number + format: float + resolution_note: + type: string + nullable: true + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + resolved_at: + type: string + format: date-time + nullable: true AiQualityDashboard: type: object properties: @@ -1467,6 +1664,10 @@ components: $ref: "#/components/schemas/AiQualityPendingLabel" attribution: $ref: "#/components/schemas/AiQualityAttribution" + improvement_actions: + type: array + items: + $ref: "#/components/schemas/AiQualityImprovementAction" AiRun: type: object required: diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py index b48ec1a1..bbbcfa87 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py @@ -16,6 +16,9 @@ from yuqei_ai_platform_api.repository import ( AiQualityLabel, AiQualityLabelReviewUpdate, AiQualityDashboard, + AiQualityImprovementAction, + AiQualityImprovementActionCreate, + AiQualityImprovementActionUpdate, AiQualityRuleConfig, AiQualityRuleConfigCreate, AiRunAudit, @@ -246,6 +249,41 @@ def create_app( ) -> AiQualityDashboard: return store.get_quality_dashboard(days=days, pending_limit=pending_limit) + @app.get( + f"{resolved_settings.api_prefix}/audit/quality-improvement-actions", + response_model=list[AiQualityImprovementAction], + tags=["audit"], + ) + def list_quality_improvement_actions( + status: str | None = Query(None, pattern="^(open|in_progress|resolved|false_positive|closed)$"), + limit: int = Query(20, ge=1, le=100), + ) -> list[AiQualityImprovementAction]: + return store.list_quality_improvement_actions(status=status, limit=limit) + + @app.post( + f"{resolved_settings.api_prefix}/audit/quality-improvement-actions", + response_model=AiQualityImprovementAction, + tags=["audit"], + ) + def create_quality_improvement_action( + payload: AiQualityImprovementActionCreate, + ) -> AiQualityImprovementAction: + return store.create_quality_improvement_action(payload) + + @app.patch( + f"{resolved_settings.api_prefix}/audit/quality-improvement-actions/{{action_id}}", + response_model=AiQualityImprovementAction, + tags=["audit"], + ) + def update_quality_improvement_action( + action_id: str, + payload: AiQualityImprovementActionUpdate, + ) -> AiQualityImprovementAction: + try: + return store.update_quality_improvement_action(action_id, payload) + except KeyError as error: + raise HTTPException(status_code=404, detail="AI quality improvement action not found.") from error + @app.patch( f"{resolved_settings.api_prefix}/audit/ai-runs/{{run_id}}/quality-labels/{{label_code}}", response_model=AiRunAudit, diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py index 48a0c78b..72b93f8c 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py @@ -151,3 +151,38 @@ class AiRunAuditModel(Base): default=lambda: datetime.now(UTC), index=True, ) + + +class AiQualityImprovementActionModel(Base): + __tablename__ = "ai_quality_improvement_actions" + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + dimension: Mapped[str] = mapped_column(String(64), index=True) + key: Mapped[str] = mapped_column(String(256), index=True) + label: Mapped[str] = mapped_column(String(256)) + action_type: Mapped[str] = mapped_column(String(64), index=True) + title: Mapped[str] = mapped_column(String(256)) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + owner: Mapped[str | None] = mapped_column(String(128), nullable=True) + priority: Mapped[str] = mapped_column(String(32), default="medium", index=True) + status: Mapped[str] = mapped_column(String(32), default="open", index=True) + source_label_codes_json: Mapped[str] = mapped_column(Text, default="[]") + source_run_ids_json: Mapped[str] = mapped_column(Text, default="[]") + root_cause_hint: Mapped[str | None] = mapped_column(Text, nullable=True) + evaluation_days: Mapped[int] = mapped_column(Integer, default=14) + baseline_total_runs: Mapped[int] = mapped_column(Integer, default=0) + baseline_affected_runs: Mapped[int] = mapped_column(Integer, default=0) + baseline_pending_labels: Mapped[int] = mapped_column(Integer, default=0) + baseline_quality_score: Mapped[float] = mapped_column(Float, default=0.0) + resolution_note: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + index=True, + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + index=True, + ) + resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py index 7e29532d..8f308c94 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py @@ -5,7 +5,7 @@ import math from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta from threading import Lock -from typing import Protocol +from typing import Iterable, Protocol from uuid import uuid4 from pydantic import BaseModel, Field @@ -24,6 +24,7 @@ from yuqei_ai_platform_api.embeddings import ( from yuqei_ai_platform_api.legal_qa import LegalCitation from yuqei_ai_platform_api.models import ( AiProviderConfigModel, + AiQualityImprovementActionModel, AiQualityRuleConfigModel, AiRunAuditModel, Base, @@ -307,6 +308,52 @@ class AiQualityAttribution(BaseModel): by_label_code: list[AiQualityAttributionItem] = Field(default_factory=list) +class AiQualityImprovementActionCreate(BaseModel): + dimension: str = Field(pattern="^(model|prompt_template|knowledge_base|label_code)$") + key: str = Field(min_length=1, max_length=256) + label: str = Field(min_length=1, max_length=256) + action_type: str = Field( + default="investigate", + pattern="^(investigate|tune_prompt|improve_knowledge|tune_model|mark_false_positive)$", + ) + title: str | None = Field(default=None, max_length=256) + description: str | None = None + owner: str | None = Field(default=None, max_length=128) + priority: str = Field(default="medium", pattern="^(low|medium|high|critical)$") + source_label_codes: list[str] = Field(default_factory=list) + source_run_ids: list[str] = Field(default_factory=list) + root_cause_hint: str | None = None + evaluation_days: int = Field(default=14, ge=1, le=90) + + +class AiQualityImprovementActionUpdate(BaseModel): + status: str = Field(pattern="^(open|in_progress|resolved|false_positive|closed)$") + owner: str | None = Field(default=None, max_length=128) + priority: str | None = Field(default=None, pattern="^(low|medium|high|critical)$") + resolution_note: str | None = None + + +class AiQualityImprovementAction(AiQualityImprovementActionCreate): + id: str + title: str + status: str = "open" + baseline_total_runs: int = 0 + baseline_affected_runs: int = 0 + baseline_pending_labels: int = 0 + baseline_quality_score: float = 0.0 + current_total_runs: int = 0 + current_affected_runs: int = 0 + current_pending_labels: int = 0 + current_quality_score: float = 0.0 + affected_rate_delta: float = 0.0 + pending_labels_delta: int = 0 + quality_score_delta: float = 0.0 + resolution_note: str | None = None + created_at: datetime + updated_at: datetime + resolved_at: datetime | None = None + + class AiQualityDashboardSummary(BaseModel): days: int total_runs: int = 0 @@ -329,6 +376,7 @@ class AiQualityDashboard(BaseModel): daily_metrics: list[AiQualityDailyMetric] = Field(default_factory=list) pending_labels: list[AiQualityPendingLabel] = Field(default_factory=list) attribution: AiQualityAttribution = Field(default_factory=AiQualityAttribution) + improvement_actions: list[AiQualityImprovementAction] = Field(default_factory=list) class AiPlatformRepository(Protocol): @@ -396,6 +444,24 @@ class AiPlatformRepository(Protocol): def get_quality_dashboard(self, *, days: int = 14, pending_limit: int = 20) -> AiQualityDashboard: ... + def list_quality_improvement_actions( + self, + *, + status: str | None = None, + limit: int = 20, + ) -> list[AiQualityImprovementAction]: ... + + def create_quality_improvement_action( + self, + payload: AiQualityImprovementActionCreate, + ) -> AiQualityImprovementAction: ... + + def update_quality_improvement_action( + self, + action_id: str, + payload: AiQualityImprovementActionUpdate, + ) -> AiQualityImprovementAction: ... + def update_ai_run_quality_label( self, run_id: str, @@ -414,6 +480,7 @@ class InMemoryAiPlatformRepository: knowledge_documents: dict[str, KnowledgeDocument] = field(default_factory=dict) knowledge_chunks: dict[str, KnowledgeChunk] = field(default_factory=dict) ai_run_audits: list[AiRunAudit] = field(default_factory=list) + quality_improvement_actions: dict[str, AiQualityImprovementAction] = field(default_factory=dict) quality_rule_config: AiQualityRuleConfig = field( default_factory=lambda: AiQualityRuleConfig( id="default", @@ -683,7 +750,75 @@ class InMemoryAiPlatformRepository: for audit in self.ai_run_audits if _ensure_utc(audit.created_at) >= cutoff.replace(hour=0, minute=0, second=0, microsecond=0) ] - return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit) + actions = _materialize_improvement_actions( + self.quality_improvement_actions.values(), + self.ai_run_audits, + ) + return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit, improvement_actions=actions) + + def list_quality_improvement_actions( + self, + *, + status: str | None = None, + limit: int = 20, + ) -> list[AiQualityImprovementAction]: + with self._lock: + actions = _materialize_improvement_actions( + self.quality_improvement_actions.values(), + self.ai_run_audits, + ) + return _filter_improvement_actions(actions, status=status, limit=limit) + + def create_quality_improvement_action( + self, + payload: AiQualityImprovementActionCreate, + ) -> AiQualityImprovementAction: + now = datetime.now(UTC) + with self._lock: + baseline = _calculate_improvement_action_stats(payload, self.ai_run_audits) + action = AiQualityImprovementAction( + id=f"qa-action-{uuid4().hex[:12]}", + title=payload.title or _default_improvement_action_title(payload), + status="open", + baseline_total_runs=baseline["total_runs"], + baseline_affected_runs=baseline["affected_runs"], + baseline_pending_labels=baseline["pending_labels"], + baseline_quality_score=baseline["quality_score"], + current_total_runs=baseline["total_runs"], + current_affected_runs=baseline["affected_runs"], + current_pending_labels=baseline["pending_labels"], + current_quality_score=baseline["quality_score"], + created_at=now, + updated_at=now, + **payload.model_dump(exclude={"title"}), + ) + self.quality_improvement_actions[action.id] = action + return action + + def update_quality_improvement_action( + self, + action_id: str, + payload: AiQualityImprovementActionUpdate, + ) -> AiQualityImprovementAction: + now = datetime.now(UTC) + with self._lock: + action = self.quality_improvement_actions.get(action_id) + if action is None: + raise KeyError(action_id) + update: dict[str, object] = { + "status": payload.status, + "updated_at": now, + "resolved_at": now if payload.status in {"resolved", "false_positive", "closed"} else None, + } + if payload.owner is not None: + update["owner"] = payload.owner + if payload.priority is not None: + update["priority"] = payload.priority + if payload.resolution_note is not None: + update["resolution_note"] = payload.resolution_note + updated = action.model_copy(update=update) + self.quality_improvement_actions[action_id] = updated + return _materialize_improvement_action(updated, self.ai_run_audits) def update_ai_run_quality_label( self, @@ -1187,13 +1322,133 @@ class SqlAlchemyAiPlatformRepository: cutoff = datetime.now(UTC) - timedelta(days=max(days - 1, 0)) cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0) with self._session_factory() as session: + action_rows = session.scalars( + select(AiQualityImprovementActionModel) + .order_by(AiQualityImprovementActionModel.updated_at.desc()) + .limit(20) + ).all() + action_days = [row.evaluation_days for row in action_rows] + stats_cutoff = datetime.now(UTC) - timedelta(days=max([days, *action_days], default=days) - 1) + stats_cutoff = stats_cutoff.replace(hour=0, minute=0, second=0, microsecond=0) rows = session.scalars( + select(AiRunAuditModel) + .where(AiRunAuditModel.created_at >= stats_cutoff) + .order_by(AiRunAuditModel.created_at.desc()) + ).all() + all_runs = [_ai_run_audit_from_model(row) for row in rows] + dashboard_runs = [ + run for run in all_runs if _ensure_utc(run.created_at) >= cutoff + ] + actions = _materialize_improvement_actions( + [_quality_improvement_action_from_model(row) for row in action_rows], + all_runs, + ) + return _build_quality_dashboard( + dashboard_runs, + days=days, + pending_limit=pending_limit, + improvement_actions=_filter_improvement_actions(actions, limit=20), + ) + + def list_quality_improvement_actions( + self, + *, + status: str | None = None, + limit: int = 20, + ) -> list[AiQualityImprovementAction]: + with self._session_factory() as session: + statement = select(AiQualityImprovementActionModel).order_by( + AiQualityImprovementActionModel.updated_at.desc() + ) + if status: + statement = statement.where(AiQualityImprovementActionModel.status == status) + rows = session.scalars(statement.limit(limit)).all() + max_days = max((row.evaluation_days for row in rows), default=14) + cutoff = datetime.now(UTC) - timedelta(days=max(max_days - 1, 0)) + cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0) + audit_rows = session.scalars( select(AiRunAuditModel) .where(AiRunAuditModel.created_at >= cutoff) .order_by(AiRunAuditModel.created_at.desc()) ).all() - runs = [_ai_run_audit_from_model(row) for row in rows] - return _build_quality_dashboard(runs, days=days, pending_limit=pending_limit) + runs = [_ai_run_audit_from_model(row) for row in audit_rows] + actions = _materialize_improvement_actions( + [_quality_improvement_action_from_model(row) for row in rows], + runs, + ) + return _filter_improvement_actions(actions, status=status, limit=limit) + + def create_quality_improvement_action( + self, + payload: AiQualityImprovementActionCreate, + ) -> AiQualityImprovementAction: + now = datetime.now(UTC) + cutoff = now - timedelta(days=max(payload.evaluation_days - 1, 0)) + cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0) + with self._session_factory.begin() as session: + audit_rows = session.scalars( + select(AiRunAuditModel) + .where(AiRunAuditModel.created_at >= cutoff) + .order_by(AiRunAuditModel.created_at.desc()) + ).all() + runs = [_ai_run_audit_from_model(row) for row in audit_rows] + baseline = _calculate_improvement_action_stats(payload, runs) + row = AiQualityImprovementActionModel( + id=f"qa-action-{uuid4().hex[:12]}", + dimension=payload.dimension, + key=payload.key, + label=payload.label, + action_type=payload.action_type, + title=payload.title or _default_improvement_action_title(payload), + description=payload.description, + owner=payload.owner, + priority=payload.priority, + status="open", + source_label_codes_json=_encode_json_list(payload.source_label_codes), + source_run_ids_json=_encode_json_list(payload.source_run_ids), + root_cause_hint=payload.root_cause_hint, + evaluation_days=payload.evaluation_days, + baseline_total_runs=baseline["total_runs"], + baseline_affected_runs=baseline["affected_runs"], + baseline_pending_labels=baseline["pending_labels"], + baseline_quality_score=baseline["quality_score"], + created_at=now, + updated_at=now, + ) + session.add(row) + session.flush() + return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs) + + def update_quality_improvement_action( + self, + action_id: str, + payload: AiQualityImprovementActionUpdate, + ) -> AiQualityImprovementAction: + now = datetime.now(UTC) + with self._session_factory.begin() as session: + row = session.get(AiQualityImprovementActionModel, action_id) + if row is None: + raise KeyError(action_id) + row.status = payload.status + if payload.owner is not None: + row.owner = payload.owner + if payload.priority is not None: + row.priority = payload.priority + if payload.resolution_note is not None: + row.resolution_note = payload.resolution_note + row.updated_at = now + row.resolved_at = now if payload.status in {"resolved", "false_positive", "closed"} else None + + cutoff = now - timedelta(days=max(row.evaluation_days - 1, 0)) + cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0) + audit_rows = session.scalars( + select(AiRunAuditModel) + .where(AiRunAuditModel.created_at >= cutoff) + .order_by(AiRunAuditModel.created_at.desc()) + ).all() + runs = [_ai_run_audit_from_model(row) for row in audit_rows] + session.flush() + return _materialize_improvement_action(_quality_improvement_action_from_model(row), runs) def update_ai_run_quality_label( self, @@ -1453,6 +1708,33 @@ def _ai_run_audit_from_model(row: AiRunAuditModel) -> AiRunAudit: ) +def _quality_improvement_action_from_model(row: AiQualityImprovementActionModel) -> AiQualityImprovementAction: + return AiQualityImprovementAction( + id=row.id, + dimension=row.dimension, + key=row.key, + label=row.label, + action_type=row.action_type, + title=row.title, + description=row.description, + owner=row.owner, + priority=row.priority, + status=row.status, + source_label_codes=[str(item) for item in _decode_json_list(row.source_label_codes_json)], + source_run_ids=[str(item) for item in _decode_json_list(row.source_run_ids_json)], + root_cause_hint=row.root_cause_hint, + evaluation_days=row.evaluation_days, + baseline_total_runs=row.baseline_total_runs, + baseline_affected_runs=row.baseline_affected_runs, + baseline_pending_labels=row.baseline_pending_labels, + baseline_quality_score=row.baseline_quality_score, + resolution_note=row.resolution_note, + created_at=_ensure_utc(row.created_at), + updated_at=_ensure_utc(row.updated_at), + resolved_at=_ensure_utc(row.resolved_at) if row.resolved_at else None, + ) + + def _ensure_utc(value: datetime) -> datetime: if value.tzinfo is None: return value.replace(tzinfo=UTC) @@ -1464,6 +1746,7 @@ def _build_quality_dashboard( *, days: int, pending_limit: int, + improvement_actions: list[AiQualityImprovementAction] | None = None, ) -> AiQualityDashboard: normalized_days = max(days, 1) today = datetime.now(UTC).date() @@ -1541,6 +1824,7 @@ def _build_quality_dashboard( daily_metrics=ordered_daily, pending_labels=pending_labels, attribution=attribution, + improvement_actions=improvement_actions or [], ) @@ -1754,6 +2038,137 @@ def _finalize_attribution_groups(groups: dict[str, dict[str, object]]) -> list[A )[:5] +def _materialize_improvement_actions( + actions: Iterable[AiQualityImprovementAction], + runs: list[AiRunAudit], +) -> list[AiQualityImprovementAction]: + return [_materialize_improvement_action(action, runs) for action in actions] + + +def _materialize_improvement_action( + action: AiQualityImprovementAction, + runs: list[AiRunAudit], +) -> AiQualityImprovementAction: + current = _calculate_improvement_action_stats(action, runs) + baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs) + current_rate = _safe_rate(current["affected_runs"], current["total_runs"]) + return action.model_copy( + update={ + "current_total_runs": current["total_runs"], + "current_affected_runs": current["affected_runs"], + "current_pending_labels": current["pending_labels"], + "current_quality_score": current["quality_score"], + "affected_rate_delta": round(current_rate - baseline_rate, 4), + "pending_labels_delta": current["pending_labels"] - action.baseline_pending_labels, + "quality_score_delta": round(current["quality_score"] - action.baseline_quality_score, 2), + } + ) + + +def _filter_improvement_actions( + actions: list[AiQualityImprovementAction], + *, + status: str | None = None, + limit: int = 20, +) -> list[AiQualityImprovementAction]: + filtered = [action for action in actions if status is None or action.status == status] + status_order = {"open": 0, "in_progress": 1, "resolved": 2, "false_positive": 3, "closed": 4} + priority_order = {"critical": 0, "high": 1, "medium": 2, "low": 3} + return sorted( + filtered, + key=lambda action: ( + status_order.get(action.status, 9), + priority_order.get(action.priority, 9), + -_ensure_utc(action.updated_at).timestamp(), + ), + )[: max(limit, 0)] + + +def _calculate_improvement_action_stats( + action: AiQualityImprovementActionCreate | AiQualityImprovementAction, + runs: list[AiRunAudit], +) -> dict[str, int | float]: + cutoff = datetime.now(UTC) - timedelta(days=max(action.evaluation_days - 1, 0)) + cutoff = cutoff.replace(hour=0, minute=0, second=0, microsecond=0) + matched_runs: list[AiRunAudit] = [] + affected_runs: set[str] = set() + pending_labels = 0 + score_sum = 0 + + for run in runs: + if _ensure_utc(run.created_at) < cutoff: + continue + if not _run_matches_improvement_dimension(run, action): + continue + matched_runs.append(run) + score_sum += run.quality_score + issue_labels = _matching_improvement_issue_labels(run, action) + if issue_labels: + affected_runs.add(run.id) + pending_labels += sum(1 for label in issue_labels if label.review_status == "pending") + + total_runs = len(matched_runs) + return { + "total_runs": total_runs, + "affected_runs": len(affected_runs), + "pending_labels": pending_labels, + "quality_score": round(score_sum / total_runs, 2) if total_runs else 0.0, + } + + +def _run_matches_improvement_dimension( + run: AiRunAudit, + action: AiQualityImprovementActionCreate | AiQualityImprovementAction, +) -> bool: + if action.dimension == "model": + return f"{run.provider or 'unknown'} / {run.model or 'unknown'}" == action.key + if action.dimension == "prompt_template": + return (run.prompt_template_id or "unknown") == action.key + if action.dimension == "knowledge_base": + knowledge_base_ids = { + source.knowledge_base_id for source in run.retrieved_sources if source.knowledge_base_id + } + if action.key in knowledge_base_ids: + return True + return action.key == "no_knowledge_source" and not knowledge_base_ids and _has_source_issue(run) + if action.dimension == "label_code": + return any(label.code == action.key for label in run.quality_labels) + return False + + +def _matching_improvement_issue_labels( + run: AiRunAudit, + action: AiQualityImprovementActionCreate | AiQualityImprovementAction, +) -> list[AiQualityLabel]: + issue_labels = [label for label in run.quality_labels if label.severity in {"warning", "error"}] + if action.dimension == "label_code": + return [label for label in issue_labels if label.code == action.key] + source_label_codes = set(action.source_label_codes) + if source_label_codes: + return [label for label in issue_labels if label.code in source_label_codes] + return issue_labels + + +def _has_source_issue(run: AiRunAudit) -> bool: + return any(label.code in {"source_insufficient", "source_limited"} for label in run.quality_labels) + + +def _safe_rate(numerator: int | float, denominator: int | float) -> float: + return round(float(numerator) / float(denominator), 4) if denominator else 0.0 + + +def _default_improvement_action_title(action: AiQualityImprovementActionCreate) -> str: + action_names = { + "investigate": "Investigate", + "tune_prompt": "Tune prompt", + "improve_knowledge": "Improve knowledge base", + "tune_model": "Tune model config", + "mark_false_positive": "Review false positive", + } + prefix = action_names.get(action.action_type, "Improve") + return f"{prefix}: {action.label}" + + def _label_title(label: AiQualityLabel) -> str: return label.title or label.code diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py index 8d2ef20f..752a271d 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py @@ -4,6 +4,8 @@ from yuqei_ai_platform_api.config import AiPlatformSettings from yuqei_ai_platform_api.embeddings import EmbeddingVector from yuqei_ai_platform_api.main import create_app from yuqei_ai_platform_api.repository import ( + AiQualityImprovementActionCreate, + AiQualityImprovementActionUpdate, AiQualityLabelReviewUpdate, AiQualityRuleConfigCreate, AiRunAudit, @@ -199,6 +201,67 @@ def test_quality_dashboard_summarizes_daily_metrics_and_pending_labels() -> None assert updated_dashboard["summary"]["pending_labels"] == dashboard["summary"]["pending_labels"] - 1 +def test_quality_attribution_can_create_and_resolve_improvement_action() -> None: + client = make_client() + + response = client.post("/api/v1/legal/qa", json={"question": "How should probation be agreed?"}) + assert response.status_code == 200 + + dashboard_response = client.get("/api/v1/audit/quality-dashboard", params={"days": 7, "pending_limit": 10}) + assert dashboard_response.status_code == 200 + dashboard = dashboard_response.json() + model_attribution = dashboard["attribution"]["by_model"][0] + + create_response = client.post( + "/api/v1/audit/quality-improvement-actions", + json={ + "dimension": model_attribution["dimension"], + "key": model_attribution["key"], + "label": model_attribution["label"], + "action_type": "tune_model", + "description": "Check provider timeout and model configuration.", + "owner": "admin", + "priority": "high", + "source_label_codes": model_attribution["top_label_codes"], + "root_cause_hint": model_attribution["root_cause_hint"], + "evaluation_days": 7, + }, + ) + assert create_response.status_code == 200 + created = create_response.json() + assert created["status"] == "open" + assert created["baseline_total_runs"] == 1 + assert created["baseline_affected_runs"] == 1 + assert created["current_affected_runs"] == 1 + + list_response = client.get( + "/api/v1/audit/quality-improvement-actions", + params={"status": "open", "limit": 10}, + ) + assert list_response.status_code == 200 + assert list_response.json()[0]["id"] == created["id"] + + progress_response = client.patch( + f"/api/v1/audit/quality-improvement-actions/{created['id']}", + json={"status": "in_progress", "owner": "ops", "priority": "critical"}, + ) + assert progress_response.status_code == 200 + assert progress_response.json()["status"] == "in_progress" + assert progress_response.json()["owner"] == "ops" + + resolved_response = client.patch( + f"/api/v1/audit/quality-improvement-actions/{created['id']}", + json={"status": "resolved", "resolution_note": "Provider config was tuned."}, + ) + assert resolved_response.status_code == 200 + resolved = resolved_response.json() + assert resolved["status"] == "resolved" + assert resolved["resolved_at"] + + updated_dashboard = client.get("/api/v1/audit/quality-dashboard", params={"days": 7}).json() + assert any(action["id"] == created["id"] for action in updated_dashboard["improvement_actions"]) + + def test_knowledge_document_can_be_added_and_searched() -> None: client = make_client() @@ -545,6 +608,12 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: "title": "Source covered", "detail": "Sufficient retrieved sources were recorded.", }, + { + "code": "source_limited", + "severity": "warning", + "title": "Source limited", + "detail": "Only one source was used.", + }, ], quality_summary="passed: 0 error(s), 0 warning(s), score 100.", status="succeeded", @@ -596,3 +665,22 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: assert updated_audit.quality_labels[0].review_status == "resolved" assert updated_audit.quality_labels[0].review_note == "已确认依据有效。" assert updated_audit.quality_labels[0].reviewed_at is not None + action = reloaded.create_quality_improvement_action( + AiQualityImprovementActionCreate( + dimension="model", + key="deepseek / deepseek-chat", + label="deepseek / deepseek-chat", + action_type="tune_model", + source_label_codes=["source_limited"], + evaluation_days=14, + ) + ) + assert action.baseline_total_runs == 1 + assert action.baseline_affected_runs == 1 + assert reloaded.list_quality_improvement_actions()[0].id == action.id + updated_action = reloaded.update_quality_improvement_action( + action.id, + AiQualityImprovementActionUpdate(status="resolved", resolution_note="done"), + ) + assert updated_action.status == "resolved" + assert updated_action.resolved_at is not None diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py index ba0e47cb..374dc155 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py @@ -5,6 +5,9 @@ from yuqei_sdk.ai_platform import ( AiQualityDashboard, AiQualityDashboardSummary, AiQualityDailyMetric, + AiQualityImprovementAction, + AiQualityImprovementActionCreate, + AiQualityImprovementActionUpdate, AiQualityLabel, AiQualityLabelReviewUpdate, AiQualityPendingLabel, @@ -39,6 +42,9 @@ __all__ = [ "AiQualityDashboard", "AiQualityDashboardSummary", "AiQualityDailyMetric", + "AiQualityImprovementAction", + "AiQualityImprovementActionCreate", + "AiQualityImprovementActionUpdate", "AiQualityLabel", "AiQualityLabelReviewUpdate", "AiQualityPendingLabel", diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py index 187f409d..3fce9765 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py @@ -321,11 +321,55 @@ class AiQualityAttribution(BaseModel): by_label_code: list[AiQualityAttributionItem] = Field(default_factory=list) +class AiQualityImprovementActionCreate(BaseModel): + dimension: str + key: str = Field(min_length=1, max_length=256) + label: str = Field(min_length=1, max_length=256) + action_type: str = "investigate" + title: str | None = Field(default=None, max_length=256) + description: str | None = None + owner: str | None = Field(default=None, max_length=128) + priority: str = "medium" + source_label_codes: list[str] = Field(default_factory=list) + source_run_ids: list[str] = Field(default_factory=list) + root_cause_hint: str | None = None + evaluation_days: int = 14 + + +class AiQualityImprovementActionUpdate(BaseModel): + status: str + owner: str | None = Field(default=None, max_length=128) + priority: str | None = None + resolution_note: str | None = None + + +class AiQualityImprovementAction(AiQualityImprovementActionCreate): + id: str + title: str + status: str = "open" + baseline_total_runs: int = 0 + baseline_affected_runs: int = 0 + baseline_pending_labels: int = 0 + baseline_quality_score: float = 0.0 + current_total_runs: int = 0 + current_affected_runs: int = 0 + current_pending_labels: int = 0 + current_quality_score: float = 0.0 + affected_rate_delta: float = 0.0 + pending_labels_delta: int = 0 + quality_score_delta: float = 0.0 + resolution_note: str | None = None + created_at: str + updated_at: str + resolved_at: str | None = None + + class AiQualityDashboard(BaseModel): summary: AiQualityDashboardSummary daily_metrics: list[AiQualityDailyMetric] = Field(default_factory=list) pending_labels: list[AiQualityPendingLabel] = Field(default_factory=list) attribution: AiQualityAttribution = Field(default_factory=AiQualityAttribution) + improvement_actions: list[AiQualityImprovementAction] = Field(default_factory=list) class AiPlatformClient: @@ -605,6 +649,53 @@ class AiPlatformClient: response.raise_for_status() return AiQualityDashboard.model_validate(response.json()) + def list_quality_improvement_actions( + self, + *, + status: str | None = None, + limit: int = 20, + context: RequestContext | None = None, + ) -> list[AiQualityImprovementAction]: + params: dict[str, Any] = {"limit": limit} + if status: + params["status"] = status + response = self._client.get( + f"{self._api_prefix}/audit/quality-improvement-actions", + params=params, + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return [AiQualityImprovementAction.model_validate(item) for item in response.json()] + + def create_quality_improvement_action( + self, + action: AiQualityImprovementActionCreate, + *, + context: RequestContext | None = None, + ) -> AiQualityImprovementAction: + response = self._client.post( + f"{self._api_prefix}/audit/quality-improvement-actions", + json=action.model_dump(exclude_none=True), + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return AiQualityImprovementAction.model_validate(response.json()) + + def update_quality_improvement_action( + self, + action_id: str, + update: AiQualityImprovementActionUpdate, + *, + context: RequestContext | None = None, + ) -> AiQualityImprovementAction: + response = self._client.patch( + f"{self._api_prefix}/audit/quality-improvement-actions/{action_id}", + json=update.model_dump(exclude_none=True), + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return AiQualityImprovementAction.model_validate(response.json()) + def update_ai_run_quality_label( self, run_id: str, diff --git a/yuqei-ai-sdk-python/tests/test_ai_platform.py b/yuqei-ai-sdk-python/tests/test_ai_platform.py index 5d61b8da..dcb19154 100644 --- a/yuqei-ai-sdk-python/tests/test_ai_platform.py +++ b/yuqei-ai-sdk-python/tests/test_ai_platform.py @@ -5,6 +5,8 @@ import httpx from yuqei_sdk import ( AiPlatformClient, + AiQualityImprovementActionCreate, + AiQualityImprovementActionUpdate, AiQualityLabelReviewUpdate, AiQualityRuleConfigUpdate, KnowledgeDocumentCreate, @@ -91,6 +93,44 @@ def test_ai_platform_client_posts_legal_qa_with_context_headers() -> None: def test_ai_platform_client_covers_admin_data_endpoints() -> None: seen_paths: list[str] = [] + def quality_action_payload( + *, + status: str = "open", + resolution_note: str | None = None, + resolved_at: str | None = None, + ) -> dict[str, object]: + return { + "id": "qa-action-1", + "dimension": "model", + "key": "deepseek / deepseek-chat", + "label": "deepseek / deepseek-chat", + "action_type": "tune_model", + "title": "Tune model config: deepseek / deepseek-chat", + "description": "Check provider timeout and model configuration.", + "owner": "admin", + "priority": "high", + "status": status, + "source_label_codes": ["source_limited"], + "source_run_ids": ["run-1"], + "root_cause_hint": "Model quality issues are concentrated.", + "evaluation_days": 7, + "baseline_total_runs": 1, + "baseline_affected_runs": 1, + "baseline_pending_labels": 1, + "baseline_quality_score": 90.0, + "current_total_runs": 1, + "current_affected_runs": 1, + "current_pending_labels": 1, + "current_quality_score": 90.0, + "affected_rate_delta": 0.0, + "pending_labels_delta": 0, + "quality_score_delta": 0.0, + "resolution_note": resolution_note, + "created_at": "2026-06-22T00:00:00Z", + "updated_at": "2026-06-22T00:02:00Z", + "resolved_at": resolved_at, + } + def handler(request: httpx.Request) -> httpx.Response: seen_paths.append(request.url.path) if request.url.path == "/api/v1/model-configs" and request.method == "POST": @@ -529,8 +569,33 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: } ], }, + "improvement_actions": [quality_action_payload()], }, ) + if request.url.path == "/api/v1/audit/quality-improvement-actions" and request.method == "POST": + payload = json.loads(request.read().decode("utf-8")) + assert payload["dimension"] == "model" + assert payload["action_type"] == "tune_model" + assert payload["source_label_codes"] == ["source_limited"] + return httpx.Response(200, json=quality_action_payload()) + if request.url.path == "/api/v1/audit/quality-improvement-actions": + query = parse_qs(request.url.query.decode()) + assert query["status"] == ["open"] + assert query["limit"] == ["5"] + return httpx.Response(200, json=[quality_action_payload()]) + if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1": + payload = json.loads(request.read().decode("utf-8")) + assert request.method == "PATCH" + assert payload["status"] == "resolved" + assert payload["resolution_note"] == "Provider config was tuned." + return httpx.Response( + 200, + json=quality_action_payload( + status="resolved", + resolution_note="Provider config was tuned.", + resolved_at="2026-06-22T00:05:00Z", + ), + ) if request.url.path == "/api/v1/audit/ai-runs/run-1/quality-labels/citation_present": payload = json.loads(request.read().decode("utf-8")) assert request.method == "PATCH" @@ -667,6 +732,34 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: assert dashboard.attribution.by_model[0].key == "deepseek / deepseek-chat" assert dashboard.attribution.by_label_code[0].top_label_codes == ["source_limited"] assert dashboard.attribution.by_label_code[0].root_cause_hint is not None + assert dashboard.improvement_actions[0].id == "qa-action-1" + created_action = client.create_quality_improvement_action( + AiQualityImprovementActionCreate( + dimension="model", + key="deepseek / deepseek-chat", + label="deepseek / deepseek-chat", + action_type="tune_model", + description="Check provider timeout and model configuration.", + owner="admin", + priority="high", + source_label_codes=["source_limited"], + source_run_ids=["run-1"], + evaluation_days=7, + ) + ) + assert created_action.status == "open" + assert created_action.baseline_affected_runs == 1 + listed_actions = client.list_quality_improvement_actions(status="open", limit=5) + assert listed_actions[0].id == "qa-action-1" + updated_action = client.update_quality_improvement_action( + "qa-action-1", + AiQualityImprovementActionUpdate( + status="resolved", + resolution_note="Provider config was tuned.", + ), + ) + assert updated_action.status == "resolved" + assert updated_action.resolved_at == "2026-06-22T00:05:00Z" updated_audit = client.update_ai_run_quality_label( "run-1", "citation_present", @@ -680,4 +773,5 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: assert updated_audit.quality_labels[0].reviewed_at == "2026-06-22T00:01:00Z" assert "/api/v1/audit/ai-runs" in seen_paths assert "/api/v1/audit/quality-dashboard" in seen_paths + assert "/api/v1/audit/quality-improvement-actions" in seen_paths assert "/api/v1/quality-rules/default" in seen_paths diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx index f6c476eb..34492aad 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx @@ -3,7 +3,18 @@ import Link from "next/link"; import type { ReactNode } from "react"; import { useEffect, useState } from "react"; -import { AlertTriangle, Clock3, Database, RefreshCcw, ShieldCheck, Undo2 } from "lucide-react"; +import { + AlertTriangle, + CheckCircle2, + Clock3, + Database, + PlayCircle, + RefreshCcw, + ShieldCheck, + Undo2, + Wrench, + XCircle +} from "lucide-react"; type QualityDailyMetric = { date: string; @@ -59,11 +70,44 @@ type QualityAttribution = { by_label_code: QualityAttributionItem[]; }; +type QualityImprovementAction = { + id: string; + dimension: string; + key: string; + label: string; + action_type: string; + title: string; + description?: string | null; + owner?: string | null; + priority: string; + status: string; + source_label_codes: string[]; + source_run_ids: string[]; + root_cause_hint?: string | null; + evaluation_days: number; + baseline_total_runs: number; + baseline_affected_runs: number; + baseline_pending_labels: number; + baseline_quality_score: number; + current_total_runs: number; + current_affected_runs: number; + current_pending_labels: number; + current_quality_score: number; + affected_rate_delta: number; + pending_labels_delta: number; + quality_score_delta: number; + resolution_note?: string | null; + created_at: string; + updated_at: string; + resolved_at?: string | null; +}; + type QualityDashboard = { summary: QualityDailyMetric & { days: number }; daily_metrics: QualityDailyMetric[]; pending_labels: QualityPendingLabel[]; attribution?: QualityAttribution; + improvement_actions?: QualityImprovementAction[]; }; type LoadState = @@ -77,9 +121,42 @@ const severityTone: Record = { error: "tag-danger" }; +const statusTone: Record = { + open: "tag-warning", + in_progress: "tag-primary", + resolved: "tag-success", + false_positive: "", + closed: "" +}; + +const statusLabels: Record = { + open: "待处理", + in_progress: "处理中", + resolved: "已完成", + false_positive: "误报关闭", + closed: "已关闭" +}; + +const actionTypeLabels: Record = { + investigate: "排查原因", + tune_prompt: "调整 Prompt", + improve_knowledge: "补知识库", + tune_model: "调模型参数", + mark_false_positive: "处理误报" +}; + +const dimensionLabels: Record = { + model: "模型", + prompt_template: "Prompt", + knowledge_base: "知识库", + label_code: "标签" +}; + export default function AiQualityDashboardPage() { const [state, setState] = useState({ status: "loading" }); const [days, setDays] = useState(14); + const [actionMessage, setActionMessage] = useState(null); + const [actionBusyKey, setActionBusyKey] = useState(null); async function loadDashboard(nextDays = days) { setState({ status: "loading" }); @@ -107,18 +184,83 @@ export default function AiQualityDashboardPage() { }, []); const dashboard = state.status === "ready" ? state.dashboard : null; + const improvementActions = dashboard?.improvement_actions ?? []; + const openActionCount = improvementActions.filter((action) => ["open", "in_progress"].includes(action.status)).length; + function changeDays(value: string) { const nextDays = Number(value); setDays(nextDays); + setActionMessage(null); void loadDashboard(nextDays); } + async function createActionFromAttribution(item: QualityAttributionItem) { + const busyKey = attributionBusyKey(item); + setActionBusyKey(busyKey); + setActionMessage(null); + try { + const response = await fetch("/api/ai-platform/audit/quality-improvement-actions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + dimension: item.dimension, + key: item.key, + label: item.label, + action_type: inferActionType(item), + description: item.root_cause_hint || `来自 ${dimensionLabels[item.dimension] ?? item.dimension} 归因的质量改进项。`, + owner: "admin", + priority: inferPriority(item), + source_label_codes: item.top_label_codes, + root_cause_hint: item.root_cause_hint, + evaluation_days: days + }) + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + throw new Error(payload?.message ?? `HTTP ${response.status}`); + } + setActionMessage("已生成改进项，可以在下方队列继续处理。"); + await loadDashboard(days); + } catch (error) { + setActionMessage(error instanceof Error ? error.message : "生成改进项失败"); + } finally { + setActionBusyKey(null); + } + } + + async function updateAction(action: QualityImprovementAction, status: string) { + const busyKey = actionBusyKeyFor(action, status); + setActionBusyKey(busyKey); + setActionMessage(null); + try { + const response = await fetch(`/api/ai-platform/audit/quality-improvement-actions/${encodeURIComponent(action.id)}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status, + owner: "admin", + resolution_note: defaultResolutionNote(status) + }) + }); + const payload = await response.json().catch(() => null); + if (!response.ok) { + throw new Error(payload?.message ?? `HTTP ${response.status}`); + } + setActionMessage("改进项状态已更新，指标会随新的 AI 调用继续回看。"); + await loadDashboard(days); + } catch (error) { + setActionMessage(error instanceof Error ? error.message : "更新改进项失败"); + } finally { + setActionBusyKey(null); + } + } + return ( <>

AI 质量运营

-

按日期查看质量通过率、fallback、来源不足、耗时异常和待处理标签。

+

按日期跟踪质量通过率、fallback、来源不足、耗时异常，并把高频问题转成可处理的改进项。

- @@ -151,13 +293,61 @@ export default function AiQualityDashboardPage() { } label="待处理标签" value={String(dashboard.summary.pending_labels)} />
+ {actionMessage &&
{actionMessage}
} +
- - - - + + + +
+
+
+

+ + 改进动作闭环 +

+ 待推进 {openActionCount} +
+
+ {improvementActions.length === 0 ? ( +
暂无改进项，可先从上方归因卡片生成。
+ ) : ( +
+ {improvementActions.map((action) => ( + + ))} +
+ )} +
+
+
@@ -243,7 +433,17 @@ function Metric({ icon, label, value }: { icon: ReactNode; label: string; value: ); } -function AttributionPanel({ title, items }: { title: string; items: QualityAttributionItem[] }) { +function AttributionPanel({ + title, + items, + busyKey, + onCreateAction +}: { + title: string; + items: QualityAttributionItem[]; + busyKey: string | null; + onCreateAction: (item: QualityAttributionItem) => void | Promise; +}) { return (
@@ -254,34 +454,161 @@ function AttributionPanel({ title, items }: { title: string; items: QualityAttri {items.length === 0 ? (
暂无异常归因。
) : ( - items.map((item) => ( -
-
- {item.label} - {item.affected_runs}/{item.total_runs} + items.map((item) => { + const busy = busyKey === attributionBusyKey(item); + return ( +
+
+ {item.label} + {item.affected_runs}/{item.total_runs} +
+
+ 待处理 {item.pending_labels} + warning {item.warning_labels} + error {item.error_labels} + 均分 {item.average_quality_score} +
+ {item.top_label_codes.length > 0 && ( + 常见标签：{item.top_label_codes.join("、")} + )} + {item.root_cause_hint && {item.root_cause_hint}} +
+ +
-
- 待处理 {item.pending_labels} - warning {item.warning_labels} - error {item.error_labels} - 均分 {item.average_quality_score} -
- {item.top_label_codes.length > 0 && ( - 常见标签：{item.top_label_codes.join("、")} - )} - {item.root_cause_hint && {item.root_cause_hint}} -
- )) + ); + }) )}
); } +function QualityActionItem({ + action, + busyKey, + onUpdate +}: { + action: QualityImprovementAction; + busyKey: string | null; + onUpdate: (action: QualityImprovementAction, status: string) => void | Promise; +}) { + const busy = busyKey?.startsWith(`${action.id}:`) ?? false; + const baselineRate = safeRate(action.baseline_affected_runs, action.baseline_total_runs); + const currentRate = safeRate(action.current_affected_runs, action.current_total_runs); + + return ( +
+
+
+ {action.title} +
+ {dimensionLabels[action.dimension] ?? action.dimension} + {actionTypeLabels[action.action_type] ?? action.action_type} + 优先级 {action.priority} + {action.owner || "未分配"} +
+
+ {statusLabels[action.status] ?? action.status} +
+ {(action.description || action.root_cause_hint) && ( + {action.description || action.root_cause_hint} + )} +
+ 异常率 {formatPercent(baselineRate)} → {formatPercent(currentRate)} + 待处理 {action.baseline_pending_labels} → {action.current_pending_labels} + 质量分 {action.baseline_quality_score} → {action.current_quality_score} + 变化 {formatSignedPercent(action.affected_rate_delta)} +
+ {action.resolution_note && 处理说明：{action.resolution_note}} +
+ {action.status === "open" && ( + + )} + {["open", "in_progress"].includes(action.status) && ( + <> + + + + )} +
+
+ ); +} + +function inferActionType(item: QualityAttributionItem) { + if (item.dimension === "prompt_template") { + return "tune_prompt"; + } + if (item.dimension === "knowledge_base") { + return "improve_knowledge"; + } + if (item.dimension === "model") { + return "tune_model"; + } + const codes = [item.key, ...item.top_label_codes].join(" "); + if (codes.includes("source")) { + return "improve_knowledge"; + } + if (codes.includes("fallback") || codes.includes("token") || codes.includes("latency") || codes.includes("failed")) { + return "tune_model"; + } + return "investigate"; +} + +function inferPriority(item: QualityAttributionItem) { + if (item.error_labels > 0 || item.fallback_runs > 0) { + return "high"; + } + if (safeRate(item.affected_runs, item.total_runs) >= 0.5 || item.pending_labels >= 10) { + return "high"; + } + return "medium"; +} + +function defaultResolutionNote(status: string) { + if (status === "resolved") { + return "已处理，等待后续 AI 调用数据验证异常率是否下降。"; + } + if (status === "false_positive") { + return "已确认为误报，后续会结合质量规则继续校准。"; + } + return undefined; +} + +function attributionBusyKey(item: QualityAttributionItem) { + return `create:${item.dimension}:${item.key}`; +} + +function actionBusyKeyFor(action: QualityImprovementAction, status: string) { + return `${action.id}:${status}`; +} + +function safeRate(numerator: number, denominator: number) { + return denominator ? numerator / denominator : 0; +} + function formatPercent(value: number) { return `${Math.round(value * 100)}%`; } +function formatSignedPercent(value: number) { + const percent = Math.round(value * 100); + return `${percent > 0 ? "+" : ""}${percent}%`; +} + function formatShortDate(value: string) { const date = new Date(`${value}T00:00:00`); if (Number.isNaN(date.getTime())) { diff --git a/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/[actionId]/route.ts b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/[actionId]/route.ts new file mode 100644 index 00000000..48506a2d --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/[actionId]/route.ts @@ -0,0 +1,47 @@ +import { NextResponse } from "next/server"; + +const aiPlatformBaseUrl = + process.env.AI_PLATFORM_API_BASE_URL ?? + process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ?? + "http://localhost:8101"; + +export async function PATCH( + request: Request, + context: { params: Promise<{ actionId: string }> } +) { + const { actionId } = await context.params; + const targetUrl = new URL( + `/api/v1/audit/quality-improvement-actions/${encodeURIComponent(actionId)}`, + aiPlatformBaseUrl + ); + const body: unknown = await request.json().catch(() => null); + + try { + const response = await fetch(targetUrl, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + cache: "no-store" + }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform quality improvement action service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform quality improvement action service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/route.ts b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/route.ts new file mode 100644 index 00000000..fdae9511 --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/route.ts @@ -0,0 +1,75 @@ +import { NextResponse } from "next/server"; + +const aiPlatformBaseUrl = + process.env.AI_PLATFORM_API_BASE_URL ?? + process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ?? + "http://localhost:8101"; + +export async function GET(request: Request) { + const { searchParams } = new URL(request.url); + const status = searchParams.get("status"); + const limit = searchParams.get("limit") ?? "20"; + const targetUrl = new URL("/api/v1/audit/quality-improvement-actions", aiPlatformBaseUrl); + targetUrl.searchParams.set("limit", limit); + if (status) { + targetUrl.searchParams.set("status", status); + } + + try { + const response = await fetch(targetUrl, { cache: "no-store" }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform quality improvement action service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform quality improvement action service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} + +export async function POST(request: Request) { + const targetUrl = new URL("/api/v1/audit/quality-improvement-actions", aiPlatformBaseUrl); + const body: unknown = await request.json().catch(() => null); + + try { + const response = await fetch(targetUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + cache: "no-store" + }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform quality improvement action service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform quality improvement action service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/globals.css b/yuqei-contract-platform/apps/contract-web/src/app/globals.css index ba5b39b5..87740a55 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/globals.css +++ b/yuqei-contract-platform/apps/contract-web/src/app/globals.css @@ -722,6 +722,52 @@ select { background: #fbfcfe; } +.action-row { + display: flex; + flex-wrap: wrap; + gap: 8px; + align-items: center; +} + +.button:disabled { + cursor: not-allowed; + opacity: 0.58; +} + +.improvement-panel { + margin-bottom: 16px; +} + +.quality-action-list { + display: grid; + gap: 12px; +} + +.quality-action-item { + display: grid; + gap: 10px; + min-width: 0; + padding: 14px; + border: 1px solid var(--line); + border-radius: 8px; + background: #fbfcfe; +} + +.quality-action-stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 8px; + color: var(--muted); + font-size: 13px; +} + +.quality-action-stats span { + min-width: 0; + padding: 8px 10px; + border-radius: 8px; + background: #f1f5f9; +} + .trend-list { display: grid; gap: 10px;