From f145a38ee509f44960e93b80cd10f546aeb3be3d Mon Sep 17 00:00:00 2001 From: yuqei Date: Tue, 23 Jun 2026 08:38:51 +0800 Subject: [PATCH] feat: enhance ai quality improvement operations --- ...-improvement-ops-enhancement-2026-06-23.md | 41 +++++++ .../packages/ai-openapi/openapi.yaml | 31 +++++ .../src/yuqei_ai_platform_api/main.py | 16 ++- .../src/yuqei_ai_platform_api/repository.py | 108 ++++++++++++++++-- .../tests/test_platform_data.py | 16 ++- .../src/yuqei_sdk/ai_platform.py | 14 +++ yuqei-ai-sdk-python/tests/test_ai_platform.py | 17 ++- .../app/admin/ai-quality-dashboard/page.tsx | 91 ++++++++++++++- .../quality-improvement-actions/route.ts | 8 +- .../apps/contract-web/src/app/globals.css | 11 ++ 10 files changed, 337 insertions(+), 16 deletions(-) create mode 100644 project-docs/V2-ai-quality-improvement-ops-enhancement-2026-06-23.md diff --git a/project-docs/V2-ai-quality-improvement-ops-enhancement-2026-06-23.md b/project-docs/V2-ai-quality-improvement-ops-enhancement-2026-06-23.md new file mode 100644 index 00000000..b565004b --- /dev/null +++ b/project-docs/V2-ai-quality-improvement-ops-enhancement-2026-06-23.md @@ -0,0 +1,41 @@ +# V2 AI 质量改进项运营增强记录 - 2026-06-23 + +## 本轮目标 + +延续 AI 质量改进动作闭环，补齐管理员日常运营需要的筛选和效果复盘能力，避免改进项只停留在“创建/关闭”，而是能持续看出处理后是否改善。 + +## 已完成能力 + +1. AI Platform 改进项列表接口新增过滤参数： + - `status` + - `owner` + - `priority` + - `action_type` + - `dimension` +2. 改进项响应新增效果复盘字段： + - `effect_status`: `pending_data`、`improved`、`unchanged`、`regressed` + - `effect_summary`: 基线异常率、当前异常率、待处理标签变化、质量分变化 +3. 内存仓库与 SQL 仓库统一支持筛选和效果复盘。 +4. OpenAPI 契约同步新增查询参数和效果字段。 +5. Python SDK 同步支持筛选参数和效果字段。 +6. 合同系统 AI 质量运营页新增改进项筛选栏： + - 状态 + - 负责人 + - 优先级 + - 动作类型 + - 归因维度 +7. 改进项卡片新增效果标签和复盘摘要，管理员可以直接看到处理后异常率是否下降。 + +## 验证命令 + +```powershell +python -m pytest yuqei-ai-platform/services/ai-platform-api/tests yuqei-ai-sdk-python/tests -q +python -c "import yaml; yaml.safe_load(open('yuqei-ai-platform/packages/ai-openapi/openapi.yaml', encoding='utf-8')); print('openapi ok')" +npm run web:typecheck +npm run web:build +git diff --check +``` + +## 下一步建议 + +继续围绕 V2 方案推进 AI 中台主线，下一步可进入 Prompt 版本发布/回滚闭环，或知识库导入批次与改进项关联，让“调整 Prompt”“补知识库”不只是文字状态，而能关联到真实配置变更记录。 diff --git a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml index 300bc6a3..e0db05c9 100644 --- a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml +++ b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml @@ -480,6 +480,31 @@ paths: schema: type: string enum: [open, in_progress, resolved, false_positive, closed] + - name: owner + in: query + required: false + schema: + type: string + minLength: 1 + maxLength: 128 + - name: priority + in: query + required: false + schema: + type: string + enum: [low, medium, high, critical] + - name: action_type + in: query + required: false + schema: + type: string + enum: [investigate, tune_prompt, improve_knowledge, tune_model, mark_false_positive] + - name: dimension + in: query + required: false + schema: + type: string + enum: [model, prompt_template, knowledge_base, label_code] - name: limit in: query required: false @@ -1636,6 +1661,12 @@ components: quality_score_delta: type: number format: float + effect_status: + type: string + enum: [pending_data, improved, unchanged, regressed] + effect_summary: + type: string + nullable: true resolution_note: type: string nullable: true diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py index bbbcfa87..cd4f93d6 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py @@ -256,9 +256,23 @@ def create_app( ) def list_quality_improvement_actions( status: str | None = Query(None, pattern="^(open|in_progress|resolved|false_positive|closed)$"), + owner: str | None = Query(None, min_length=1, max_length=128), + priority: str | None = Query(None, pattern="^(low|medium|high|critical)$"), + action_type: str | None = Query( + None, + pattern="^(investigate|tune_prompt|improve_knowledge|tune_model|mark_false_positive)$", + ), + dimension: str | None = Query(None, pattern="^(model|prompt_template|knowledge_base|label_code)$"), limit: int = Query(20, ge=1, le=100), ) -> list[AiQualityImprovementAction]: - return store.list_quality_improvement_actions(status=status, limit=limit) + return store.list_quality_improvement_actions( + status=status, + owner=owner, + priority=priority, + action_type=action_type, + dimension=dimension, + limit=limit, + ) @app.post( f"{resolved_settings.api_prefix}/audit/quality-improvement-actions", diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py index 8f308c94..e0694c35 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py @@ -348,6 +348,8 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate): affected_rate_delta: float = 0.0 pending_labels_delta: int = 0 quality_score_delta: float = 0.0 + effect_status: str = "pending_data" + effect_summary: str | None = None resolution_note: str | None = None created_at: datetime updated_at: datetime @@ -448,6 +450,10 @@ class AiPlatformRepository(Protocol): self, *, status: str | None = None, + owner: str | None = None, + priority: str | None = None, + action_type: str | None = None, + dimension: str | None = None, limit: int = 20, ) -> list[AiQualityImprovementAction]: ... @@ -760,6 +766,10 @@ class InMemoryAiPlatformRepository: self, *, status: str | None = None, + owner: str | None = None, + priority: str | None = None, + action_type: str | None = None, + dimension: str | None = None, limit: int = 20, ) -> list[AiQualityImprovementAction]: with self._lock: @@ -767,7 +777,15 @@ class InMemoryAiPlatformRepository: self.quality_improvement_actions.values(), self.ai_run_audits, ) - return _filter_improvement_actions(actions, status=status, limit=limit) + return _filter_improvement_actions( + actions, + status=status, + owner=owner, + priority=priority, + action_type=action_type, + dimension=dimension, + limit=limit, + ) def create_quality_improvement_action( self, @@ -793,7 +811,7 @@ class InMemoryAiPlatformRepository: **payload.model_dump(exclude={"title"}), ) self.quality_improvement_actions[action.id] = action - return action + return _materialize_improvement_action(action, self.ai_run_audits) def update_quality_improvement_action( self, @@ -1354,6 +1372,10 @@ class SqlAlchemyAiPlatformRepository: self, *, status: str | None = None, + owner: str | None = None, + priority: str | None = None, + action_type: str | None = None, + dimension: str | None = None, limit: int = 20, ) -> list[AiQualityImprovementAction]: with self._session_factory() as session: @@ -1362,6 +1384,14 @@ class SqlAlchemyAiPlatformRepository: ) if status: statement = statement.where(AiQualityImprovementActionModel.status == status) + if owner: + statement = statement.where(AiQualityImprovementActionModel.owner == owner) + if priority: + statement = statement.where(AiQualityImprovementActionModel.priority == priority) + if action_type: + statement = statement.where(AiQualityImprovementActionModel.action_type == action_type) + if dimension: + statement = statement.where(AiQualityImprovementActionModel.dimension == dimension) rows = session.scalars(statement.limit(limit)).all() max_days = max((row.evaluation_days for row in rows), default=14) cutoff = datetime.now(UTC) - timedelta(days=max(max_days - 1, 0)) @@ -1376,7 +1406,15 @@ class SqlAlchemyAiPlatformRepository: [_quality_improvement_action_from_model(row) for row in rows], runs, ) - return _filter_improvement_actions(actions, status=status, limit=limit) + return _filter_improvement_actions( + actions, + status=status, + owner=owner, + priority=priority, + action_type=action_type, + dimension=dimension, + limit=limit, + ) def create_quality_improvement_action( self, @@ -2052,15 +2090,28 @@ def _materialize_improvement_action( current = _calculate_improvement_action_stats(action, runs) baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs) current_rate = _safe_rate(current["affected_runs"], current["total_runs"]) + affected_rate_delta = round(current_rate - baseline_rate, 4) + pending_labels_delta = current["pending_labels"] - action.baseline_pending_labels + quality_score_delta = round(current["quality_score"] - action.baseline_quality_score, 2) + effect_status, effect_summary = _summarize_improvement_effect( + action, + current_total_runs=current["total_runs"], + current_rate=current_rate, + affected_rate_delta=affected_rate_delta, + pending_labels_delta=pending_labels_delta, + quality_score_delta=quality_score_delta, + ) return action.model_copy( update={ "current_total_runs": current["total_runs"], "current_affected_runs": current["affected_runs"], "current_pending_labels": current["pending_labels"], "current_quality_score": current["quality_score"], - "affected_rate_delta": round(current_rate - baseline_rate, 4), - "pending_labels_delta": current["pending_labels"] - action.baseline_pending_labels, - "quality_score_delta": round(current["quality_score"] - action.baseline_quality_score, 2), + "affected_rate_delta": affected_rate_delta, + "pending_labels_delta": pending_labels_delta, + "quality_score_delta": quality_score_delta, + "effect_status": effect_status, + "effect_summary": effect_summary, } ) @@ -2069,9 +2120,21 @@ def _filter_improvement_actions( actions: list[AiQualityImprovementAction], *, status: str | None = None, + owner: str | None = None, + priority: str | None = None, + action_type: str | None = None, + dimension: str | None = None, limit: int = 20, ) -> list[AiQualityImprovementAction]: - filtered = [action for action in actions if status is None or action.status == status] + filtered = [ + action + for action in actions + if (status is None or action.status == status) + and (owner is None or action.owner == owner) + and (priority is None or action.priority == priority) + and (action_type is None or action.action_type == action_type) + and (dimension is None or action.dimension == dimension) + ] status_order = {"open": 0, "in_progress": 1, "resolved": 2, "false_positive": 3, "closed": 4} priority_order = {"critical": 0, "high": 1, "medium": 2, "low": 3} return sorted( @@ -2157,6 +2220,37 @@ def _safe_rate(numerator: int | float, denominator: int | float) -> float: return round(float(numerator) / float(denominator), 4) if denominator else 0.0 +def _summarize_improvement_effect( + action: AiQualityImprovementAction, + *, + current_total_runs: int | float, + current_rate: float, + affected_rate_delta: float, + pending_labels_delta: int | float, + quality_score_delta: float, +) -> tuple[str, str]: + baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs) + if action.baseline_total_runs <= 0 or current_total_runs <= 0: + return "pending_data", "Need more AI run data to evaluate this improvement action." + improved = ( + affected_rate_delta < -0.01 + or pending_labels_delta < 0 + or quality_score_delta > 1 + ) + regressed = ( + affected_rate_delta > 0.01 + or pending_labels_delta > 0 + or quality_score_delta < -1 + ) + status = "improved" if improved and not regressed else "regressed" if regressed and not improved else "unchanged" + summary = ( + f"Baseline affected rate {baseline_rate:.0%}, current {current_rate:.0%}; " + f"pending labels delta {pending_labels_delta:+.0f}; " + f"quality score delta {quality_score_delta:+.1f}." + ) + return status, summary + + def _default_improvement_action_title(action: AiQualityImprovementActionCreate) -> str: action_names = { "investigate": "Investigate", diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py index 752a271d..114dd7c4 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py @@ -233,10 +233,19 @@ def test_quality_attribution_can_create_and_resolve_improvement_action() -> None assert created["baseline_total_runs"] == 1 assert created["baseline_affected_runs"] == 1 assert created["current_affected_runs"] == 1 + assert created["effect_status"] == "unchanged" + assert "Baseline affected rate" in created["effect_summary"] list_response = client.get( "/api/v1/audit/quality-improvement-actions", - params={"status": "open", "limit": 10}, + params={ + "status": "open", + "owner": "admin", + "priority": "high", + "action_type": "tune_model", + "dimension": "model", + "limit": 10, + }, ) assert list_response.status_code == 200 assert list_response.json()[0]["id"] == created["id"] @@ -257,9 +266,12 @@ def test_quality_attribution_can_create_and_resolve_improvement_action() -> None resolved = resolved_response.json() assert resolved["status"] == "resolved" assert resolved["resolved_at"] + assert resolved["effect_summary"] updated_dashboard = client.get("/api/v1/audit/quality-dashboard", params={"days": 7}).json() - assert any(action["id"] == created["id"] for action in updated_dashboard["improvement_actions"]) + dashboard_action = next(action for action in updated_dashboard["improvement_actions"] if action["id"] == created["id"]) + assert dashboard_action["effect_status"] in {"unchanged", "improved", "regressed", "pending_data"} + assert dashboard_action["effect_summary"] def test_knowledge_document_can_be_added_and_searched() -> None: diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py index 3fce9765..a187b618 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py @@ -358,6 +358,8 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate): affected_rate_delta: float = 0.0 pending_labels_delta: int = 0 quality_score_delta: float = 0.0 + effect_status: str = "pending_data" + effect_summary: str | None = None resolution_note: str | None = None created_at: str updated_at: str @@ -653,12 +655,24 @@ class AiPlatformClient: self, *, status: str | None = None, + owner: str | None = None, + priority: str | None = None, + action_type: str | None = None, + dimension: str | None = None, limit: int = 20, context: RequestContext | None = None, ) -> list[AiQualityImprovementAction]: params: dict[str, Any] = {"limit": limit} if status: params["status"] = status + if owner: + params["owner"] = owner + if priority: + params["priority"] = priority + if action_type: + params["action_type"] = action_type + if dimension: + params["dimension"] = dimension response = self._client.get( f"{self._api_prefix}/audit/quality-improvement-actions", params=params, diff --git a/yuqei-ai-sdk-python/tests/test_ai_platform.py b/yuqei-ai-sdk-python/tests/test_ai_platform.py index dcb19154..11297e06 100644 --- a/yuqei-ai-sdk-python/tests/test_ai_platform.py +++ b/yuqei-ai-sdk-python/tests/test_ai_platform.py @@ -125,6 +125,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: "affected_rate_delta": 0.0, "pending_labels_delta": 0, "quality_score_delta": 0.0, + "effect_status": "unchanged", + "effect_summary": "Baseline affected rate 100%, current 100%; pending labels delta +0; quality score delta +0.0.", "resolution_note": resolution_note, "created_at": "2026-06-22T00:00:00Z", "updated_at": "2026-06-22T00:02:00Z", @@ -581,6 +583,10 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: if request.url.path == "/api/v1/audit/quality-improvement-actions": query = parse_qs(request.url.query.decode()) assert query["status"] == ["open"] + assert query["owner"] == ["admin"] + assert query["priority"] == ["high"] + assert query["action_type"] == ["tune_model"] + assert query["dimension"] == ["model"] assert query["limit"] == ["5"] return httpx.Response(200, json=[quality_action_payload()]) if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1": @@ -749,8 +755,17 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: ) assert created_action.status == "open" assert created_action.baseline_affected_runs == 1 - listed_actions = client.list_quality_improvement_actions(status="open", limit=5) + assert created_action.effect_status == "unchanged" + listed_actions = client.list_quality_improvement_actions( + status="open", + owner="admin", + priority="high", + action_type="tune_model", + dimension="model", + limit=5, + ) assert listed_actions[0].id == "qa-action-1" + assert listed_actions[0].effect_summary is not None updated_action = client.update_quality_improvement_action( "qa-action-1", AiQualityImprovementActionUpdate( diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx index 34492aad..4f5f5810 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx @@ -96,6 +96,8 @@ type QualityImprovementAction = { affected_rate_delta: number; pending_labels_delta: number; quality_score_delta: number; + effect_status?: string; + effect_summary?: string | null; resolution_note?: string | null; created_at: string; updated_at: string; @@ -115,6 +117,14 @@ type LoadState = | { status: "ready"; dashboard: QualityDashboard } | { status: "error"; message: string }; +type ActionFilters = { + status: string; + owner: string; + priority: string; + actionType: string; + dimension: string; +}; + const severityTone: Record = { info: "tag-primary", warning: "tag-warning", @@ -152,11 +162,32 @@ const dimensionLabels: Record = { label_code: "标签" }; +const effectTone: Record = { + improved: "tag-success", + unchanged: "tag-warning", + regressed: "tag-danger", + pending_data: "" +}; + +const effectLabels: Record = { + improved: "已改善", + unchanged: "未明显变化", + regressed: "有恶化", + pending_data: "待观察" +}; + export default function AiQualityDashboardPage() { const [state, setState] = useState({ status: "loading" }); const [days, setDays] = useState(14); const [actionMessage, setActionMessage] = useState(null); const [actionBusyKey, setActionBusyKey] = useState(null); + const [filters, setFilters] = useState({ + status: "", + owner: "", + priority: "", + actionType: "", + dimension: "" + }); async function loadDashboard(nextDays = days) { setState({ status: "loading" }); @@ -185,7 +216,9 @@ export default function AiQualityDashboardPage() { const dashboard = state.status === "ready" ? state.dashboard : null; const improvementActions = dashboard?.improvement_actions ?? []; + const filteredActions = filterImprovementActions(improvementActions, filters); const openActionCount = improvementActions.filter((action) => ["open", "in_progress"].includes(action.status)).length; + const ownerOptions = Array.from(new Set(improvementActions.map((action) => action.owner).filter(Boolean))).sort() as string[]; function changeDays(value: string) { const nextDays = Number(value); @@ -194,6 +227,10 @@ export default function AiQualityDashboardPage() { void loadDashboard(nextDays); } + function updateFilter(key: keyof ActionFilters, value: string) { + setFilters((current) => ({ ...current, [key]: value })); + } + async function createActionFromAttribution(item: QualityAttributionItem) { const busyKey = attributionBusyKey(item); setActionBusyKey(busyKey); @@ -331,11 +368,46 @@ export default function AiQualityDashboardPage() { 待推进 {openActionCount}
+
+ + + + + +
{improvementActions.length === 0 ? (
暂无改进项，可先从上方归因卡片生成。
+ ) : filteredActions.length === 0 ? ( +
当前筛选条件下没有改进项。
) : (
- {improvementActions.map((action) => ( + {filteredActions.map((action) => ( @@ -512,7 +585,10 @@ function QualityActionItem({ {action.owner || "未分配"}
- {statusLabels[action.status] ?? action.status} +
+ {effectLabels[effectStatus] ?? effectStatus} + {statusLabels[action.status] ?? action.status} +
{(action.description || action.root_cause_hint) && ( {action.description || action.root_cause_hint} @@ -524,6 +600,7 @@ function QualityActionItem({ 变化 {formatSignedPercent(action.affected_rate_delta)} {action.resolution_note && 处理说明：{action.resolution_note}} + {action.effect_summary && 效果复盘：{action.effect_summary}}
{action.status === "open" && (