diff --git a/project-docs/V2-ai-quality-improvement-action-execution-2026-06-23.md b/project-docs/V2-ai-quality-improvement-action-execution-2026-06-23.md new file mode 100644 index 00000000..9441f1d6 --- /dev/null +++ b/project-docs/V2-ai-quality-improvement-action-execution-2026-06-23.md @@ -0,0 +1,79 @@ +# V2 AI 质量改进动作执行闭环记录 - 2026-06-23 + +## 路线核查 + +本轮继续对照《AI 合同平台 V2 重构版开发方案》推进，没有跑偏： + +- AI 质量归因、Prompt、知识库、审计仍归属 `yuqei-ai-platform`。 +- 合同侧前端只通过 `/api/ai-platform/*` 代理调用 AI 中台，不直接拼模型请求、不直接操作 AI 内部仓库。 +- 旧 `shared-*` 仓库没有新增能力，也没有恢复 vendored SDK。 +- 本轮仍围绕 “AI 中台质量评测 -> 归因 -> 改进动作 -> 真实变更 -> 效果回看” 主线推进。 + +## 本轮完成内容 + +### 1. 改进项完成前校验 + +AI Platform 后端新增改进项完成校验： + +- `tune_prompt` 类型必须关联 `linked_prompt_template_id` 和 `linked_prompt_template_version` 后才能标记 `resolved`。 +- `improve_knowledge` 类型必须关联 `linked_knowledge_base_id` 和 `linked_import_batch_id` 后才能标记 `resolved`。 +- 缺少真实变更对象时，API 返回 `422` 和明确原因，避免“纸面完成”。 + +### 2. Prompt 改进执行入口 + +质量运营看板的 Prompt 类改进项新增： + +- “生成 Prompt 草稿”入口。 +- 自动基于当前 Prompt 或默认模板生成新草稿。 +- 自动把新草稿的 Prompt ID/版本回填到改进项。 +- 管理员可继续进入 Prompt 中心编辑、发布和观察后续指标。 + +### 3. 知识库改进执行入口 + +质量运营看板的知识库类改进项新增： + +- 轻量补知识库表单。 +- 创建知识库导入批次。 +- 写入知识文档。 +- 自动把知识库 ID 和导入批次 ID 回填到改进项。 + +### 4. 前端代理补齐 + +合同侧前端新增/增强 AI Platform 代理： + +- `POST /api/ai-platform/knowledge-import-batches` +- `POST /api/ai-platform/knowledge-documents` + +这些代理只承担管理后台调用转发，不改变 AI 能力归属边界。 + +## 闭环价值 + +本轮之后，质量运营链路从： + +```text +发现问题 -> 生成改进项 -> 手工标记完成 +``` + +升级为： + +```text +发现问题 -> 生成改进项 -> 生成 Prompt 草稿 / 补知识库 -> 挂真实变更 -> 再标记完成 -> 后续指标回看 +``` + +这样 `调整 Prompt`、`补知识库` 不再只是文字状态，而是能追溯到真实配置变更记录。 + +## 验证重点 + +- 没有真实 Prompt/知识库关联时，改进项不能标记完成。 +- Prompt 改进项能生成草稿并回填关联。 +- 知识库改进项能创建导入批次、写入文档并回填关联。 +- 质量看板仍能加载、筛选、更新改进项状态。 + +## 下一步建议 + +下一步可以进入 “改进效果复盘增强”： + +- 将改进项完成时间作为效果观察窗口起点。 +- 在质量看板中展示完成前后异常率对比。 +- 针对同一个 Prompt/知识库变更聚合关联的多个改进项。 +- 给 Prompt 发布、知识库导入增加更明确的变更审计事件。 diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py index ee3137ba..b026f5fc 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py @@ -321,6 +321,8 @@ def create_app( return store.update_quality_improvement_action(action_id, payload) except KeyError as error: raise HTTPException(status_code=404, detail="AI quality improvement action not found.") from error + except ValueError as error: + raise HTTPException(status_code=422, detail=str(error)) from error @app.patch( f"{resolved_settings.api_prefix}/audit/ai-runs/{{run_id}}/quality-labels/{{label_code}}", diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py index 884ba70a..f2f76219 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py @@ -908,6 +908,16 @@ class InMemoryAiPlatformRepository: action = self.quality_improvement_actions.get(action_id) if action is None: raise KeyError(action_id) + _validate_quality_action_completion( + action_type=action.action_type, + status=payload.status, + linked_prompt_template_id=payload.linked_prompt_template_id or action.linked_prompt_template_id, + linked_prompt_template_version=( + payload.linked_prompt_template_version or action.linked_prompt_template_version + ), + linked_knowledge_base_id=payload.linked_knowledge_base_id or action.linked_knowledge_base_id, + linked_import_batch_id=payload.linked_import_batch_id or action.linked_import_batch_id, + ) update: dict[str, object] = { "status": payload.status, "updated_at": now, @@ -1684,6 +1694,16 @@ class SqlAlchemyAiPlatformRepository: row = session.get(AiQualityImprovementActionModel, action_id) if row is None: raise KeyError(action_id) + _validate_quality_action_completion( + action_type=row.action_type, + status=payload.status, + linked_prompt_template_id=payload.linked_prompt_template_id or row.linked_prompt_template_id, + linked_prompt_template_version=( + payload.linked_prompt_template_version or row.linked_prompt_template_version + ), + linked_knowledge_base_id=payload.linked_knowledge_base_id or row.linked_knowledge_base_id, + linked_import_batch_id=payload.linked_import_batch_id or row.linked_import_batch_id, + ) row.status = payload.status if payload.owner is not None: row.owner = payload.owner @@ -2032,6 +2052,29 @@ def _with_inferred_action_links( return payload.model_copy(update=update) if update else payload +def _validate_quality_action_completion( + *, + action_type: str, + status: str, + linked_prompt_template_id: str | None, + linked_prompt_template_version: str | None, + linked_knowledge_base_id: str | None, + linked_import_batch_id: str | None, +) -> None: + if status != "resolved": + return + if action_type == "tune_prompt" and not (linked_prompt_template_id and linked_prompt_template_version): + raise ValueError( + "Prompt improvement actions require linked_prompt_template_id and " + "linked_prompt_template_version before they can be resolved." + ) + if action_type == "improve_knowledge" and not (linked_knowledge_base_id and linked_import_batch_id): + raise ValueError( + "Knowledge improvement actions require linked_knowledge_base_id and " + "linked_import_batch_id before they can be resolved." + ) + + def _find_prompt_template_link( key: str, prompt_templates: Iterable[PromptTemplate], diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py index 50fd5457..eafe9daa 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py @@ -1,3 +1,4 @@ +import pytest from fastapi.testclient import TestClient from yuqei_ai_platform_api.config import AiPlatformSettings @@ -373,6 +374,82 @@ def test_quality_improvement_action_links_prompt_and_knowledge_changes() -> None assert updated.linked_import_batch_id == "kb-batch-manual" +def test_quality_improvement_action_requires_real_change_before_resolved() -> None: + repository = InMemoryAiPlatformRepository() + + prompt_action = repository.create_quality_improvement_action( + AiQualityImprovementActionCreate( + dimension="label_code", + key="prompt_missing_citation", + label="Prompt missing citation", + action_type="tune_prompt", + evaluation_days=7, + ) + ) + with pytest.raises(ValueError, match="Prompt improvement actions require"): + repository.update_quality_improvement_action( + prompt_action.id, + AiQualityImprovementActionUpdate(status="resolved"), + ) + + knowledge_action = repository.create_quality_improvement_action( + AiQualityImprovementActionCreate( + dimension="label_code", + key="source_limited", + label="Source limited", + action_type="improve_knowledge", + evaluation_days=7, + ) + ) + with pytest.raises(ValueError, match="Knowledge improvement actions require"): + repository.update_quality_improvement_action( + knowledge_action.id, + AiQualityImprovementActionUpdate(status="resolved"), + ) + + batch = repository.create_knowledge_import_batch( + KnowledgeImportBatchCreate( + knowledge_base_id="default", + source_name="quality-action-source-fix", + source_type="quality_improvement", + ) + ) + resolved = repository.update_quality_improvement_action( + knowledge_action.id, + AiQualityImprovementActionUpdate( + status="resolved", + linked_knowledge_base_id="default", + linked_import_batch_id=batch.id, + resolution_note="Knowledge base was supplemented.", + ), + ) + assert resolved.status == "resolved" + assert resolved.linked_import_batch_id == batch.id + + +def test_quality_improvement_action_resolve_api_returns_422_without_real_change() -> None: + client = make_client() + create_response = client.post( + "/api/v1/audit/quality-improvement-actions", + json={ + "dimension": "label_code", + "key": "source_limited", + "label": "Source limited", + "action_type": "improve_knowledge", + "evaluation_days": 7, + }, + ) + assert create_response.status_code == 200 + action = create_response.json() + + resolve_response = client.patch( + f"/api/v1/audit/quality-improvement-actions/{action['id']}", + json={"status": "resolved"}, + ) + assert resolve_response.status_code == 422 + assert "linked_knowledge_base_id" in resolve_response.json()["detail"] + + def test_knowledge_document_can_be_added_and_searched() -> None: client = make_client() diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx index 7c14c564..2d417718 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx @@ -5,9 +5,12 @@ import type { ReactNode } from "react"; import { useEffect, useState } from "react"; import { AlertTriangle, + BookOpen, CheckCircle2, Clock3, Database, + FilePlus2, + PencilLine, PlayCircle, RefreshCcw, ShieldCheck, @@ -120,6 +123,7 @@ type PromptTemplateOption = { id: string; name: string; version: string; + content?: string; status: string; published_at?: string | null; updated_at: string; @@ -147,6 +151,13 @@ type ActionFilters = { dimension: string; }; +type KnowledgeDraft = { + sourceName: string; + title: string; + reference: string; + content: string; +}; + const severityTone: Record = { info: "tag-primary", warning: "tag-warning", @@ -205,6 +216,7 @@ export default function AiQualityDashboardPage() { const [actionBusyKey, setActionBusyKey] = useState(null); const [promptOptions, setPromptOptions] = useState([]); const [importBatchOptions, setImportBatchOptions] = useState([]); + const [knowledgeDrafts, setKnowledgeDrafts] = useState>({}); const [filters, setFilters] = useState({ status: "", owner: "", @@ -295,7 +307,7 @@ export default function AiQualityDashboardPage() { }); const payload = await response.json().catch(() => null); if (!response.ok) { - throw new Error(payload?.message ?? `HTTP ${response.status}`); + throw new Error(readApiError(payload, response.status)); } setActionMessage("已生成改进项，可以在下方队列继续处理。"); await loadLinkOptions(); @@ -334,6 +346,139 @@ export default function AiQualityDashboardPage() { } } + async function createPromptDraftForAction(action: QualityImprovementAction) { + const busyKey = actionBusyKeyFor(action, "prompt-draft"); + setActionBusyKey(busyKey); + setActionMessage(null); + try { + const sourcePrompt = findPromptOption(action.linked_prompt_template_id || action.key, promptOptions); + const promptName = sourcePrompt?.name || (action.key.includes(":") ? action.key.split(":")[0] : "legal_qa.default"); + const version = `quality-${new Date().toISOString().slice(0, 10).replaceAll("-", "")}-${action.id.slice(-4)}`; + const response = await fetch("/api/ai-platform/prompt-templates", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name: promptName, + version, + content: buildPromptDraftContent(action, sourcePrompt?.content), + enabled: false + }) + }); + const prompt = await response.json().catch(() => null); + if (!response.ok) { + throw new Error(readApiError(prompt, response.status)); + } + + const updateResponse = await fetch(`/api/ai-platform/audit/quality-improvement-actions/${encodeURIComponent(action.id)}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: "in_progress", + owner: "admin", + linked_prompt_template_id: prompt.id, + linked_prompt_template_version: prompt.version, + resolution_note: "已生成 Prompt 草稿，待编辑发布并继续观察质量指标。" + }) + }); + const updatePayload = await updateResponse.json().catch(() => null); + if (!updateResponse.ok) { + throw new Error(readApiError(updatePayload, updateResponse.status)); + } + + setActionMessage(`已生成 Prompt 草稿 ${prompt.name} / ${prompt.version}，可进入 Prompt 中心编辑发布。`); + await loadLinkOptions(); + await loadDashboard(days); + } catch (error) { + setActionMessage(error instanceof Error ? error.message : "生成 Prompt 草稿失败"); + } finally { + setActionBusyKey(null); + } + } + + function updateKnowledgeDraft(action: QualityImprovementAction, patch: Partial) { + setKnowledgeDrafts((current) => ({ + ...current, + [action.id]: { + ...defaultKnowledgeDraft(action), + ...current[action.id], + ...patch + } + })); + } + + async function submitKnowledgeDraftForAction(action: QualityImprovementAction) { + const busyKey = actionBusyKeyFor(action, "knowledge-import"); + const draft = knowledgeDrafts[action.id] ?? defaultKnowledgeDraft(action); + const knowledgeBaseId = action.linked_knowledge_base_id || (action.dimension === "knowledge_base" ? action.key : "default"); + if (!draft.title.trim() || !draft.content.trim()) { + setActionMessage("请先填写知识标题和补充内容，再创建导入批次。"); + return; + } + setActionBusyKey(busyKey); + setActionMessage(null); + try { + const batchResponse = await fetch("/api/ai-platform/knowledge-import-batches", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + knowledge_base_id: knowledgeBaseId, + source_name: draft.sourceName.trim() || `quality-action-${action.id}`, + source_type: "quality_improvement" + }) + }); + const batch = await batchResponse.json().catch(() => null); + if (!batchResponse.ok) { + throw new Error(readApiError(batch, batchResponse.status)); + } + + const documentResponse = await fetch("/api/ai-platform/knowledge-documents", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + import_batch_id: batch.id, + knowledge_base_id: knowledgeBaseId, + title: draft.title.trim(), + source_type: "quality_improvement", + reference: draft.reference.trim() || action.label, + content: draft.content.trim() + }) + }); + const documentPayload = await documentResponse.json().catch(() => null); + if (!documentResponse.ok) { + throw new Error(readApiError(documentPayload, documentResponse.status)); + } + + const updateResponse = await fetch(`/api/ai-platform/audit/quality-improvement-actions/${encodeURIComponent(action.id)}`, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + status: "in_progress", + owner: "admin", + linked_knowledge_base_id: knowledgeBaseId, + linked_import_batch_id: batch.id, + resolution_note: "已补充知识库导入批次，待后续 AI 调用验证来源不足率。" + }) + }); + const updatePayload = await updateResponse.json().catch(() => null); + if (!updateResponse.ok) { + throw new Error(readApiError(updatePayload, updateResponse.status)); + } + + setKnowledgeDrafts((current) => { + const next = { ...current }; + delete next[action.id]; + return next; + }); + setActionMessage(`已写入知识库 ${knowledgeBaseId}，导入批次 ${batch.id} 已关联到改进项。`); + await loadLinkOptions(); + await loadDashboard(days); + } catch (error) { + setActionMessage(error instanceof Error ? error.message : "补充知识库失败"); + } finally { + setActionBusyKey(null); + } + } + return ( <>
@@ -453,8 +598,12 @@ export default function AiQualityDashboardPage() { ))}
@@ -604,16 +753,27 @@ function AttributionPanel({ function QualityActionItem({ action, busyKey, - onUpdate + knowledgeDraft, + onCreatePromptDraft, + onSubmitKnowledgeDraft, + onUpdate, + onUpdateKnowledgeDraft }: { action: QualityImprovementAction; busyKey: string | null; + knowledgeDraft: KnowledgeDraft; + onCreatePromptDraft: (action: QualityImprovementAction) => void | Promise; + onSubmitKnowledgeDraft: (action: QualityImprovementAction) => void | Promise; onUpdate: (action: QualityImprovementAction, status: string) => void | Promise; + onUpdateKnowledgeDraft: (action: QualityImprovementAction, patch: Partial) => void; }) { const busy = busyKey?.startsWith(`${action.id}:`) ?? false; const baselineRate = safeRate(action.baseline_affected_runs, action.baseline_total_runs); const currentRate = safeRate(action.current_affected_runs, action.current_total_runs); const effectStatus = action.effect_status ?? "pending_data"; + const needsPromptDraft = action.action_type === "tune_prompt"; + const needsKnowledgeImport = action.action_type === "improve_knowledge"; + const completionBlockReason = getCompletionBlockReason(action); return (
@@ -644,12 +804,81 @@ function QualityActionItem({ {action.resolution_note && 处理说明：{action.resolution_note}} {action.effect_summary && 效果复盘：{action.effect_summary}} + {["open", "in_progress"].includes(action.status) && (needsPromptDraft || needsKnowledgeImport) && ( +
+ {needsPromptDraft && ( + + )} + {needsKnowledgeImport && ( +
+
+ + + + +
+