yuqei/ai-contract-platform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: enhance ai quality improvement operations

Browse Source
This commit is contained in:
yuqei
2026-06-23 08:38:51 +08:00
parent 87bb3c2b75
commit f145a38ee5
10 changed files with 337 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
project-docs/V2-ai-quality-improvement-ops-enhancement-2026-06-23.md
+41
View File
@@ -0,0 +1,41 @@
# V2 AI 质量改进项运营增强记录 - 2026-06-23
## 本轮目标
延续 AI 质量改进动作闭环，补齐管理员日常运营需要的筛选和效果复盘能力，避免改进项只停留在“创建/关闭”，而是能持续看出处理后是否改善。
## 已完成能力
1. AI Platform 改进项列表接口新增过滤参数：
- `status`
- `owner`
- `priority`
- `action_type`
- `dimension`
2. 改进项响应新增效果复盘字段：
- `effect_status`: `pending_data`、`improved`、`unchanged`、`regressed`
- `effect_summary`: 基线异常率、当前异常率、待处理标签变化、质量分变化
3. 内存仓库与 SQL 仓库统一支持筛选和效果复盘。
4. OpenAPI 契约同步新增查询参数和效果字段。
5. Python SDK 同步支持筛选参数和效果字段。
6. 合同系统 AI 质量运营页新增改进项筛选栏：
- 状态
- 负责人
- 优先级
- 动作类型
- 归因维度
7. 改进项卡片新增效果标签和复盘摘要，管理员可以直接看到处理后异常率是否下降。
## 验证命令
```powershell
python -m pytest yuqei-ai-platform/services/ai-platform-api/tests yuqei-ai-sdk-python/tests -q
python -c "import yaml; yaml.safe_load(open('yuqei-ai-platform/packages/ai-openapi/openapi.yaml', encoding='utf-8')); print('openapi ok')"
npm run web:typecheck
npm run web:build
git diff --check
```
## 下一步建议
继续围绕 V2 方案推进 AI 中台主线，下一步可进入 Prompt 版本发布/回滚闭环，或知识库导入批次与改进项关联，让“调整 Prompt”“补知识库”不只是文字状态，而能关联到真实配置变更记录。
yuqei-ai-platform/packages/ai-openapi/openapi.yaml
+31
View File
@@ -480,6 +480,31 @@ paths:
schema:
type: string
enum: [open, in_progress, resolved, false_positive, closed]
- name: owner
in: query
required: false
schema:
type: string
minLength: 1
maxLength: 128
- name: priority
in: query
required: false
schema:
type: string
enum: [low, medium, high, critical]
- name: action_type
in: query
required: false
schema:
type: string
enum: [investigate, tune_prompt, improve_knowledge, tune_model, mark_false_positive]
- name: dimension
in: query
required: false
schema:
type: string
enum: [model, prompt_template, knowledge_base, label_code]
- name: limit
in: query
required: false
@@ -1636,6 +1661,12 @@ components:
quality_score_delta:
type: number
format: float
effect_status:
type: string
enum: [pending_data, improved, unchanged, regressed]
effect_summary:
type: string
nullable: true
resolution_note:
type: string
nullable: true
yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py
+15 -1
View File
@@ -256,9 +256,23 @@ def create_app(
)
def list_quality_improvement_actions(
status: str | None = Query(None, pattern="^(open|in_progress|resolved|false_positive|closed)$"),
owner: str | None = Query(None, min_length=1, max_length=128),
priority: str | None = Query(None, pattern="^(low|medium|high|critical)$"),
action_type: str | None = Query(
None,
pattern="^(investigate|tune_prompt|improve_knowledge|tune_model|mark_false_positive)$",
),
dimension: str | None = Query(None, pattern="^(model|prompt_template|knowledge_base|label_code)$"),
limit: int = Query(20, ge=1, le=100),
) -> list[AiQualityImprovementAction]:
return store.list_quality_improvement_actions(status=status, limit=limit)
return store.list_quality_improvement_actions(
status=status,
owner=owner,
priority=priority,
action_type=action_type,
dimension=dimension,
limit=limit,
)
@app.post(
f"{resolved_settings.api_prefix}/audit/quality-improvement-actions",
yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py
+101 -7
View File
@@ -348,6 +348,8 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate):
affected_rate_delta: float = 0.0
pending_labels_delta: int = 0
quality_score_delta: float = 0.0
effect_status: str = "pending_data"
effect_summary: str | None = None
resolution_note: str | None = None
created_at: datetime
updated_at: datetime
@@ -448,6 +450,10 @@ class AiPlatformRepository(Protocol):
self,
*,
status: str | None = None,
owner: str | None = None,
priority: str | None = None,
action_type: str | None = None,
dimension: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]: ...
@@ -760,6 +766,10 @@ class InMemoryAiPlatformRepository:
self,
*,
status: str | None = None,
owner: str | None = None,
priority: str | None = None,
action_type: str | None = None,
dimension: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]:
with self._lock:
@@ -767,7 +777,15 @@ class InMemoryAiPlatformRepository:
self.quality_improvement_actions.values(),
self.ai_run_audits,
)
return _filter_improvement_actions(actions, status=status, limit=limit)
return _filter_improvement_actions(
actions,
status=status,
owner=owner,
priority=priority,
action_type=action_type,
dimension=dimension,
limit=limit,
)
def create_quality_improvement_action(
self,
@@ -793,7 +811,7 @@ class InMemoryAiPlatformRepository:
**payload.model_dump(exclude={"title"}),
)
self.quality_improvement_actions[action.id] = action
return action
return _materialize_improvement_action(action, self.ai_run_audits)
def update_quality_improvement_action(
self,
@@ -1354,6 +1372,10 @@ class SqlAlchemyAiPlatformRepository:
self,
*,
status: str | None = None,
owner: str | None = None,
priority: str | None = None,
action_type: str | None = None,
dimension: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]:
with self._session_factory() as session:
@@ -1362,6 +1384,14 @@ class SqlAlchemyAiPlatformRepository:
)
if status:
statement = statement.where(AiQualityImprovementActionModel.status == status)
if owner:
statement = statement.where(AiQualityImprovementActionModel.owner == owner)
if priority:
statement = statement.where(AiQualityImprovementActionModel.priority == priority)
if action_type:
statement = statement.where(AiQualityImprovementActionModel.action_type == action_type)
if dimension:
statement = statement.where(AiQualityImprovementActionModel.dimension == dimension)
rows = session.scalars(statement.limit(limit)).all()
max_days = max((row.evaluation_days for row in rows), default=14)
cutoff = datetime.now(UTC) - timedelta(days=max(max_days - 1, 0))
@@ -1376,7 +1406,15 @@ class SqlAlchemyAiPlatformRepository:
[_quality_improvement_action_from_model(row) for row in rows],
runs,
)
return _filter_improvement_actions(actions, status=status, limit=limit)
return _filter_improvement_actions(
actions,
status=status,
owner=owner,
priority=priority,
action_type=action_type,
dimension=dimension,
limit=limit,
)
def create_quality_improvement_action(
self,
@@ -2052,15 +2090,28 @@ def _materialize_improvement_action(
current = _calculate_improvement_action_stats(action, runs)
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
current_rate = _safe_rate(current["affected_runs"], current["total_runs"])
affected_rate_delta = round(current_rate - baseline_rate, 4)
pending_labels_delta = current["pending_labels"] - action.baseline_pending_labels
quality_score_delta = round(current["quality_score"] - action.baseline_quality_score, 2)
effect_status, effect_summary = _summarize_improvement_effect(
action,
current_total_runs=current["total_runs"],
current_rate=current_rate,
affected_rate_delta=affected_rate_delta,
pending_labels_delta=pending_labels_delta,
quality_score_delta=quality_score_delta,
)
return action.model_copy(
update={
"current_total_runs": current["total_runs"],
"current_affected_runs": current["affected_runs"],
"current_pending_labels": current["pending_labels"],
"current_quality_score": current["quality_score"],
"affected_rate_delta": round(current_rate - baseline_rate, 4),
"pending_labels_delta": current["pending_labels"] - action.baseline_pending_labels,
"quality_score_delta": round(current["quality_score"] - action.baseline_quality_score, 2),
"affected_rate_delta": affected_rate_delta,
"pending_labels_delta": pending_labels_delta,
"quality_score_delta": quality_score_delta,
"effect_status": effect_status,
"effect_summary": effect_summary,
}
)
@@ -2069,9 +2120,21 @@ def _filter_improvement_actions(
actions: list[AiQualityImprovementAction],
*,
status: str | None = None,
owner: str | None = None,
priority: str | None = None,
action_type: str | None = None,
dimension: str | None = None,
limit: int = 20,
) -> list[AiQualityImprovementAction]:
filtered = [action for action in actions if status is None or action.status == status]
filtered = [
action
for action in actions
if (status is None or action.status == status)
and (owner is None or action.owner == owner)
and (priority is None or action.priority == priority)
and (action_type is None or action.action_type == action_type)
and (dimension is None or action.dimension == dimension)
]
status_order = {"open": 0, "in_progress": 1, "resolved": 2, "false_positive": 3, "closed": 4}
priority_order = {"critical": 0, "high": 1, "medium": 2, "low": 3}
return sorted(
@@ -2157,6 +2220,37 @@ def _safe_rate(numerator: int | float, denominator: int | float) -> float:
return round(float(numerator) / float(denominator), 4) if denominator else 0.0
def _summarize_improvement_effect(
action: AiQualityImprovementAction,
*,
current_total_runs: int | float,
current_rate: float,
affected_rate_delta: float,
pending_labels_delta: int | float,
quality_score_delta: float,
) -> tuple[str, str]:
baseline_rate = _safe_rate(action.baseline_affected_runs, action.baseline_total_runs)
if action.baseline_total_runs <= 0 or current_total_runs <= 0:
return "pending_data", "Need more AI run data to evaluate this improvement action."
improved = (
affected_rate_delta < -0.01
or pending_labels_delta < 0
or quality_score_delta > 1
)
regressed = (
affected_rate_delta > 0.01
or pending_labels_delta > 0
or quality_score_delta < -1
)
status = "improved" if improved and not regressed else "regressed" if regressed and not improved else "unchanged"
summary = (
f"Baseline affected rate {baseline_rate:.0%}, current {current_rate:.0%}; "
f"pending labels delta {pending_labels_delta:+.0f}; "
f"quality score delta {quality_score_delta:+.1f}."
)
return status, summary
def _default_improvement_action_title(action: AiQualityImprovementActionCreate) -> str:
action_names = {
"investigate": "Investigate",
yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py
+14 -2
View File
@@ -233,10 +233,19 @@ def test_quality_attribution_can_create_and_resolve_improvement_action() -> None
assert created["baseline_total_runs"] == 1
assert created["baseline_affected_runs"] == 1
assert created["current_affected_runs"] == 1
assert created["effect_status"] == "unchanged"
assert "Baseline affected rate" in created["effect_summary"]
list_response = client.get(
"/api/v1/audit/quality-improvement-actions",
params={"status": "open", "limit": 10},
params={
"status": "open",
"owner": "admin",
"priority": "high",
"action_type": "tune_model",
"dimension": "model",
"limit": 10,
},
)
assert list_response.status_code == 200
assert list_response.json()[0]["id"] == created["id"]
@@ -257,9 +266,12 @@ def test_quality_attribution_can_create_and_resolve_improvement_action() -> None
resolved = resolved_response.json()
assert resolved["status"] == "resolved"
assert resolved["resolved_at"]
assert resolved["effect_summary"]
updated_dashboard = client.get("/api/v1/audit/quality-dashboard", params={"days": 7}).json()
assert any(action["id"] == created["id"] for action in updated_dashboard["improvement_actions"])
dashboard_action = next(action for action in updated_dashboard["improvement_actions"] if action["id"] == created["id"])
assert dashboard_action["effect_status"] in {"unchanged", "improved", "regressed", "pending_data"}
assert dashboard_action["effect_summary"]
def test_knowledge_document_can_be_added_and_searched() -> None:
yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py
+14
View File
@@ -358,6 +358,8 @@ class AiQualityImprovementAction(AiQualityImprovementActionCreate):
affected_rate_delta: float = 0.0
pending_labels_delta: int = 0
quality_score_delta: float = 0.0
effect_status: str = "pending_data"
effect_summary: str | None = None
resolution_note: str | None = None
created_at: str
updated_at: str
@@ -653,12 +655,24 @@ class AiPlatformClient:
self,
*,
status: str | None = None,
owner: str | None = None,
priority: str | None = None,
action_type: str | None = None,
dimension: str | None = None,
limit: int = 20,
context: RequestContext | None = None,
) -> list[AiQualityImprovementAction]:
params: dict[str, Any] = {"limit": limit}
if status:
params["status"] = status
if owner:
params["owner"] = owner
if priority:
params["priority"] = priority
if action_type:
params["action_type"] = action_type
if dimension:
params["dimension"] = dimension
response = self._client.get(
f"{self._api_prefix}/audit/quality-improvement-actions",
params=params,
yuqei-ai-sdk-python/tests/test_ai_platform.py
+16 -1
View File
@@ -125,6 +125,8 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
"affected_rate_delta": 0.0,
"pending_labels_delta": 0,
"quality_score_delta": 0.0,
"effect_status": "unchanged",
"effect_summary": "Baseline affected rate 100%, current 100%; pending labels delta +0; quality score delta +0.0.",
"resolution_note": resolution_note,
"created_at": "2026-06-22T00:00:00Z",
"updated_at": "2026-06-22T00:02:00Z",
@@ -581,6 +583,10 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
if request.url.path == "/api/v1/audit/quality-improvement-actions":
query = parse_qs(request.url.query.decode())
assert query["status"] == ["open"]
assert query["owner"] == ["admin"]
assert query["priority"] == ["high"]
assert query["action_type"] == ["tune_model"]
assert query["dimension"] == ["model"]
assert query["limit"] == ["5"]
return httpx.Response(200, json=[quality_action_payload()])
if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1":
@@ -749,8 +755,17 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None:
)
assert created_action.status == "open"
assert created_action.baseline_affected_runs == 1
listed_actions = client.list_quality_improvement_actions(status="open", limit=5)
assert created_action.effect_status == "unchanged"
listed_actions = client.list_quality_improvement_actions(
status="open",
owner="admin",
priority="high",
action_type="tune_model",
dimension="model",
limit=5,
)
assert listed_actions[0].id == "qa-action-1"
assert listed_actions[0].effect_summary is not None
updated_action = client.update_quality_improvement_action(
"qa-action-1",
AiQualityImprovementActionUpdate(
yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality-dashboard/page.tsx
+88 -1
View File
@@ -96,6 +96,8 @@ type QualityImprovementAction = {
affected_rate_delta: number;
pending_labels_delta: number;
quality_score_delta: number;
effect_status?: string;
effect_summary?: string | null;
resolution_note?: string | null;
created_at: string;
updated_at: string;
@@ -115,6 +117,14 @@ type LoadState =
| { status: "ready"; dashboard: QualityDashboard }
| { status: "error"; message: string };
type ActionFilters = {
status: string;
owner: string;
priority: string;
actionType: string;
dimension: string;
};
const severityTone: Record<string, string> = {
info: "tag-primary",
warning: "tag-warning",
@@ -152,11 +162,32 @@ const dimensionLabels: Record<string, string> = {
label_code: "标签"
};
const effectTone: Record<string, string> = {
improved: "tag-success",
unchanged: "tag-warning",
regressed: "tag-danger",
pending_data: ""
};
const effectLabels: Record<string, string> = {
improved: "已改善",
unchanged: "未明显变化",
regressed: "有恶化",
pending_data: "待观察"
};
export default function AiQualityDashboardPage() {
const [state, setState] = useState<LoadState>({ status: "loading" });
const [days, setDays] = useState(14);
const [actionMessage, setActionMessage] = useState<string | null>(null);
const [actionBusyKey, setActionBusyKey] = useState<string | null>(null);
const [filters, setFilters] = useState<ActionFilters>({
status: "",
owner: "",
priority: "",
actionType: "",
dimension: ""
});
async function loadDashboard(nextDays = days) {
setState({ status: "loading" });
@@ -185,7 +216,9 @@ export default function AiQualityDashboardPage() {
const dashboard = state.status === "ready" ? state.dashboard : null;
const improvementActions = dashboard?.improvement_actions ?? [];
const filteredActions = filterImprovementActions(improvementActions, filters);
const openActionCount = improvementActions.filter((action) => ["open", "in_progress"].includes(action.status)).length;
const ownerOptions = Array.from(new Set(improvementActions.map((action) => action.owner).filter(Boolean))).sort() as string[];
function changeDays(value: string) {
const nextDays = Number(value);
@@ -194,6 +227,10 @@ export default function AiQualityDashboardPage() {
void loadDashboard(nextDays);
}
function updateFilter(key: keyof ActionFilters, value: string) {
setFilters((current) => ({ ...current, [key]: value }));
}
async function createActionFromAttribution(item: QualityAttributionItem) {
const busyKey = attributionBusyKey(item);
setActionBusyKey(busyKey);
@@ -331,11 +368,46 @@ export default function AiQualityDashboardPage() {
<span className="tag tag-primary">待推进 {openActionCount}</span>
</div>
<div className="panel-body">
<div className="quality-action-filters">
<select className="search-input select-input-compact" value={filters.status} onChange={(event) => updateFilter("status", event.target.value)}>
<option value="">全部状态</option>
{Object.entries(statusLabels).map(([value, label]) => (
<option key={value} value={value}>{label}</option>
))}
</select>
<select className="search-input select-input-compact" value={filters.owner} onChange={(event) => updateFilter("owner", event.target.value)}>
<option value="">全部负责人</option>
{ownerOptions.map((owner) => (
<option key={owner} value={owner}>{owner}</option>
))}
</select>
<select className="search-input select-input-compact" value={filters.priority} onChange={(event) => updateFilter("priority", event.target.value)}>
<option value="">全部优先级</option>
<option value="critical">critical</option>
<option value="high">high</option>
<option value="medium">medium</option>
<option value="low">low</option>
</select>
<select className="search-input select-input-compact" value={filters.actionType} onChange={(event) => updateFilter("actionType", event.target.value)}>
<option value="">全部动作</option>
{Object.entries(actionTypeLabels).map(([value, label]) => (
<option key={value} value={value}>{label}</option>
))}
</select>
<select className="search-input select-input-compact" value={filters.dimension} onChange={(event) => updateFilter("dimension", event.target.value)}>
<option value="">全部维度</option>
{Object.entries(dimensionLabels).map(([value, label]) => (
<option key={value} value={value}>{label}</option>
))}
</select>
</div>
{improvementActions.length === 0 ? (
<div className="empty-state empty-state-compact">暂无改进项，可先从上方归因卡片生成。</div>
) : filteredActions.length === 0 ? (
<div className="empty-state empty-state-compact">当前筛选条件下没有改进项。</div>
) : (
<div className="quality-action-list">
{improvementActions.map((action) => (
{filteredActions.map((action) => (
<QualityActionItem
action={action}
busyKey={actionBusyKey}
@@ -499,6 +571,7 @@ function QualityActionItem({
const busy = busyKey?.startsWith(`${action.id}:`) ?? false;
const baselineRate = safeRate(action.baseline_affected_runs, action.baseline_total_runs);
const currentRate = safeRate(action.current_affected_runs, action.current_total_runs);
const effectStatus = action.effect_status ?? "pending_data";
return (
<div className="quality-action-item">
@@ -512,8 +585,11 @@ function QualityActionItem({
<span>{action.owner || "未分配"}</span>
</div>
</div>
<div className="action-row">
<span className={`tag ${effectTone[effectStatus] ?? ""}`}>{effectLabels[effectStatus] ?? effectStatus}</span>
<span className={`tag ${statusTone[action.status] ?? ""}`}>{statusLabels[action.status] ?? action.status}</span>
</div>
</div>
{(action.description || action.root_cause_hint) && (
<span className="item-meta">{action.description || action.root_cause_hint}</span>
)}
@@ -524,6 +600,7 @@ function QualityActionItem({
<span>变化 {formatSignedPercent(action.affected_rate_delta)}</span>
</div>
{action.resolution_note && <span className="item-meta">处理说明：{action.resolution_note}</span>}
{action.effect_summary && <span className="item-meta">效果复盘：{action.effect_summary}</span>}
<div className="action-row">
{action.status === "open" && (
<button className="button button-soft" disabled={busy} onClick={() => void onUpdate(action, "in_progress")} type="button">
@@ -548,6 +625,16 @@ function QualityActionItem({
);
}
function filterImprovementActions(actions: QualityImprovementAction[], filters: ActionFilters) {
return actions.filter((action) => (
(!filters.status || action.status === filters.status)
&& (!filters.owner || action.owner === filters.owner)
&& (!filters.priority || action.priority === filters.priority)
&& (!filters.actionType || action.action_type === filters.actionType)
&& (!filters.dimension || action.dimension === filters.dimension)
));
}
function inferActionType(item: QualityAttributionItem) {
if (item.dimension === "prompt_template") {
return "tune_prompt";
yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/quality-improvement-actions/route.ts
+5 -3
View File
@@ -7,12 +7,14 @@ const aiPlatformBaseUrl =
export async function GET(request: Request) {
const { searchParams } = new URL(request.url);
const status = searchParams.get("status");
const limit = searchParams.get("limit") ?? "20";
const targetUrl = new URL("/api/v1/audit/quality-improvement-actions", aiPlatformBaseUrl);
targetUrl.searchParams.set("limit", limit);
if (status) {
targetUrl.searchParams.set("status", status);
for (const name of ["status", "owner", "priority", "action_type", "dimension"]) {
const value = searchParams.get(name);
if (value) {
targetUrl.searchParams.set(name, value);
}
}
try {
yuqei-contract-platform/apps/contract-web/src/app/globals.css
+11
View File
@@ -743,6 +743,17 @@ select {
gap: 12px;
}
.quality-action-filters {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin-bottom: 14px;
}
.quality-action-filters .search-input {
min-width: 128px;
}
.quality-action-item {
display: grid;
gap: 10px;