yuqei/ai-contract-platform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add ai run quality evaluation

Browse Source
This commit is contained in:
yuqei
2026-06-23 00:56:50 +08:00
parent 81977ead8f
commit 25b42f1abd
13 changed files with 533 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx
+108 -4
View File
@@ -10,6 +10,7 @@ import {
FileText,
RefreshCcw,
Search,
ShieldCheck,
Sigma
} from "lucide-react";
@@ -29,6 +30,13 @@ type RetrievalSource = {
matched_terms?: string[];
};
type AiQualityLabel = {
code: string;
severity?: string;
title: string;
detail: string;
};
type AiRunAudit = {
id: string;
operation: string;
@@ -51,6 +59,10 @@ type AiRunAudit = {
total_tokens?: number;
estimated_cost?: number;
latency_ms?: number;
quality_status?: string;
quality_score?: number;
quality_labels?: AiQualityLabel[];
quality_summary?: string | null;
status: string;
created_at: string;
};
@@ -80,9 +92,48 @@ const statusTone: Record<string, string> = {
cancelled: "tag-danger"
};
const qualityStatusLabels: Record<string, string> = {
passed: "质量通过",
needs_review: "需要复核",
failed: "质量失败",
unknown: "未评测"
};
const qualityStatusTone: Record<string, string> = {
passed: "tag-success",
needs_review: "tag-warning",
failed: "tag-danger",
unknown: ""
};
const qualityLabelText: Record<string, { title: string; detail: string }> = {
citation_present: { title: "已引用依据", detail: "回答已返回法条或证据来源。" },
missing_citation: { title: "缺少依据", detail: "回答未返回法条或证据来源，需要复核。" },
source_covered: { title: "来源覆盖", detail: "本次回答记录了多个知识命中来源。" },
source_limited: { title: "来源偏少", detail: "本次回答仅记录一个知识命中来源，建议复核覆盖面。" },
source_insufficient: { title: "来源不足", detail: "本次回答没有记录知识命中来源。" },
fallback_used: { title: "使用回退", detail: "本次回答走了本地回退链路。" },
run_failed: { title: "调用失败", detail: "本次 AI 调用未成功完成。" },
provider_succeeded: { title: "模型成功", detail: "模型服务返回成功响应。" },
provider_failed: { title: "模型失败", detail: "模型服务返回失败状态。" },
token_normal: { title: "Token 正常", detail: "本次调用 token 用量在阈值内。" },
token_missing: { title: "Token 缺失", detail: "本次调用没有返回 token 用量。" },
token_high: { title: "Token 偏高", detail: "本次调用 token 用量超过 MVP 复核阈值。" },
latency_normal: { title: "耗时正常", detail: "本次调用耗时在阈值内。" },
latency_high: { title: "耗时偏高", detail: "本次调用耗时超过 15 秒。" },
latency_critical: { title: "耗时严重", detail: "本次调用耗时超过 30 秒。" }
};
const severityTone: Record<string, string> = {
info: "tag-primary",
warning: "tag-warning",
error: "tag-danger"
};
export default function AiTracesPage() {
const [state, setState] = useState<LoadState>({ status: "loading" });
const [statusFilter, setStatusFilter] = useState("all");
const [qualityFilter, setQualityFilter] = useState("all");
const [keyword, setKeyword] = useState("");
async function loadRuns() {
@@ -111,6 +162,8 @@ export default function AiTracesPage() {
const normalizedKeyword = keyword.trim().toLowerCase();
return runs.filter((run) => {
const matchesStatus = statusFilter === "all" || run.status === statusFilter;
const qualityStatus = run.quality_status ?? "unknown";
const matchesQuality = qualityFilter === "all" || qualityStatus === qualityFilter;
const haystack = [
run.id,
run.operation,
@@ -120,20 +173,27 @@ export default function AiTracesPage() {
run.answer,
run.provider_error,
run.retrieval_query,
run.quality_status,
run.quality_summary,
...(run.quality_labels ?? []).map((label) => `${label.code} ${label.title} ${label.detail}`),
...(run.retrieved_sources ?? []).map((source) => `${source.title} ${source.reference}`)
].join("\n").toLowerCase();
return matchesStatus && (!normalizedKeyword || haystack.includes(normalizedKeyword));
return matchesStatus && matchesQuality && (!normalizedKeyword || haystack.includes(normalizedKeyword));
});
}, [keyword, runs, statusFilter]);
}, [keyword, qualityFilter, runs, statusFilter]);
const metrics = useMemo(() => {
const totalTokens = runs.reduce((sum, run) => sum + (run.total_tokens ?? 0), 0);
const estimatedCost = runs.reduce((sum, run) => sum + (run.estimated_cost ?? 0), 0);
const failedCount = runs.filter((run) => run.status === "failed" || run.provider_status === "failed").length;
const qualityReviewCount = runs.filter((run) => {
const qualityStatus = run.quality_status ?? "unknown";
return qualityStatus === "needs_review" || qualityStatus === "failed";
}).length;
const averageLatency = runs.length
? Math.round(runs.reduce((sum, run) => sum + (run.latency_ms ?? 0), 0) / runs.length)
: 0;
return { totalTokens, estimatedCost, failedCount, averageLatency };
return { totalTokens, estimatedCost, failedCount, qualityReviewCount, averageLatency };
}, [runs]);
return (
@@ -153,6 +213,7 @@ export default function AiTracesPage() {
<Metric icon={<Bot size={18} />} label="最近调用" value={String(runs.length)} />
<Metric icon={<Sigma size={18} />} label="总 token" value={formatInteger(metrics.totalTokens)} />
<Metric icon={<Clock3 size={18} />} label="平均耗时" value={`${metrics.averageLatency} ms`} />
<Metric icon={<ShieldCheck size={18} />} label="质量待复核" value={String(metrics.qualityReviewCount)} />
<Metric icon={<AlertTriangle size={18} />} label="异常调用" value={String(metrics.failedCount)} />
</div>
@@ -182,6 +243,17 @@ export default function AiTracesPage() {
<option value="fallback">已回退</option>
<option value="failed">失败</option>
</select>
<select
className="search-input select-input"
value={qualityFilter}
onChange={(event) => setQualityFilter(event.target.value)}
>
<option value="all">全部质量</option>
<option value="passed">质量通过</option>
<option value="needs_review">需要复核</option>
<option value="failed">质量失败</option>
<option value="unknown">未评测</option>
</select>
</div>
{state.status === "loading" && <div className="empty-state">正在加载 AI 调用审计...</div>}
@@ -220,6 +292,9 @@ function Metric({ icon, label, value }: { icon: ReactNode; label: string; value:
function TraceItem({ run }: { run: AiRunAudit }) {
const statusClass = statusTone[run.status] ?? "";
const providerStatusClass = statusTone[run.provider_status ?? ""] ?? "";
const qualityStatus = run.quality_status ?? "unknown";
const qualityClass = qualityStatusTone[qualityStatus] ?? "";
const qualityLabels = run.quality_labels ?? [];
return (
<details className="trace-item">
@@ -227,7 +302,12 @@ function TraceItem({ run }: { run: AiRunAudit }) {
<div className="trace-summary-main">
<div className="list-item-header">
<strong>{run.question || run.operation}</strong>
<span className={`tag ${statusClass}`}>{statusLabels[run.status] ?? run.status}</span>
<div className="trace-summary-badges">
<span className={`tag ${qualityClass}`}>
{qualityStatusLabels[qualityStatus] ?? qualityStatus} · {run.quality_score ?? 0}
</span>
<span className={`tag ${statusClass}`}>{statusLabels[run.status] ?? run.status}</span>
</div>
</div>
<div className="trace-meta">
<span>{formatDate(run.created_at)}</span>
@@ -271,6 +351,30 @@ function TraceItem({ run }: { run: AiRunAudit }) {
</div>
)}
</section>
<section className="trace-block">
<h3><ShieldCheck size={16} /> 质量评测</h3>
<div className="trace-facts">
<span>质量状态</span><strong className={`tag ${qualityClass}`}>{qualityStatusLabels[qualityStatus] ?? qualityStatus}</strong>
<span>质量分</span><strong>{run.quality_score ?? 0}</strong>
<span>摘要</span><strong>{run.quality_summary || "暂无质量摘要"}</strong>
</div>
{qualityLabels.length === 0 ? (
<p className="muted">暂无质量标签。</p>
) : (
<div className="quality-labels">
{qualityLabels.map((label) => {
const text = qualityLabelText[label.code];
const severityClass = severityTone[label.severity ?? "info"] ?? "";
return (
<div className="quality-label" key={label.code}>
<span className={`tag ${severityClass}`}>{text?.title ?? label.title}</span>
<span>{text?.detail ?? label.detail}</span>
</div>
);
})}
</div>
)}
</section>
<section className="trace-block">
<h3><AlertTriangle size={16} /> 运行状态</h3>
<div className="trace-facts">
yuqei-contract-platform/apps/contract-web/src/app/globals.css
+23 -1
View File
@@ -237,7 +237,7 @@ select {
.metric-row {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
grid-template-columns: repeat(auto-fit, minmax(160px, 1fr));
gap: 12px;
margin-bottom: 16px;
}
@@ -534,6 +534,13 @@ select {
gap: 8px;
}
.trace-summary-badges {
display: flex;
flex-wrap: wrap;
justify-content: flex-end;
gap: 6px;
}
.trace-meta {
display: flex;
flex-wrap: wrap;
@@ -603,6 +610,21 @@ select {
overflow-wrap: anywhere;
}
.quality-labels {
display: grid;
gap: 8px;
}
.quality-label {
display: grid;
grid-template-columns: max-content minmax(0, 1fr);
gap: 8px 10px;
align-items: center;
color: var(--muted);
font-size: 13px;
line-height: 1.5;
}
.compact-list-item {
padding: 12px;
}