From 803b80dc6f39f3638e0ba975433ffbe0f5fab015 Mon Sep 17 00:00:00 2001 From: yuqei Date: Tue, 23 Jun 2026 01:14:40 +0800 Subject: [PATCH] feat: add ai quality rule administration --- .../V2-ai-quality-rules-admin-2026-06-23.md | 65 ++++++ .../packages/ai-openapi/openapi.yaml | 134 ++++++++++++ ...260623_0008_add_ai_quality_rule_configs.py | 37 ++++ .../src/yuqei_ai_platform_api/main.py | 77 +++++-- .../src/yuqei_ai_platform_api/models.py | 17 ++ .../src/yuqei_ai_platform_api/repository.py | 173 ++++++++++++++- .../tests/test_platform_data.py | 100 +++++++++ yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py | 6 + .../src/yuqei_sdk/ai_platform.py | 67 ++++++ yuqei-ai-sdk-python/tests/test_ai_platform.py | 110 ++++++++++ .../src/app/admin/ai-quality/page.tsx | 204 ++++++++++++++++++ .../src/app/admin/ai-traces/page.tsx | 92 +++++++- .../apps/contract-web/src/app/admin/page.tsx | 4 +- .../quality-labels/[labelCode]/route.ts | 47 ++++ .../quality-rules/default/route.ts | 68 ++++++ .../apps/contract-web/src/app/globals.css | 73 ++++++- 16 files changed, 1250 insertions(+), 24 deletions(-) create mode 100644 project-docs/V2-ai-quality-rules-admin-2026-06-23.md create mode 100644 yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0008_add_ai_quality_rule_configs.py create mode 100644 yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality/page.tsx create mode 100644 yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/ai-runs/[runId]/quality-labels/[labelCode]/route.ts create mode 100644 yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/quality-rules/default/route.ts diff --git a/project-docs/V2-ai-quality-rules-admin-2026-06-23.md b/project-docs/V2-ai-quality-rules-admin-2026-06-23.md new file mode 100644 index 00000000..214c935c --- /dev/null +++ b/project-docs/V2-ai-quality-rules-admin-2026-06-23.md @@ -0,0 +1,65 @@ +# V2 AI 质量规则后台化与人工闭环记录 + +日期：2026-06-23 + +## 本轮目标 + +把 AI 质量评测 MVP 中写死的 token、耗时、来源数量等阈值改成后台可热更新配置，并给每个质量标签增加人工处理状态，形成“自动评测 + 管理员复核”的闭环。 + +## 已落地内容 + +1. AI Platform 新增质量规则配置。 + - 新表：`ai_quality_rule_configs` + - 迁移：`20260623_0008_add_ai_quality_rule_configs.py` + - 默认配置 ID：`default` + - 配置项： + - `min_citation_count` + - `min_source_count` + - `high_token_threshold` + - `high_latency_ms` + - `critical_latency_ms` + - `warning_penalty` + - `error_penalty` + +2. AI 质量评测改为读取后台规则。 + - 法律问答写入 AI run audit 前，会读取 `quality-rules/default`。 + - 来源不足、token 过高、耗时异常、扣分规则均由配置决定。 + - `critical_latency_ms` 必须大于等于 `high_latency_ms`。 + +3. 质量标签新增人工处理状态。 + - `pending`：待处理 + - `reviewed`：已复核 + - `false_positive`：误报 + - `resolved`：已处理 + - 标签同时记录 `review_note`、`reviewed_by`、`reviewed_at`。 + +4. 后端 API 与 SDK 同步。 + - `GET /api/v1/quality-rules/default` + - `PUT /api/v1/quality-rules/default` + - `PATCH /api/v1/audit/ai-runs/{run_id}/quality-labels/{label_code}` + - OpenAPI 和 `yuqei-ai-sdk-python` 已同步。 + +5. 合同 Web 管理区新增页面。 + - 新页面：`/admin/ai-quality` + - 管理区“AI 参数”入口指向该页面。 + - 可配置依据数量、来源数量、token 阈值、耗时阈值和扣分规则。 + +6. AI Trace 页增强。 + - 每个质量标签显示人工处理状态。 + - 管理员可直接在 Trace 详情中切换：待处理、已复核、误报、已处理。 + - 状态回写 AI Platform 审计记录。 + +## 验收重点 + +- 打开 `/admin/ai-quality` 能读取并保存质量规则。 +- 保存规则后，新产生的 AI run audit 使用新阈值评测。 +- 打开 `/admin/ai-traces`，展开任一 Trace，可修改质量标签处理状态。 +- 修改后的状态再次拉取审计列表时仍然存在。 + +## 下一步建议 + +进入 AI 质量运营看板： + +- 按日期展示质量通过率、fallback 率、来源不足率、耗时异常率。 +- 支持只看“待处理”标签，形成管理员每日处理队列。 +- 给每类异常增加趋势和 Top 问题，辅助调整模型、Prompt 和知识库。 diff --git a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml index 49995be9..2aa67373 100644 --- a/yuqei-ai-platform/packages/ai-openapi/openapi.yaml +++ b/yuqei-ai-platform/packages/ai-openapi/openapi.yaml @@ -441,6 +441,61 @@ paths: type: array items: $ref: "#/components/schemas/AiRun" + /api/v1/audit/ai-runs/{run_id}/quality-labels/{label_code}: + patch: + operationId: updateAiRunQualityLabel + summary: Update AI run quality label review status + parameters: + - name: run_id + in: path + required: true + schema: + type: string + - name: label_code + in: path + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityLabelReviewUpdate" + responses: + "200": + description: Updated AI run audit + content: + application/json: + schema: + $ref: "#/components/schemas/AiRun" + /api/v1/quality-rules/default: + get: + operationId: getQualityRuleConfig + summary: Get default AI quality rule config + responses: + "200": + description: AI quality rule config + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityRuleConfig" + put: + operationId: upsertQualityRuleConfig + summary: Update default AI quality rule config + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityRuleConfigUpdate" + responses: + "200": + description: AI quality rule config + content: + application/json: + schema: + $ref: "#/components/schemas/AiQualityRuleConfig" components: parameters: TraceIdHeader: @@ -1165,6 +1220,85 @@ components: type: string detail: type: string + review_status: + type: string + enum: [pending, reviewed, false_positive, resolved] + default: pending + review_note: + type: string + nullable: true + reviewed_by: + type: string + nullable: true + reviewed_at: + type: string + format: date-time + nullable: true + AiQualityLabelReviewUpdate: + type: object + required: + - review_status + properties: + review_status: + type: string + enum: [pending, reviewed, false_positive, resolved] + review_note: + type: string + nullable: true + reviewed_by: + type: string + nullable: true + AiQualityRuleConfigUpdate: + type: object + properties: + min_citation_count: + type: integer + minimum: 0 + maximum: 10 + default: 1 + min_source_count: + type: integer + minimum: 0 + maximum: 10 + default: 2 + high_token_threshold: + type: integer + minimum: 1 + maximum: 1000000 + default: 8000 + high_latency_ms: + type: integer + minimum: 1 + maximum: 600000 + default: 15000 + critical_latency_ms: + type: integer + minimum: 1 + maximum: 600000 + default: 30000 + warning_penalty: + type: integer + minimum: 0 + maximum: 100 + default: 10 + error_penalty: + type: integer + minimum: 0 + maximum: 100 + default: 30 + AiQualityRuleConfig: + allOf: + - $ref: "#/components/schemas/AiQualityRuleConfigUpdate" + - type: object + required: + - id + - updated_at + properties: + id: + type: string + updated_at: + type: string + format: date-time AiRun: type: object required: diff --git a/yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0008_add_ai_quality_rule_configs.py b/yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0008_add_ai_quality_rule_configs.py new file mode 100644 index 00000000..a8fb778f --- /dev/null +++ b/yuqei-ai-platform/services/ai-platform-api/migrations/versions/20260623_0008_add_ai_quality_rule_configs.py @@ -0,0 +1,37 @@ +"""add ai quality rule configs + +Revision ID: 20260623_0008 +Revises: 20260623_0007 +Create Date: 2026-06-23 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260623_0008" +down_revision: Union[str, None] = "20260623_0007" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "ai_quality_rule_configs", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("min_citation_count", sa.Integer(), nullable=False, server_default="1"), + sa.Column("min_source_count", sa.Integer(), nullable=False, server_default="2"), + sa.Column("high_token_threshold", sa.Integer(), nullable=False, server_default="8000"), + sa.Column("high_latency_ms", sa.Integer(), nullable=False, server_default="15000"), + sa.Column("critical_latency_ms", sa.Integer(), nullable=False, server_default="30000"), + sa.Column("warning_penalty", sa.Integer(), nullable=False, server_default="10"), + sa.Column("error_penalty", sa.Integer(), nullable=False, server_default="30"), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + + +def downgrade() -> None: + op.drop_table("ai_quality_rule_configs") diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py index f0d5dbc3..9e26e6fc 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/main.py @@ -2,7 +2,7 @@ from time import perf_counter from typing import Literal from uuid import uuid4 -from fastapi import FastAPI, Query +from fastapi import FastAPI, HTTPException, Query from pydantic import BaseModel from yuqei_ai_platform_api import __version__ @@ -14,6 +14,9 @@ from yuqei_ai_platform_api.providers import ProviderRequest, build_provider_adap from yuqei_ai_platform_api.repository import ( AiPlatformRepository, AiQualityLabel, + AiQualityLabelReviewUpdate, + AiQualityRuleConfig, + AiQualityRuleConfigCreate, AiRunAudit, KnowledgeChunk, KnowledgeDocument, @@ -105,6 +108,27 @@ def create_app( def upsert_prompt_template(payload: PromptTemplateCreate) -> PromptTemplate: return store.upsert_prompt_template(payload) + @app.get( + f"{resolved_settings.api_prefix}/quality-rules/default", + response_model=AiQualityRuleConfig, + tags=["quality"], + ) + def get_quality_rule_config() -> AiQualityRuleConfig: + return store.get_quality_rule_config() + + @app.put( + f"{resolved_settings.api_prefix}/quality-rules/default", + response_model=AiQualityRuleConfig, + tags=["quality"], + ) + def upsert_quality_rule_config(payload: AiQualityRuleConfigCreate) -> AiQualityRuleConfig: + if payload.critical_latency_ms < payload.high_latency_ms: + raise HTTPException( + status_code=422, + detail="critical_latency_ms must be greater than or equal to high_latency_ms.", + ) + return store.upsert_quality_rule_config(payload) + @app.post( f"{resolved_settings.api_prefix}/knowledge-documents", response_model=KnowledgeDocument, @@ -210,6 +234,21 @@ def create_app( def list_ai_runs(limit: int = Query(20, ge=1, le=100)) -> list[AiRunAudit]: return store.list_ai_run_audits(limit=limit) + @app.patch( + f"{resolved_settings.api_prefix}/audit/ai-runs/{{run_id}}/quality-labels/{{label_code}}", + response_model=AiRunAudit, + tags=["audit"], + ) + def update_ai_run_quality_label( + run_id: str, + label_code: str, + payload: AiQualityLabelReviewUpdate, + ) -> AiRunAudit: + try: + return store.update_ai_run_quality_label(run_id, label_code, payload) + except KeyError as error: + raise HTTPException(status_code=404, detail="AI run or quality label not found.") from error + @app.post( f"{resolved_settings.api_prefix}/legal/qa", response_model=LegalQaResponse, @@ -251,7 +290,9 @@ def create_app( ) audit_status = "failed" latency_ms = max(0, int((perf_counter() - started_at) * 1000)) + quality_rules = store.get_quality_rule_config() quality_status, quality_score, quality_labels, quality_summary = _evaluate_ai_run_quality( + rules=quality_rules, status=audit_status, provider_status=provider_response.status, provider_error=provider_response.error_message, @@ -306,6 +347,7 @@ def _preview_text(value: str, *, limit: int = 4000) -> str: def _evaluate_ai_run_quality( *, + rules: AiQualityRuleConfig, status: str, provider_status: str | None, provider_error: str | None, @@ -316,13 +358,16 @@ def _evaluate_ai_run_quality( ) -> tuple[str, int, list[AiQualityLabel], str]: labels: list[AiQualityLabel] = [] - if citations_count <= 0: + if citations_count < rules.min_citation_count: labels.append( AiQualityLabel( code="missing_citation", severity="error", title="Missing citation", - detail="The answer did not return any legal citation or evidence source.", + detail=( + f"The answer returned {citations_count} citation(s), " + f"below the required {rules.min_citation_count}." + ), ) ) else: @@ -335,7 +380,7 @@ def _evaluate_ai_run_quality( ) ) - if retrieved_sources_count <= 0: + if retrieved_sources_count <= 0 and rules.min_source_count > 0: labels.append( AiQualityLabel( code="source_insufficient", @@ -344,13 +389,16 @@ def _evaluate_ai_run_quality( detail="No retrieved knowledge chunk source was recorded for this run.", ) ) - elif retrieved_sources_count < 2: + elif retrieved_sources_count < rules.min_source_count: labels.append( AiQualityLabel( code="source_limited", severity="warning", title="Source limited", - detail="Only one retrieved source was recorded; reviewer may need to verify coverage.", + detail=( + f"{retrieved_sources_count} retrieved source(s) were recorded, " + f"below the required {rules.min_source_count}." + ), ) ) else: @@ -410,13 +458,16 @@ def _evaluate_ai_run_quality( detail="Token usage was not reported for this run.", ) ) - elif total_tokens > 8000: + elif total_tokens > rules.high_token_threshold: labels.append( AiQualityLabel( code="token_high", severity="warning", title="Token usage high", - detail=f"The run used {total_tokens} tokens, which is above the MVP review threshold.", + detail=( + f"The run used {total_tokens} tokens, above the configured " + f"{rules.high_token_threshold} token threshold." + ), ) ) else: @@ -429,22 +480,22 @@ def _evaluate_ai_run_quality( ) ) - if latency_ms >= 30000: + if latency_ms >= rules.critical_latency_ms: labels.append( AiQualityLabel( code="latency_critical", severity="error", title="Latency critical", - detail=f"The run took {latency_ms} ms.", + detail=f"The run took {latency_ms} ms, above the critical threshold.", ) ) - elif latency_ms >= 15000: + elif latency_ms >= rules.high_latency_ms: labels.append( AiQualityLabel( code="latency_high", severity="warning", title="Latency high", - detail=f"The run took {latency_ms} ms.", + detail=f"The run took {latency_ms} ms, above the high latency threshold.", ) ) else: @@ -459,7 +510,7 @@ def _evaluate_ai_run_quality( error_count = sum(1 for label in labels if label.severity == "error") warning_count = sum(1 for label in labels if label.severity == "warning") - score = max(0, 100 - error_count * 30 - warning_count * 10) + score = max(0, 100 - error_count * rules.error_penalty - warning_count * rules.warning_penalty) if error_count: quality_status = "failed" elif warning_count: diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py index 7b24501b..48a0c78b 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/models.py @@ -40,6 +40,23 @@ class PromptTemplateModel(Base): ) +class AiQualityRuleConfigModel(Base): + __tablename__ = "ai_quality_rule_configs" + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + min_citation_count: Mapped[int] = mapped_column(Integer, default=1) + min_source_count: Mapped[int] = mapped_column(Integer, default=2) + high_token_threshold: Mapped[int] = mapped_column(Integer, default=8000) + high_latency_ms: Mapped[int] = mapped_column(Integer, default=15000) + critical_latency_ms: Mapped[int] = mapped_column(Integer, default=30000) + warning_penalty: Mapped[int] = mapped_column(Integer, default=10) + error_penalty: Mapped[int] = mapped_column(Integer, default=30) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(UTC), + ) + + class KnowledgeDocumentModel(Base): __tablename__ = "knowledge_documents" diff --git a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py index 3afbdb19..c095ccae 100644 --- a/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py +++ b/yuqei-ai-platform/services/ai-platform-api/src/yuqei_ai_platform_api/repository.py @@ -24,6 +24,7 @@ from yuqei_ai_platform_api.embeddings import ( from yuqei_ai_platform_api.legal_qa import LegalCitation from yuqei_ai_platform_api.models import ( AiProviderConfigModel, + AiQualityRuleConfigModel, AiRunAuditModel, Base, KnowledgeChunkModel, @@ -67,6 +68,21 @@ class PromptTemplate(PromptTemplateCreate): updated_at: datetime +class AiQualityRuleConfigCreate(BaseModel): + min_citation_count: int = Field(default=1, ge=0, le=10) + min_source_count: int = Field(default=2, ge=0, le=10) + high_token_threshold: int = Field(default=8000, ge=1, le=1_000_000) + high_latency_ms: int = Field(default=15000, ge=1, le=600_000) + critical_latency_ms: int = Field(default=30000, ge=1, le=600_000) + warning_penalty: int = Field(default=10, ge=0, le=100) + error_penalty: int = Field(default=30, ge=0, le=100) + + +class AiQualityRuleConfig(AiQualityRuleConfigCreate): + id: str = "default" + updated_at: datetime + + class KnowledgeDocumentCreate(BaseModel): import_batch_id: str | None = None knowledge_base_id: str = Field(default="default") @@ -195,6 +211,16 @@ class AiQualityLabel(BaseModel): severity: str = "info" title: str detail: str + review_status: str = "pending" + review_note: str | None = None + reviewed_by: str | None = None + reviewed_at: datetime | None = None + + +class AiQualityLabelReviewUpdate(BaseModel): + review_status: str = Field(pattern="^(pending|reviewed|false_positive|resolved)$") + review_note: str | None = None + reviewed_by: str | None = None class AiRunAudit(BaseModel): @@ -240,6 +266,10 @@ class AiPlatformRepository(Protocol): def get_prompt_template(self, name: str) -> PromptTemplate | None: ... + def get_quality_rule_config(self) -> AiQualityRuleConfig: ... + + def upsert_quality_rule_config(self, payload: AiQualityRuleConfigCreate) -> AiQualityRuleConfig: ... + def add_knowledge_document(self, payload: KnowledgeDocumentCreate) -> KnowledgeDocument: ... def create_knowledge_import_batch(self, payload: KnowledgeImportBatchCreate) -> KnowledgeImportBatch: ... @@ -286,6 +316,13 @@ class AiPlatformRepository(Protocol): def list_ai_run_audits(self, *, limit: int = 20) -> list[AiRunAudit]: ... + def update_ai_run_quality_label( + self, + run_id: str, + label_code: str, + payload: AiQualityLabelReviewUpdate, + ) -> AiRunAudit: ... + def seed_defaults(self) -> None: ... @@ -297,6 +334,12 @@ class InMemoryAiPlatformRepository: knowledge_documents: dict[str, KnowledgeDocument] = field(default_factory=dict) knowledge_chunks: dict[str, KnowledgeChunk] = field(default_factory=dict) ai_run_audits: list[AiRunAudit] = field(default_factory=list) + quality_rule_config: AiQualityRuleConfig = field( + default_factory=lambda: AiQualityRuleConfig( + id="default", + updated_at=datetime.now(UTC), + ) + ) embedding_provider: EmbeddingProvider = field(default_factory=LocalHashEmbeddingProvider) _lock: Lock = field(default_factory=Lock) @@ -371,6 +414,20 @@ class InMemoryAiPlatformRepository: return None return sorted(templates, key=lambda item: item.updated_at, reverse=True)[0] + def get_quality_rule_config(self) -> AiQualityRuleConfig: + with self._lock: + return self.quality_rule_config + + def upsert_quality_rule_config(self, payload: AiQualityRuleConfigCreate) -> AiQualityRuleConfig: + with self._lock: + config = AiQualityRuleConfig( + id="default", + updated_at=datetime.now(UTC), + **payload.model_dump(), + ) + self.quality_rule_config = config + return config + def add_knowledge_document(self, payload: KnowledgeDocumentCreate) -> KnowledgeDocument: with self._lock: document = KnowledgeDocument( @@ -538,6 +595,22 @@ class InMemoryAiPlatformRepository: with self._lock: return self.ai_run_audits[:limit] + def update_ai_run_quality_label( + self, + run_id: str, + label_code: str, + payload: AiQualityLabelReviewUpdate, + ) -> AiRunAudit: + with self._lock: + for index, audit in enumerate(self.ai_run_audits): + if audit.id != run_id: + continue + updated_labels = _update_quality_label_list(audit.quality_labels, label_code, payload) + updated_audit = audit.model_copy(update={"quality_labels": updated_labels}) + self.ai_run_audits[index] = updated_audit + return updated_audit + raise KeyError(run_id) + def seed_defaults(self) -> None: if not self.list_provider_configs(): self.upsert_provider_config(_default_provider_config()) @@ -652,6 +725,32 @@ class SqlAlchemyAiPlatformRepository: ) return _prompt_template_from_model(template) if template else None + def get_quality_rule_config(self) -> AiQualityRuleConfig: + with self._session_factory.begin() as session: + row = session.get(AiQualityRuleConfigModel, "default") + if row is None: + row = AiQualityRuleConfigModel( + id="default", + updated_at=datetime.now(UTC), + **AiQualityRuleConfigCreate().model_dump(), + ) + session.add(row) + session.flush() + return _quality_rule_config_from_model(row) + + def upsert_quality_rule_config(self, payload: AiQualityRuleConfigCreate) -> AiQualityRuleConfig: + now = datetime.now(UTC) + with self._session_factory.begin() as session: + row = session.get(AiQualityRuleConfigModel, "default") + if row is None: + row = AiQualityRuleConfigModel(id="default") + session.add(row) + for key, value in payload.model_dump().items(): + setattr(row, key, value) + row.updated_at = now + session.flush() + return _quality_rule_config_from_model(row) + def add_knowledge_document(self, payload: KnowledgeDocumentCreate) -> KnowledgeDocument: document = KnowledgeDocumentModel( id=f"doc-{uuid4().hex[:12]}", @@ -976,7 +1075,7 @@ class SqlAlchemyAiPlatformRepository: quality_status=audit.quality_status, quality_score=audit.quality_score, quality_labels_json=_encode_json_list( - [label.model_dump() for label in audit.quality_labels] + [label.model_dump(mode="json") for label in audit.quality_labels] ), quality_summary=audit.quality_summary, status=audit.status, @@ -994,6 +1093,27 @@ class SqlAlchemyAiPlatformRepository: ) return [_ai_run_audit_from_model(row) for row in rows] + def update_ai_run_quality_label( + self, + run_id: str, + label_code: str, + payload: AiQualityLabelReviewUpdate, + ) -> AiRunAudit: + with self._session_factory.begin() as session: + row = session.get(AiRunAuditModel, run_id) + if row is None: + raise KeyError(run_id) + labels = [ + AiQualityLabel.model_validate(item) + for item in _decode_json_list(row.quality_labels_json) + ] + updated_labels = _update_quality_label_list(labels, label_code, payload) + row.quality_labels_json = _encode_json_list( + [label.model_dump(mode="json") for label in updated_labels] + ) + session.flush() + return _ai_run_audit_from_model(row) + def seed_defaults(self) -> None: if not self.list_provider_configs(): self.upsert_provider_config(_default_provider_config()) @@ -1113,6 +1233,20 @@ def _prompt_template_from_model(row: PromptTemplateModel) -> PromptTemplate: ) +def _quality_rule_config_from_model(row: AiQualityRuleConfigModel) -> AiQualityRuleConfig: + return AiQualityRuleConfig( + id=row.id, + min_citation_count=row.min_citation_count, + min_source_count=row.min_source_count, + high_token_threshold=row.high_token_threshold, + high_latency_ms=row.high_latency_ms, + critical_latency_ms=row.critical_latency_ms, + warning_penalty=row.warning_penalty, + error_penalty=row.error_penalty, + updated_at=_ensure_utc(row.updated_at), + ) + + def _knowledge_document_from_model(row: KnowledgeDocumentModel) -> KnowledgeDocument: return KnowledgeDocument( id=row.id, @@ -1223,6 +1357,35 @@ def _ensure_utc(value: datetime) -> datetime: return value +def _update_quality_label_list( + labels: list[AiQualityLabel], + label_code: str, + payload: AiQualityLabelReviewUpdate, +) -> list[AiQualityLabel]: + updated_labels: list[AiQualityLabel] = [] + found = False + for label in labels: + if label.code != label_code: + updated_labels.append(label) + continue + found = True + reviewed_at = None if payload.review_status == "pending" else datetime.now(UTC) + reviewed_by = None if payload.review_status == "pending" else payload.reviewed_by + updated_labels.append( + label.model_copy( + update={ + "review_status": payload.review_status, + "review_note": payload.review_note, + "reviewed_by": reviewed_by, + "reviewed_at": reviewed_at, + } + ) + ) + if not found: + raise KeyError(label_code) + return updated_labels + + def _default_provider_config() -> ProviderConfigCreate: return ProviderConfigCreate( provider="mock", @@ -1642,7 +1805,7 @@ def _to_pgvector_literal(vector: list[float]) -> str: def _encode_json_list(values: list[object]) -> str: - return json.dumps(values, ensure_ascii=False, separators=(",", ":")) + return json.dumps(values, ensure_ascii=False, separators=(",", ":"), default=_json_default) def _decode_json_list(payload: str | None) -> list[object]: @@ -1653,3 +1816,9 @@ def _decode_json_list(payload: str | None) -> list[object]: except json.JSONDecodeError: return [] return parsed if isinstance(parsed, list) else [] + + +def _json_default(value: object) -> str: + if isinstance(value, datetime): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") diff --git a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py index 92037b99..63d07a91 100644 --- a/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py +++ b/yuqei-ai-platform/services/ai-platform-api/tests/test_platform_data.py @@ -4,6 +4,8 @@ from yuqei_ai_platform_api.config import AiPlatformSettings from yuqei_ai_platform_api.embeddings import EmbeddingVector from yuqei_ai_platform_api.main import create_app from yuqei_ai_platform_api.repository import ( + AiQualityLabelReviewUpdate, + AiQualityRuleConfigCreate, AiRunAudit, InMemoryAiPlatformRepository, KnowledgeDocumentCreate, @@ -91,6 +93,79 @@ def test_prompt_template_can_be_hot_updated_and_listed() -> None: assert templates[0]["version"] == "v2" +def test_quality_rules_can_be_hot_updated_and_drive_audit_labels() -> None: + client = make_client() + + default_response = client.get("/api/v1/quality-rules/default") + assert default_response.status_code == 200 + assert default_response.json()["min_source_count"] == 2 + + update_response = client.put( + "/api/v1/quality-rules/default", + json={ + "min_citation_count": 1, + "min_source_count": 1, + "high_token_threshold": 8000, + "high_latency_ms": 15000, + "critical_latency_ms": 30000, + "warning_penalty": 5, + "error_penalty": 25, + }, + ) + assert update_response.status_code == 200 + assert update_response.json()["warning_penalty"] == 5 + + client.post( + "/api/v1/knowledge-documents", + json={ + "knowledge_base_id": "enterprise", + "title": "Lease Review Policy", + "source_type": "internal_policy", + "reference": "Lease-001", + "content": "Deposit refund conditions must be written into the lease contract.", + }, + ) + response = client.post( + "/api/v1/legal/qa", + json={ + "question": "How should deposit refund be handled?", + "knowledge_base_ids": ["enterprise"], + }, + ) + assert response.status_code == 200 + + audit = client.get("/api/v1/audit/ai-runs").json()[0] + quality_codes = {label["code"] for label in audit["quality_labels"]} + assert "source_limited" not in quality_codes + assert "source_covered" in quality_codes + assert audit["quality_status"] == "needs_review" + assert audit["quality_score"] == 90 + + +def test_ai_run_quality_label_review_status_can_be_updated() -> None: + client = make_client() + + response = client.post("/api/v1/legal/qa", json={"question": "试用期如何约定？"}) + assert response.status_code == 200 + audit = client.get("/api/v1/audit/ai-runs").json()[0] + + update_response = client.patch( + f"/api/v1/audit/ai-runs/{audit['id']}/quality-labels/fallback_used", + json={ + "review_status": "reviewed", + "review_note": "已确认 fallback 原因，等待模型配置修复。", + "reviewed_by": "admin", + }, + ) + assert update_response.status_code == 200 + updated = update_response.json() + label = next(item for item in updated["quality_labels"] if item["code"] == "fallback_used") + assert label["review_status"] == "reviewed" + assert label["review_note"] == "已确认 fallback 原因，等待模型配置修复。" + assert label["reviewed_by"] == "admin" + assert label["reviewed_at"] + + def test_knowledge_document_can_be_added_and_searched() -> None: client = make_client() @@ -356,6 +431,17 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: content="Search evidence first, then answer.", ) ) + repository.upsert_quality_rule_config( + AiQualityRuleConfigCreate( + min_citation_count=1, + min_source_count=1, + high_token_threshold=9000, + high_latency_ms=12000, + critical_latency_ms=24000, + warning_penalty=7, + error_penalty=21, + ) + ) batch = repository.create_knowledge_import_batch( KnowledgeImportBatchCreate( knowledge_base_id="laws-cn", @@ -436,6 +522,8 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: assert reloaded.get_default_provider_config().model == "deepseek-chat" # type: ignore[union-attr] assert reloaded.list_prompt_templates()[0].version == "v2" + assert reloaded.get_quality_rule_config().warning_penalty == 7 + assert reloaded.get_quality_rule_config().critical_latency_ms == 24000 assert reloaded.search_knowledge("lease", knowledge_base_ids=["laws-cn"])[0].reference == "Article 703" assert reloaded.list_knowledge_import_batches(knowledge_base_id="laws-cn")[0].documents_count == 1 reindex_result = reloaded.reindex_knowledge_documents(KnowledgeReindexRequest(import_batch_id=batch.id)) @@ -463,3 +551,15 @@ def test_sqlalchemy_repository_persists_platform_data(tmp_path) -> None: assert audit.quality_score == 100 assert audit.quality_labels[0].code == "citation_present" assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100." + updated_audit = reloaded.update_ai_run_quality_label( + "run-test-001", + "citation_present", + AiQualityLabelReviewUpdate( + review_status="resolved", + review_note="已确认依据有效。", + reviewed_by="admin", + ), + ) + assert updated_audit.quality_labels[0].review_status == "resolved" + assert updated_audit.quality_labels[0].review_note == "已确认依据有效。" + assert updated_audit.quality_labels[0].reviewed_at is not None diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py index 32842712..e59fe53b 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/__init__.py @@ -1,6 +1,9 @@ from yuqei_sdk.ai_platform import ( AiPlatformClient, AiQualityLabel, + AiQualityLabelReviewUpdate, + AiQualityRuleConfig, + AiQualityRuleConfigUpdate, AiRunAudit, KnowledgeChunk, KnowledgeDocument, @@ -26,6 +29,9 @@ from yuqei_sdk.context import RequestContext __all__ = [ "AiPlatformClient", "AiQualityLabel", + "AiQualityLabelReviewUpdate", + "AiQualityRuleConfig", + "AiQualityRuleConfigUpdate", "AiRunAudit", "KnowledgeChunk", "KnowledgeDocument", diff --git a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py index e8284b20..ceb39c7a 100644 --- a/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py +++ b/yuqei-ai-sdk-python/src/yuqei_sdk/ai_platform.py @@ -66,6 +66,21 @@ class PromptTemplate(PromptTemplateCreate): updated_at: str +class AiQualityRuleConfigUpdate(BaseModel): + min_citation_count: int = 1 + min_source_count: int = 2 + high_token_threshold: int = 8000 + high_latency_ms: int = 15000 + critical_latency_ms: int = 30000 + warning_penalty: int = 10 + error_penalty: int = 30 + + +class AiQualityRuleConfig(AiQualityRuleConfigUpdate): + id: str = "default" + updated_at: str + + class KnowledgeDocumentCreate(BaseModel): import_batch_id: str | None = None knowledge_base_id: str = "default" @@ -193,6 +208,16 @@ class AiQualityLabel(BaseModel): severity: str = "info" title: str detail: str + review_status: str = "pending" + review_note: str | None = None + reviewed_by: str | None = None + reviewed_at: str | None = None + + +class AiQualityLabelReviewUpdate(BaseModel): + review_status: str + review_note: str | None = None + reviewed_by: str | None = None class AiRunAudit(BaseModel): @@ -309,6 +334,32 @@ class AiPlatformClient: response.raise_for_status() return PromptTemplate.model_validate(response.json()) + def get_quality_rule_config( + self, + *, + context: RequestContext | None = None, + ) -> AiQualityRuleConfig: + response = self._client.get( + f"{self._api_prefix}/quality-rules/default", + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return AiQualityRuleConfig.model_validate(response.json()) + + def upsert_quality_rule_config( + self, + config: AiQualityRuleConfigUpdate, + *, + context: RequestContext | None = None, + ) -> AiQualityRuleConfig: + response = self._client.put( + f"{self._api_prefix}/quality-rules/default", + json=config.model_dump(exclude_none=True), + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return AiQualityRuleConfig.model_validate(response.json()) + def add_knowledge_document( self, document: KnowledgeDocumentCreate, @@ -460,3 +511,19 @@ class AiPlatformClient: ) response.raise_for_status() return [AiRunAudit.model_validate(item) for item in response.json()] + + def update_ai_run_quality_label( + self, + run_id: str, + label_code: str, + update: AiQualityLabelReviewUpdate, + *, + context: RequestContext | None = None, + ) -> AiRunAudit: + response = self._client.patch( + f"{self._api_prefix}/audit/ai-runs/{run_id}/quality-labels/{label_code}", + json=update.model_dump(exclude_none=True), + headers=(context or RequestContext()).to_headers(), + ) + response.raise_for_status() + return AiRunAudit.model_validate(response.json()) diff --git a/yuqei-ai-sdk-python/tests/test_ai_platform.py b/yuqei-ai-sdk-python/tests/test_ai_platform.py index 923161f1..6fefa048 100644 --- a/yuqei-ai-sdk-python/tests/test_ai_platform.py +++ b/yuqei-ai-sdk-python/tests/test_ai_platform.py @@ -5,6 +5,8 @@ import httpx from yuqei_sdk import ( AiPlatformClient, + AiQualityLabelReviewUpdate, + AiQualityRuleConfigUpdate, KnowledgeDocumentCreate, KnowledgeImportBatchCreate, KnowledgeReindexRequest, @@ -151,6 +153,38 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: } ], ) + if request.url.path == "/api/v1/quality-rules/default" and request.method == "PUT": + payload = json.loads(request.read().decode("utf-8")) + assert payload["min_source_count"] == 1 + return httpx.Response( + 200, + json={ + "id": "default", + "min_citation_count": 1, + "min_source_count": 1, + "high_token_threshold": 9000, + "high_latency_ms": 12000, + "critical_latency_ms": 24000, + "warning_penalty": 7, + "error_penalty": 21, + "updated_at": "2026-06-22T00:00:00Z", + }, + ) + if request.url.path == "/api/v1/quality-rules/default": + return httpx.Response( + 200, + json={ + "id": "default", + "min_citation_count": 1, + "min_source_count": 2, + "high_token_threshold": 8000, + "high_latency_ms": 15000, + "critical_latency_ms": 30000, + "warning_penalty": 10, + "error_penalty": 30, + "updated_at": "2026-06-22T00:00:00Z", + }, + ) if request.url.path == "/api/v1/knowledge-import-batches" and request.method == "POST": return httpx.Response( 200, @@ -390,6 +424,10 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: "severity": "info", "title": "Citation present", "detail": "The answer returned 1 citation.", + "review_status": "pending", + "review_note": None, + "reviewed_by": None, + "reviewed_at": None, } ], "quality_summary": "passed: 0 error(s), 0 warning(s), score 100.", @@ -398,6 +436,53 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: } ], ) + if request.url.path == "/api/v1/audit/ai-runs/run-1/quality-labels/citation_present": + payload = json.loads(request.read().decode("utf-8")) + assert request.method == "PATCH" + assert payload["review_status"] == "resolved" + return httpx.Response( + 200, + json={ + "id": "run-1", + "operation": "legal_qa", + "provider_config_id": "provider-1", + "prompt_template_id": "prompt-1", + "provider": "deepseek", + "model": "deepseek-chat", + "provider_status": "succeeded", + "provider_error": None, + "prompt_preview": "Search evidence first, then answer. Question: How should lease deposit refund be handled?", + "question": "How should lease deposit refund be handled?", + "answer": "The contract should clearly state deposit refund conditions.", + "citations_count": 1, + "retrieval_query": "How should lease deposit refund be handled?", + "retrieval_strategy": "hybrid", + "retrieved_chunk_ids": ["chunk-1"], + "retrieved_sources": [], + "prompt_tokens": 12, + "completion_tokens": 8, + "total_tokens": 20, + "estimated_cost": 0.00028, + "latency_ms": 12, + "quality_status": "passed", + "quality_score": 100, + "quality_labels": [ + { + "code": "citation_present", + "severity": "info", + "title": "Citation present", + "detail": "The answer returned 1 citation.", + "review_status": "resolved", + "review_note": "已确认依据有效。", + "reviewed_by": "admin", + "reviewed_at": "2026-06-22T00:01:00Z", + } + ], + "quality_summary": "passed: 0 error(s), 0 warning(s), score 100.", + "status": "succeeded", + "created_at": "2026-06-22T00:00:00Z", + }, + ) return httpx.Response(404) client = AiPlatformClient("http://ai-platform.test", transport=httpx.MockTransport(handler)) @@ -416,6 +501,18 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: PromptTemplateCreate(name="legal_qa.default", version="v2", content="Search evidence first.") ).version == "v2" assert client.list_prompt_templates()[0].name == "legal_qa.default" + assert client.get_quality_rule_config().min_source_count == 2 + assert client.upsert_quality_rule_config( + AiQualityRuleConfigUpdate( + min_citation_count=1, + min_source_count=1, + high_token_threshold=9000, + high_latency_ms=12000, + critical_latency_ms=24000, + warning_penalty=7, + error_penalty=21, + ) + ).warning_penalty == 7 assert client.create_knowledge_import_batch( KnowledgeImportBatchCreate( knowledge_base_id="laws-cn", @@ -466,5 +563,18 @@ def test_ai_platform_client_covers_admin_data_endpoints() -> None: assert audit.quality_status == "passed" assert audit.quality_score == 100 assert audit.quality_labels[0].code == "citation_present" + assert audit.quality_labels[0].review_status == "pending" assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100." + updated_audit = client.update_ai_run_quality_label( + "run-1", + "citation_present", + AiQualityLabelReviewUpdate( + review_status="resolved", + review_note="已确认依据有效。", + reviewed_by="admin", + ), + ) + assert updated_audit.quality_labels[0].review_status == "resolved" + assert updated_audit.quality_labels[0].reviewed_at == "2026-06-22T00:01:00Z" assert "/api/v1/audit/ai-runs" in seen_paths + assert "/api/v1/quality-rules/default" in seen_paths diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality/page.tsx new file mode 100644 index 00000000..ebcf7e97 --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-quality/page.tsx @@ -0,0 +1,204 @@ +"use client"; + +import Link from "next/link"; +import { useEffect, useMemo, useState } from "react"; +import { RefreshCcw, Save, SlidersHorizontal } from "lucide-react"; + +type QualityRuleConfig = { + id?: string; + min_citation_count: number; + min_source_count: number; + high_token_threshold: number; + high_latency_ms: number; + critical_latency_ms: number; + warning_penalty: number; + error_penalty: number; + updated_at?: string; +}; + +type LoadState = + | { status: "loading" } + | { status: "ready"; config: QualityRuleConfig } + | { status: "error"; message: string }; + +const defaultConfig: QualityRuleConfig = { + min_citation_count: 1, + min_source_count: 2, + high_token_threshold: 8000, + high_latency_ms: 15000, + critical_latency_ms: 30000, + warning_penalty: 10, + error_penalty: 30 +}; + +const fields: Array<{ + key: keyof QualityRuleConfig; + label: string; + suffix: string; + min: number; + max: number; +}> = [ + { key: "min_citation_count", label: "最少引用依据", suffix: "条", min: 0, max: 10 }, + { key: "min_source_count", label: "最少知识来源", suffix: "个", min: 0, max: 10 }, + { key: "high_token_threshold", label: "Token 复核阈值", suffix: "tokens", min: 1, max: 1000000 }, + { key: "high_latency_ms", label: "耗时偏高阈值", suffix: "ms", min: 1, max: 600000 }, + { key: "critical_latency_ms", label: "耗时严重阈值", suffix: "ms", min: 1, max: 600000 }, + { key: "warning_penalty", label: "Warning 扣分", suffix: "分", min: 0, max: 100 }, + { key: "error_penalty", label: "Error 扣分", suffix: "分", min: 0, max: 100 } +]; + +export default function AiQualityPage() { + const [state, setState] = useState({ status: "loading" }); + const [draft, setDraft] = useState(defaultConfig); + const [saving, setSaving] = useState(false); + const [message, setMessage] = useState(null); + + async function loadConfig() { + setState({ status: "loading" }); + setMessage(null); + try { + const response = await fetch("/api/ai-platform/quality-rules/default", { cache: "no-store" }); + const payload = await response.json(); + if (!response.ok) { + throw new Error(payload?.message ?? `HTTP ${response.status}`); + } + setDraft(payload); + setState({ status: "ready", config: payload }); + } catch (error) { + setState({ + status: "error", + message: error instanceof Error ? error.message : "AI 质量规则加载失败" + }); + } + } + + useEffect(() => { + void loadConfig(); + }, []); + + const validationMessage = useMemo(() => { + if (draft.critical_latency_ms < draft.high_latency_ms) { + return "耗时严重阈值不能小于耗时偏高阈值。"; + } + return null; + }, [draft.critical_latency_ms, draft.high_latency_ms]); + + async function saveConfig() { + if (validationMessage) { + setMessage(validationMessage); + return; + } + setSaving(true); + setMessage(null); + try { + const response = await fetch("/api/ai-platform/quality-rules/default", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(draft) + }); + const payload = await response.json(); + if (!response.ok) { + throw new Error(payload?.message ?? `HTTP ${response.status}`); + } + setDraft(payload); + setState({ status: "ready", config: payload }); + setMessage("已保存质量评测规则。"); + } catch (error) { + setMessage(error instanceof Error ? error.message : "AI 质量规则保存失败"); + } finally { + setSaving(false); + } + } + + function updateNumber(key: keyof QualityRuleConfig, value: string) { + const numericValue = Number(value); + setDraft((current) => ({ + ...current, + [key]: Number.isFinite(numericValue) ? numericValue : 0 + })); + } + + return ( + <> +
+
+

AI 质量规则

+

配置回答依据、来源覆盖、token、耗时和扣分阈值。

+
+
+ + +
+
+ +
+
+

+ + 默认规则 +

+ + 查看 Trace + +
+
+ {state.status === "loading" &&
正在加载 AI 质量规则...
} + {state.status === "error" && ( +
+ 无法连接 AI Platform 质量规则接口 + {state.message} +
+ )} + {state.status === "ready" && ( +
+ {fields.map((field) => ( + + ))} +
+ )} + {(message || validationMessage) && ( +
+ {validationMessage || message} +
+ )} + {state.status === "ready" && draft.updated_at && ( +

最后更新：{formatDate(draft.updated_at)}

+ )} +
+
+ + ); +} + +function formatDate(value: string) { + const date = new Date(value); + if (Number.isNaN(date.getTime())) { + return value; + } + return new Intl.DateTimeFormat("zh-CN", { + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit" + }).format(date); +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx index f0c6e035..5458cf15 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/ai-traces/page.tsx @@ -35,6 +35,10 @@ type AiQualityLabel = { severity?: string; title: string; detail: string; + review_status?: string; + review_note?: string | null; + reviewed_by?: string | null; + reviewed_at?: string | null; }; type AiRunAudit = { @@ -130,6 +134,20 @@ const severityTone: Record = { error: "tag-danger" }; +const reviewStatusLabels: Record = { + pending: "待处理", + reviewed: "已复核", + false_positive: "误报", + resolved: "已处理" +}; + +const reviewStatusTone: Record = { + pending: "", + reviewed: "tag-primary", + false_positive: "tag-warning", + resolved: "tag-success" +}; + export default function AiTracesPage() { const [state, setState] = useState({ status: "loading" }); const [statusFilter, setStatusFilter] = useState("all"); @@ -157,6 +175,18 @@ export default function AiTracesPage() { void loadRuns(); }, []); + function updateRun(updatedRun: AiRunAudit) { + setState((current) => { + if (current.status !== "ready") { + return current; + } + return { + status: "ready", + runs: current.runs.map((run) => (run.id === updatedRun.id ? updatedRun : run)) + }; + }); + } + const runs = state.status === "ready" ? state.runs : []; const filteredRuns = useMemo(() => { const normalizedKeyword = keyword.trim().toLowerCase(); @@ -175,7 +205,9 @@ export default function AiTracesPage() { run.retrieval_query, run.quality_status, run.quality_summary, - ...(run.quality_labels ?? []).map((label) => `${label.code} ${label.title} ${label.detail}`), + ...(run.quality_labels ?? []).map((label) => ( + `${label.code} ${label.title} ${label.detail} ${label.review_status ?? ""} ${label.review_note ?? ""}` + )), ...(run.retrieved_sources ?? []).map((source) => `${source.title} ${source.reference}`) ].join("\n").toLowerCase(); return matchesStatus && matchesQuality && (!normalizedKeyword || haystack.includes(normalizedKeyword)); @@ -269,7 +301,7 @@ export default function AiTracesPage() { {state.status === "ready" && filteredRuns.length > 0 && (
{filteredRuns.map((run) => ( - + ))}
)} @@ -289,12 +321,42 @@ function Metric({ icon, label, value }: { icon: ReactNode; label: string; value: ); } -function TraceItem({ run }: { run: AiRunAudit }) { +function TraceItem({ run, onRunUpdate }: { run: AiRunAudit; onRunUpdate: (run: AiRunAudit) => void }) { const statusClass = statusTone[run.status] ?? ""; const providerStatusClass = statusTone[run.provider_status ?? ""] ?? ""; const qualityStatus = run.quality_status ?? "unknown"; const qualityClass = qualityStatusTone[qualityStatus] ?? ""; const qualityLabels = run.quality_labels ?? []; + const [updatingLabelCode, setUpdatingLabelCode] = useState(null); + const [labelUpdateError, setLabelUpdateError] = useState(null); + + async function updateLabelReview(label: AiQualityLabel, reviewStatus: string) { + setUpdatingLabelCode(label.code); + setLabelUpdateError(null); + try { + const response = await fetch( + `/api/ai-platform/audit/ai-runs/${encodeURIComponent(run.id)}/quality-labels/${encodeURIComponent(label.code)}`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + review_status: reviewStatus, + review_note: reviewStatus === "pending" ? null : reviewStatusLabels[reviewStatus], + reviewed_by: reviewStatus === "pending" ? null : "admin" + }) + } + ); + const payload = await response.json(); + if (!response.ok) { + throw new Error(payload?.message ?? `HTTP ${response.status}`); + } + onRunUpdate(payload); + } catch (error) { + setLabelUpdateError(error instanceof Error ? error.message : "质量标签状态更新失败"); + } finally { + setUpdatingLabelCode(null); + } + } return (
@@ -367,13 +429,33 @@ function TraceItem({ run }: { run: AiRunAudit }) { const severityClass = severityTone[label.severity ?? "info"] ?? ""; return (
- {text?.title ?? label.title} - {text?.detail ?? label.detail} +
+ {text?.title ?? label.title} + {text?.detail ?? label.detail} +
+
+ + {reviewStatusLabels[label.review_status ?? "pending"] ?? label.review_status} + + +
+ {label.review_note && {label.review_note}}
); })} )} + {labelUpdateError &&
{labelUpdateError}
}

运行状态

diff --git a/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx b/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx index d0907271..65b95692 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx +++ b/yuqei-contract-platform/apps/contract-web/src/app/admin/page.tsx @@ -17,9 +17,9 @@ const adminSections: AdminSection[] = [ }, { title: "AI 参数", - description: "模型连接、运行参数、提示词版本和审计策略。", + description: "模型运行、质量规则、提示词版本和审计策略。", icon: SlidersHorizontal, - href: "/admin/ai-traces" + href: "/admin/ai-quality" }, { title: "AI 调用 Trace", diff --git a/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/ai-runs/[runId]/quality-labels/[labelCode]/route.ts b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/ai-runs/[runId]/quality-labels/[labelCode]/route.ts new file mode 100644 index 00000000..ce6fd3d9 --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/audit/ai-runs/[runId]/quality-labels/[labelCode]/route.ts @@ -0,0 +1,47 @@ +import { NextResponse } from "next/server"; + +const aiPlatformBaseUrl = + process.env.AI_PLATFORM_API_BASE_URL ?? + process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ?? + "http://localhost:8101"; + +export async function PATCH( + request: Request, + context: { params: Promise<{ runId: string; labelCode: string }> } +) { + const { runId, labelCode } = await context.params; + const targetUrl = new URL( + `/api/v1/audit/ai-runs/${encodeURIComponent(runId)}/quality-labels/${encodeURIComponent(labelCode)}`, + aiPlatformBaseUrl + ); + const body: unknown = await request.json().catch(() => null); + + try { + const response = await fetch(targetUrl, { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + cache: "no-store" + }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform audit quality label service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform audit service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/quality-rules/default/route.ts b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/quality-rules/default/route.ts new file mode 100644 index 00000000..3c9c581e --- /dev/null +++ b/yuqei-contract-platform/apps/contract-web/src/app/api/ai-platform/quality-rules/default/route.ts @@ -0,0 +1,68 @@ +import { NextResponse } from "next/server"; + +const aiPlatformBaseUrl = + process.env.AI_PLATFORM_API_BASE_URL ?? + process.env.NEXT_PUBLIC_AI_PLATFORM_API_BASE_URL ?? + "http://localhost:8101"; + +export async function GET() { + const targetUrl = new URL("/api/v1/quality-rules/default", aiPlatformBaseUrl); + + try { + const response = await fetch(targetUrl, { cache: "no-store" }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform quality rule service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform quality rule service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} + +export async function PUT(request: Request) { + const targetUrl = new URL("/api/v1/quality-rules/default", aiPlatformBaseUrl); + const body: unknown = await request.json().catch(() => null); + + try { + const response = await fetch(targetUrl, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + cache: "no-store" + }); + const payload: unknown = await response.json().catch(() => null); + if (!response.ok) { + return NextResponse.json( + { + message: "AI Platform quality rule service returned an error.", + status: response.status, + payload + }, + { status: response.status } + ); + } + return NextResponse.json(payload); + } catch (error) { + return NextResponse.json( + { + message: error instanceof Error ? error.message : "AI Platform quality rule service is unavailable.", + baseUrl: aiPlatformBaseUrl + }, + { status: 502 } + ); + } +} diff --git a/yuqei-contract-platform/apps/contract-web/src/app/globals.css b/yuqei-contract-platform/apps/contract-web/src/app/globals.css index ae524e9a..52186bc1 100644 --- a/yuqei-contract-platform/apps/contract-web/src/app/globals.css +++ b/yuqei-contract-platform/apps/contract-web/src/app/globals.css @@ -220,6 +220,9 @@ select { } .panel-title { + display: flex; + align-items: center; + gap: 8px; margin: 0; font-size: 16px; font-weight: 750; @@ -315,6 +318,10 @@ select { margin-bottom: 14px; } +.toolbar-inline { + margin-bottom: 0; +} + .search-input { min-width: min(100%, 360px); min-height: 38px; @@ -354,6 +361,11 @@ select { min-width: 160px; } +.select-input-compact { + min-width: 116px; + min-height: 32px; +} + .tag { display: inline-flex; align-items: center; @@ -617,7 +629,7 @@ select { .quality-label { display: grid; - grid-template-columns: max-content minmax(0, 1fr); + grid-template-columns: minmax(0, 1fr) max-content; gap: 8px 10px; align-items: center; color: var(--muted); @@ -625,6 +637,58 @@ select { line-height: 1.5; } +.quality-label-main, +.quality-review-control, +.setting-input-row { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + min-width: 0; +} + +.quality-review-control { + justify-content: flex-end; +} + +.quality-review-note { + grid-column: 1 / -1; +} + +.settings-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 14px; +} + +.setting-field { + display: grid; + gap: 8px; + color: var(--muted); + font-size: 13px; +} + +.setting-field .search-input { + min-width: 0; + width: 100%; +} + +.inline-message { + margin-top: 14px; + padding: 10px 12px; + border: 1px solid var(--line); + border-radius: 8px; + color: var(--primary); + background: var(--primary-soft); + font-size: 13px; +} + +.inline-message-error { + color: var(--danger); + border-color: #f2b8b8; + background: var(--danger-soft); +} + .compact-list-item { padding: 12px; } @@ -655,10 +719,15 @@ details.panel summary::-webkit-details-marker { .split-layout, .detail-grid, .grid-two, - .trace-detail-grid { + .trace-detail-grid, + .quality-label { grid-template-columns: 1fr; } + .quality-review-control { + justify-content: flex-start; + } + .metric-row, .flow-strip { grid-template-columns: repeat(2, minmax(0, 1fr));