From 124e45df8c0ff1a1c0640d65599510630ea2a085 Mon Sep 17 00:00:00 2001 From: yuqei Date: Thu, 21 May 2026 18:15:17 +0800 Subject: [PATCH] feat: track prompt evaluation runs --- backend/app/models/__init__.py | 1 + .../app/models/prompt_asset_evaluation_run.py | 36 ++++++++++ .../prompt_asset_evaluation_run_repository.py | 37 ++++++++++ backend/app/services/prompt_asset_service.py | 72 +++++++++++++++++++ ...0521_0038_create_prompt_evaluation_runs.py | 66 +++++++++++++++++ backend/tests/test_prompt_asset_service.py | 25 +++++++ backend/tests/test_prompt_assets_api.py | 45 ++++++++++++ frontend/src/app/prompt-assets/page.tsx | 53 ++++++++++++++ frontend/src/types/api.ts | 24 +++++++ ...µ·è‰ä¸Žå®¡æŸ¥åŠ©æ‰‹-当å‰å·¥ä½œè¿›åº¦ä¿å­˜-2026-05-11.md | 30 ++++++++ 10 files changed, 389 insertions(+) create mode 100644 backend/app/models/prompt_asset_evaluation_run.py create mode 100644 backend/app/repositories/prompt_asset_evaluation_run_repository.py create mode 100644 backend/migrations/versions/20260521_0038_create_prompt_evaluation_runs.py diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index c3b2c434..f844e29b 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -25,6 +25,7 @@ from app.models.legal_document_type import LegalDocumentType from app.models.message import Message from app.models.pilot_request import PilotRequest from app.models.prompt_asset import PromptAsset +from app.models.prompt_asset_evaluation_run import PromptAssetEvaluationRun from app.models.prompt_asset_run_link import PromptAssetRunLink from app.models.prompt_asset_version import PromptAssetVersion from app.models.review_result import ReviewResult diff --git a/backend/app/models/prompt_asset_evaluation_run.py b/backend/app/models/prompt_asset_evaluation_run.py new file mode 100644 index 00000000..f52146a9 --- /dev/null +++ b/backend/app/models/prompt_asset_evaluation_run.py @@ -0,0 +1,36 @@ +from datetime import datetime, timezone +from typing import Any, Optional + +from sqlalchemy import DateTime, Float, ForeignKey, Index, JSON, String +from sqlalchemy.orm import Mapped, mapped_column + +from app.db import Base + + +class PromptAssetEvaluationRun(Base): + __tablename__ = "prompt_asset_evaluation_runs" + __table_args__ = ( + Index("ix_prompt_asset_evaluation_runs_tenant_asset", "tenant_id", "prompt_asset_id"), + Index("ix_prompt_asset_evaluation_runs_tenant_status", "tenant_id", "status"), + Index("ix_prompt_asset_evaluation_runs_created_at", "created_at"), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + tenant_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + prompt_asset_id: Mapped[str] = mapped_column(String(64), ForeignKey("prompt_assets.id"), nullable=False, index=True) + prompt_asset_version_id: Mapped[str] = mapped_column(String(64), ForeignKey("prompt_asset_versions.id"), nullable=False, index=True) + prompt_key: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + prompt_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + suite_key: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(32), nullable=False, default="running") + sample_count: Mapped[int] = mapped_column(default=0) + passed_samples: Mapped[int] = mapped_column(default=0) + failed_samples: Mapped[int] = mapped_column(default=0) + pass_rate: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + quality_score: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + report_json: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON, nullable=True) + failure_reason: Mapped[Optional[str]] = mapped_column(String(500), nullable=True) + triggered_by: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) diff --git a/backend/app/repositories/prompt_asset_evaluation_run_repository.py b/backend/app/repositories/prompt_asset_evaluation_run_repository.py new file mode 100644 index 00000000..b2db1299 --- /dev/null +++ b/backend/app/repositories/prompt_asset_evaluation_run_repository.py @@ -0,0 +1,37 @@ +from typing import List, Optional + +from sqlalchemy.orm import Session + +from app.models.prompt_asset_evaluation_run import PromptAssetEvaluationRun + + +class PromptAssetEvaluationRunRepository: + def __init__(self, db: Session): + self.db = db + + def create(self, evaluation_run: PromptAssetEvaluationRun) -> PromptAssetEvaluationRun: + self.db.add(evaluation_run) + self.db.commit() + self.db.refresh(evaluation_run) + return evaluation_run + + def update(self, evaluation_run: PromptAssetEvaluationRun) -> PromptAssetEvaluationRun: + self.db.add(evaluation_run) + self.db.commit() + self.db.refresh(evaluation_run) + return evaluation_run + + def get_by_id(self, evaluation_run_id: str) -> Optional[PromptAssetEvaluationRun]: + return self.db.get(PromptAssetEvaluationRun, evaluation_run_id) + + def list_by_asset_id(self, tenant_id: str, asset_id: str, limit: int = 20) -> List[PromptAssetEvaluationRun]: + return ( + self.db.query(PromptAssetEvaluationRun) + .filter( + PromptAssetEvaluationRun.tenant_id == tenant_id, + PromptAssetEvaluationRun.prompt_asset_id == asset_id, + ) + .order_by(PromptAssetEvaluationRun.created_at.desc()) + .limit(limit) + .all() + ) diff --git a/backend/app/services/prompt_asset_service.py b/backend/app/services/prompt_asset_service.py index 75c7b6d3..7483e9c7 100644 --- a/backend/app/services/prompt_asset_service.py +++ b/backend/app/services/prompt_asset_service.py @@ -7,8 +7,10 @@ from fastapi import HTTPException, status from sqlalchemy.orm import Session from app.models.prompt_asset import PromptAsset +from app.models.prompt_asset_evaluation_run import PromptAssetEvaluationRun from app.models.prompt_asset_run_link import PromptAssetRunLink from app.models.prompt_asset_version import PromptAssetVersion +from app.repositories.prompt_asset_evaluation_run_repository import PromptAssetEvaluationRunRepository from app.repositories.prompt_asset_repository import PromptAssetRepository from app.repositories.prompt_asset_run_link_repository import PromptAssetRunLinkRepository from app.repositories.prompt_asset_version_repository import PromptAssetVersionRepository @@ -39,6 +41,7 @@ class PromptAssetService: self.repository = PromptAssetRepository(db) self.version_repository = PromptAssetVersionRepository(db) self.run_link_repository = PromptAssetRunLinkRepository(db) + self.evaluation_run_repository = PromptAssetEvaluationRunRepository(db) def list_prompt_assets( self, @@ -73,10 +76,12 @@ class PromptAssetService: asset = self._require_asset(current_user, asset_id) versions = self.version_repository.list_by_asset_id(current_user["tenant_id"], asset.id) run_links = self.run_link_repository.list_by_asset_id(current_user["tenant_id"], asset.id, limit=20) + evaluation_runs = self.evaluation_run_repository.list_by_asset_id(current_user["tenant_id"], asset.id, limit=20) return { "asset": self._to_item(asset), "versions": [self._to_version_item(item) for item in versions], "run_links": [self._to_run_link_item(item) for item in run_links], + "evaluation_runs": [self._to_evaluation_run_item(item) for item in evaluation_runs], } def create_prompt_asset( @@ -190,6 +195,25 @@ class PromptAssetService: self._require_admin(current_user) asset = self._require_asset(current_user, asset_id) version = self._require_current_version(current_user, asset) + now = self._now() + evaluation_run = self.evaluation_run_repository.create( + PromptAssetEvaluationRun( + id=f"prompteval-{uuid4().hex[:12]}", + tenant_id=asset.tenant_id, + prompt_asset_id=asset.id, + prompt_asset_version_id=version.id, + prompt_key=asset.prompt_key, + prompt_type=asset.prompt_type, + suite_key=self._suite_key_for_prompt_type(asset.prompt_type), + status="running", + sample_count=0, + passed_samples=0, + failed_samples=0, + triggered_by=current_user.get("id"), + started_at=now, + created_at=now, + ) + ) report = self._build_prompt_evaluation_report(asset, version) version.evaluation_report_json = report @@ -197,6 +221,17 @@ class PromptAssetService: version.evaluation_status = "passed" if report["passed"] else "failed" version = self.version_repository.update(version) + evaluation_run.status = version.evaluation_status + evaluation_run.sample_count = int(report.get("sample_count") or 0) + evaluation_run.passed_samples = int(report.get("passed_samples") or 0) + evaluation_run.failed_samples = int(report.get("failed_samples") or 0) + evaluation_run.pass_rate = float(report["pass_rate"]) if report.get("pass_rate") is not None else None + evaluation_run.quality_score = version.evaluation_score + evaluation_run.report_json = report + evaluation_run.failure_reason = self._build_evaluation_failure_reason(report) + evaluation_run.finished_at = self._now() + evaluation_run = self.evaluation_run_repository.update(evaluation_run) + asset.updated_by = current_user.get("id") asset.updated_at = self._now() self._copy_evaluation_to_asset(asset, version) @@ -205,6 +240,7 @@ class PromptAssetService: "asset": self._to_item(asset), "version": self._to_version_item(version), "evaluation_report": report, + "evaluation_run": self._to_evaluation_run_item(evaluation_run), } def publish_prompt_asset( @@ -471,6 +507,19 @@ class PromptAssetService: "source": "prompt_assets.evaluate_current_version", } + def _suite_key_for_prompt_type(self, prompt_type: str) -> str: + return str(self.PROMPT_GOLDEN_SUITES.get(prompt_type, {"suite_key": "custom_prompt_suite"})["suite_key"]) + + def _build_evaluation_failure_reason(self, report: dict[str, Any]) -> Optional[str]: + if report.get("passed") is True: + return None + failed_items = report.get("failed_items") + if isinstance(failed_items, list) and failed_items: + first = failed_items[0] + if isinstance(first, dict): + return str(first.get("label") or first.get("key") or "prompt evaluation failed")[:500] + return "prompt evaluation failed" + def _build_prompt_quality_checks(self, asset: PromptAsset, version: PromptAssetVersion) -> list[dict[str, Any]]: content = f"{version.system_prompt or ''}\n{version.prompt_content or ''}".strip() normalized = content.lower() @@ -711,3 +760,26 @@ class PromptAssetService: "created_by": run_link.created_by, "created_at": run_link.created_at.isoformat(), } + + def _to_evaluation_run_item(self, evaluation_run: PromptAssetEvaluationRun) -> dict: + return { + "id": evaluation_run.id, + "tenant_id": evaluation_run.tenant_id, + "prompt_asset_id": evaluation_run.prompt_asset_id, + "prompt_asset_version_id": evaluation_run.prompt_asset_version_id, + "prompt_key": evaluation_run.prompt_key, + "prompt_type": evaluation_run.prompt_type, + "suite_key": evaluation_run.suite_key, + "status": evaluation_run.status, + "sample_count": evaluation_run.sample_count, + "passed_samples": evaluation_run.passed_samples, + "failed_samples": evaluation_run.failed_samples, + "pass_rate": evaluation_run.pass_rate, + "quality_score": evaluation_run.quality_score, + "report": evaluation_run.report_json, + "failure_reason": evaluation_run.failure_reason, + "triggered_by": evaluation_run.triggered_by, + "started_at": evaluation_run.started_at.isoformat(), + "finished_at": evaluation_run.finished_at.isoformat() if evaluation_run.finished_at else None, + "created_at": evaluation_run.created_at.isoformat(), + } diff --git a/backend/migrations/versions/20260521_0038_create_prompt_evaluation_runs.py b/backend/migrations/versions/20260521_0038_create_prompt_evaluation_runs.py new file mode 100644 index 00000000..25cba0bf --- /dev/null +++ b/backend/migrations/versions/20260521_0038_create_prompt_evaluation_runs.py @@ -0,0 +1,66 @@ +"""create prompt evaluation runs""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = "20260521_0038" +down_revision = "20260521_0037" +branch_labels = None +depends_on = None + + +def _table_names(inspector: sa.Inspector) -> set[str]: + return set(inspector.get_table_names()) + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "prompt_asset_evaluation_runs" in _table_names(inspector): + return + + op.create_table( + "prompt_asset_evaluation_runs", + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("tenant_id", sa.String(length=64), nullable=False), + sa.Column("prompt_asset_id", sa.String(length=64), nullable=False), + sa.Column("prompt_asset_version_id", sa.String(length=64), nullable=False), + sa.Column("prompt_key", sa.String(length=100), nullable=False), + sa.Column("prompt_type", sa.String(length=64), nullable=False), + sa.Column("suite_key", sa.String(length=100), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False, server_default="running"), + sa.Column("sample_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("passed_samples", sa.Integer(), nullable=False, server_default="0"), + sa.Column("failed_samples", sa.Integer(), nullable=False, server_default="0"), + sa.Column("pass_rate", sa.Float(), nullable=True), + sa.Column("quality_score", sa.Float(), nullable=True), + sa.Column("report_json", sa.JSON(), nullable=True), + sa.Column("failure_reason", sa.String(length=500), nullable=True), + sa.Column("triggered_by", sa.String(length=64), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["prompt_asset_id"], ["prompt_assets.id"]), + sa.ForeignKeyConstraint(["prompt_asset_version_id"], ["prompt_asset_versions.id"]), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_prompt_asset_evaluation_runs_created_at", "prompt_asset_evaluation_runs", ["created_at"]) + op.create_index("ix_prompt_asset_evaluation_runs_prompt_asset_id", "prompt_asset_evaluation_runs", ["prompt_asset_id"]) + op.create_index("ix_prompt_asset_evaluation_runs_prompt_asset_version_id", "prompt_asset_evaluation_runs", ["prompt_asset_version_id"]) + op.create_index("ix_prompt_asset_evaluation_runs_prompt_key", "prompt_asset_evaluation_runs", ["prompt_key"]) + op.create_index("ix_prompt_asset_evaluation_runs_prompt_type", "prompt_asset_evaluation_runs", ["prompt_type"]) + op.create_index("ix_prompt_asset_evaluation_runs_suite_key", "prompt_asset_evaluation_runs", ["suite_key"]) + op.create_index("ix_prompt_asset_evaluation_runs_tenant_asset", "prompt_asset_evaluation_runs", ["tenant_id", "prompt_asset_id"]) + op.create_index("ix_prompt_asset_evaluation_runs_tenant_id", "prompt_asset_evaluation_runs", ["tenant_id"]) + op.create_index("ix_prompt_asset_evaluation_runs_tenant_status", "prompt_asset_evaluation_runs", ["tenant_id", "status"]) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "prompt_asset_evaluation_runs" not in _table_names(inspector): + return + op.drop_table("prompt_asset_evaluation_runs") diff --git a/backend/tests/test_prompt_asset_service.py b/backend/tests/test_prompt_asset_service.py index 7744b36f..2da5a67f 100644 --- a/backend/tests/test_prompt_asset_service.py +++ b/backend/tests/test_prompt_asset_service.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone from fastapi import HTTPException from app.models.prompt_asset import PromptAsset +from app.models.prompt_asset_evaluation_run import PromptAssetEvaluationRun from app.models.prompt_asset_run_link import PromptAssetRunLink from app.models.prompt_asset_version import PromptAssetVersion from app.services.prompt_asset_service import PromptAssetService @@ -95,12 +96,34 @@ class StubPromptAssetRunLinkRepository: return [item for item in self.items.values() if item.tenant_id == tenant_id and item.prompt_asset_id == asset_id][:limit] +class StubPromptAssetEvaluationRunRepository: + def __init__(self): + self.items = {} + + def create(self, evaluation_run): + evaluation_run.started_at = evaluation_run.started_at or datetime.now(timezone.utc) + evaluation_run.created_at = evaluation_run.created_at or datetime.now(timezone.utc) + self.items[evaluation_run.id] = evaluation_run + return evaluation_run + + def update(self, evaluation_run): + self.items[evaluation_run.id] = evaluation_run + return evaluation_run + + def get_by_id(self, evaluation_run_id): + return self.items.get(evaluation_run_id) + + def list_by_asset_id(self, tenant_id, asset_id, limit=20): + return [item for item in self.items.values() if item.tenant_id == tenant_id and item.prompt_asset_id == asset_id][:limit] + + class PromptAssetServiceTestCase(unittest.TestCase): def setUp(self): self.service = PromptAssetService(db=None) self.service.repository = StubPromptAssetRepository() self.service.version_repository = StubPromptAssetVersionRepository() self.service.run_link_repository = StubPromptAssetRunLinkRepository() + self.service.evaluation_run_repository = StubPromptAssetEvaluationRunRepository() self.admin_user = {"id": "user-admin", "tenant_id": "tenant-demo-001", "role": "admin", "display_name": "Admin"} self.member_user = {"id": "user-member", "tenant_id": "tenant-demo-001", "role": "member"} @@ -145,6 +168,8 @@ class PromptAssetServiceTestCase(unittest.TestCase): self.assertEqual(evaluation["asset"]["latest_evaluation_status"], "passed") self.assertEqual(evaluation["version"]["evaluation_status"], "passed") self.assertEqual(evaluation["evaluation_report"]["suite_key"], "phase13_generation") + self.assertEqual(evaluation["evaluation_run"]["status"], "passed") + self.assertEqual(evaluation["evaluation_run"]["sample_count"], 8) self.assertEqual(published["status"], "published") def test_publish_with_report_and_create_next_version(self): diff --git a/backend/tests/test_prompt_assets_api.py b/backend/tests/test_prompt_assets_api.py index be1008ec..209edf44 100644 --- a/backend/tests/test_prompt_assets_api.py +++ b/backend/tests/test_prompt_assets_api.py @@ -92,6 +92,29 @@ class StubPromptAssetService: } ], "run_links": [], + "evaluation_runs": [ + { + "id": "prompteval-123", + "tenant_id": "tenant-demo-001", + "prompt_asset_id": asset_id, + "prompt_asset_version_id": "promptver-123", + "prompt_key": "contract_generation_default", + "prompt_type": "contract_generation", + "suite_key": "phase13_generation", + "status": "passed", + "sample_count": 8, + "passed_samples": 8, + "failed_samples": 0, + "pass_rate": 1.0, + "quality_score": 0.95, + "report": {"suite_key": "phase13_generation", "passed": True}, + "failure_reason": None, + "triggered_by": "user-admin", + "started_at": "2026-05-21T00:00:00+00:00", + "finished_at": "2026-05-21T00:00:01+00:00", + "created_at": "2026-05-21T00:00:00+00:00", + } + ], } def create_prompt_asset(self, current_user, prompt_key, prompt_type, name, prompt_content, **kwargs): @@ -132,6 +155,27 @@ class StubPromptAssetService: "published_by": None, }, "evaluation_report": {"suite_key": "phase13_generation", "passed": True, "quality_score": 0.95}, + "evaluation_run": { + "id": "prompteval-123", + "tenant_id": "tenant-demo-001", + "prompt_asset_id": asset_id, + "prompt_asset_version_id": "promptver-123", + "prompt_key": "contract_generation_default", + "prompt_type": "contract_generation", + "suite_key": "phase13_generation", + "status": "passed", + "sample_count": 8, + "passed_samples": 8, + "failed_samples": 0, + "pass_rate": 1.0, + "quality_score": 0.95, + "report": {"suite_key": "phase13_generation", "passed": True}, + "failure_reason": None, + "triggered_by": current_user["id"], + "started_at": "2026-05-21T00:00:00+00:00", + "finished_at": "2026-05-21T00:00:01+00:00", + "created_at": "2026-05-21T00:00:00+00:00", + }, } def publish_prompt_asset(self, current_user, asset_id, **kwargs): @@ -219,6 +263,7 @@ class PromptAssetsApiTestCase(unittest.TestCase): self.assertEqual(create_response.status_code, 200) self.assertEqual(version_response.json()["data"]["current_version_label"], "v1.1") self.assertEqual(evaluation_response.json()["data"]["version"]["evaluation_status"], "passed") + self.assertEqual(evaluation_response.json()["data"]["evaluation_run"]["status"], "passed") self.assertEqual(publish_response.json()["data"]["status"], "published") self.assertEqual(rollback_response.json()["data"]["current_version_label"], "v1.2") self.assertEqual(run_link_response.json()["data"]["prompt_asset_version_id"], "promptver-123") diff --git a/frontend/src/app/prompt-assets/page.tsx b/frontend/src/app/prompt-assets/page.tsx index 9f4b9ff8..af304dc2 100644 --- a/frontend/src/app/prompt-assets/page.tsx +++ b/frontend/src/app/prompt-assets/page.tsx @@ -28,6 +28,7 @@ import { import { useAuthStore } from "@/stores/auth.store"; import type { PromptAssetDetailResponse, + PromptAssetEvaluationRunItem, PromptAssetItem, PromptAssetRunLinkPayload, PromptAssetType, @@ -535,6 +536,47 @@ export default function PromptAssetsPage() { }, ]; + const evaluationRunColumns: ColumnsType = [ + { + title: "套件", + dataIndex: "suite_key", + key: "suite_key", + width: 170, + }, + { + title: "状æ€", + key: "status", + width: 130, + render: (_, item) => getEvaluationTag(item.status), + }, + { + title: "样本", + key: "samples", + width: 130, + render: (_, item) => `${item.passed_samples}/${item.sample_count}`, + }, + { + title: "分数", + key: "score", + width: 100, + render: (_, item) => item.quality_score ?? "-", + }, + { + title: "失败原因", + dataIndex: "failure_reason", + key: "failure_reason", + ellipsis: true, + render: (value: string | null) => value ?? "-", + }, + { + title: "完æˆæ—¶é—´", + dataIndex: "finished_at", + key: "finished_at", + width: 180, + render: (value: string | null) => formatDateTime(value), + }, + ]; + if (currentUser?.role !== "admin") { return (
@@ -685,6 +727,17 @@ export default function PromptAssetsPage() { 当å‰çŠ¶æ€ï¼š{getStatusTag(selectedDetail.asset.status)} 最新评测：{getEvaluationTag(selectedDetail.asset.latest_evaluation_status)} + + 评测è¿è¡ŒåŽ†å² +
}} + /> + diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 56972dc6..fe4f5779 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -556,6 +556,28 @@ export type PromptAssetRunLinkItem = { created_at: string; }; +export type PromptAssetEvaluationRunItem = { + id: string; + tenant_id: string; + prompt_asset_id: string; + prompt_asset_version_id: string; + prompt_key: string; + prompt_type: PromptAssetType; + suite_key: string; + status: string; + sample_count: number; + passed_samples: number; + failed_samples: number; + pass_rate: number | null; + quality_score: number | null; + report: Record | null; + failure_reason: string | null; + triggered_by: string | null; + started_at: string; + finished_at: string | null; + created_at: string; +}; + export type PromptAssetListResponse = PageResponse & { prompt_types: PromptAssetType[]; }; @@ -564,12 +586,14 @@ export type PromptAssetDetailResponse = { asset: PromptAssetItem; versions: PromptAssetVersionItem[]; run_links: PromptAssetRunLinkItem[]; + evaluation_runs: PromptAssetEvaluationRunItem[]; }; export type PromptAssetEvaluationRunResponse = { asset: PromptAssetItem; version: PromptAssetVersionItem; evaluation_report: Record; + evaluation_run: PromptAssetEvaluationRunItem; }; export type PromptAssetCreatePayload = { diff --git a/project-docs/AIåˆåŒèµ·è‰ä¸Žå®¡æŸ¥åŠ©æ‰‹-当å‰å·¥ä½œè¿›åº¦ä¿å­˜-2026-05-11.md b/project-docs/AIåˆåŒèµ·è‰ä¸Žå®¡æŸ¥åŠ©æ‰‹-当å‰å·¥ä½œè¿›åº¦ä¿å­˜-2026-05-11.md index a1f81616..cfec4635 100644 --- a/project-docs/AIåˆåŒèµ·è‰ä¸Žå®¡æŸ¥åŠ©æ‰‹-当å‰å·¥ä½œè¿›åº¦ä¿å­˜-2026-05-11.md +++ b/project-docs/AIåˆåŒèµ·è‰ä¸Žå®¡æŸ¥åŠ©æ‰‹-当å‰å·¥ä½œè¿›åº¦ä¿å­˜-2026-05-11.md @@ -4078,3 +4078,33 @@ - 剩余边界： - 本轮没有部署到 3.90；由于无数æ®åº“è¿ç§»ï¼ŒåŽç»­åªéœ€éƒ¨ç½²åŽç«¯/å‰ç«¯å³å¯åœ¨æµ‹è¯•æœåŠ¡å™¨ä½¿ç”¨è¯¥æŒ‰é’®ã€‚ - 当å‰è¯„测接å£æ˜¯å‘布å‰é™æ€è´¨é‡é—¨ç¦ï¼Œä¸ç­‰åŒäºŽè°ƒç”¨çœŸå®žå¤§æ¨¡åž‹é€æ¡è·‘完整黄金样本输出；åŽç»­å¦‚需更强门ç¦ï¼Œå¯ç»§ç»­æŽ¥å…¥å¼‚步任务队列，把 `scripts/run-phase15-prompt-golden-eval.ps1` 或等价æœåŠ¡ç«¯è¯„测器的结果自动回填åŒä¸€å­—段。 + +## 2026-05-21 Phase15-F Prompt 评测è¿è¡Œè®°å½•ä¸Žå¯è¿½æº¯é—¨ç¦é—­çŽ¯ + +- 已继续按《AIåˆåŒèµ·è‰ä¸Žå®¡æŸ¥åŠ©æ‰‹-法律文书能力å¸æ”¶å¼€å‘方案-2026-05-20.md》推进 Phase15-F。 +- 本轮目标：把上一刀的“页é¢è§¦å‘评测并回填版本â€å‡çº§ä¸ºâ€œè¯„测è¿è¡Œæœ‰è®°å½•ã€æœ‰çŠ¶æ€ã€æœ‰æŠ¥å‘Šã€æœ‰å®¡è®¡ã€å¯è¿½æº¯â€ã€‚ +- åŽç«¯èƒ½åŠ›ï¼š + - 新增 Alembic è¿ç§» `20260521_0038_create_prompt_evaluation_runs.py`。 + - 新增 `prompt_asset_evaluation_runs` 表，记录æ¯æ¬¡ Prompt 评测è¿è¡Œã€‚ + - è¿è¡Œè®°å½•å­—段覆盖：Prompt 资产ã€ç‰ˆæœ¬ã€ç±»åž‹ã€å¥—件ã€çŠ¶æ€ã€æ ·æœ¬æ•°ã€é€šè¿‡æ•°ã€å¤±è´¥æ•°ã€é€šè¿‡çŽ‡ã€è´¨é‡åˆ†ã€æŠ¥å‘Š JSONã€å¤±è´¥åŽŸå› ã€è§¦å‘人ã€å¼€å§‹/结æŸæ—¶é—´ã€‚ + - `POST /api/v1/prompt-assets/{asset_id}/evaluations/run` 现在会先创建è¿è¡Œè®°å½•ï¼Œå†ç”Ÿæˆè¯„测报告，最åŽåŒæ­¥å›žå¡« Prompt 版本和资产最新评测状æ€ã€‚ + - `GET /api/v1/prompt-assets/{asset_id}` 详情会返回最近 20 æ¡ `evaluation_runs`，便于页é¢å’Œ QA 查è¯ã€‚ +- å‰ç«¯èƒ½åŠ›ï¼š + - `/prompt-assets` 详情区域新增“评测è¿è¡ŒåŽ†å²â€è¡¨æ ¼ã€‚ + - è¿è¡ŒåŽ†å²å±•ç¤ºå¥—件ã€çŠ¶æ€ã€æ ·æœ¬é€šè¿‡æƒ…况ã€åˆ†æ•°ã€å¤±è´¥åŽŸå› å’Œå®Œæˆæ—¶é—´ã€‚ + - å‘布按钮继续å—最新评测状æ€çº¦æŸï¼Œæœªé€šè¿‡è¯„测ä»ä¸å¯å‘布。 +- 本轮验è¯ï¼š + - `python -m compileall backend\app\models\prompt_asset_evaluation_run.py backend\app\repositories\prompt_asset_evaluation_run_repository.py backend\app\services\prompt_asset_service.py backend\app\api\prompt_assets.py` + - `python -m py_compile backend\migrations\versions\20260521_0038_create_prompt_evaluation_runs.py` + - `$env:PYTHONPATH='backend'; python -m unittest backend.tests.test_prompt_asset_service backend.tests.test_prompt_assets_api backend.tests.test_permission_metadata -v` + - `$env:PYTHONPATH='backend'; python -m unittest backend.tests.test_prompt_asset_service backend.tests.test_prompt_assets_api backend.tests.test_prompt_runtime_service backend.tests.test_chat_api backend.tests.test_clause_recommendation_service backend.tests.test_clauses_api backend.tests.test_review_service backend.tests.test_contracts_api backend.tests.test_permission_metadata -v` + - `cd frontend; npx eslint src/app/prompt-assets/page.tsx src/services/prompt-asset.ts src/types/api.ts` + - `cd frontend; npm run build` + - `cd backend; $env:PYTHONPATH='.'; python -m alembic heads` 返回 `20260521_0038 (head)`。 + - `git diff --check` + - Browser 打开 `http://localhost:3000/prompt-assets`，未登录ä¿æŠ¤é¡µæ­£å¸¸æ˜¾ç¤ºï¼Œé¡µé¢æ ‡é¢˜ `AIåˆåŒå¹³å°`，控制å°é”™è¯¯ 0。 +- 当å‰é—­çŽ¯ç»“论： + - Phase15-F 已形æˆâ€œé¡µé¢è§¦å‘评测 -> 创建评测è¿è¡Œè®°å½• -> 生æˆæŠ¥å‘Š -> 回填 Prompt 版本 -> å‘布门ç¦è¯»å–è¯„æµ‹çŠ¶æ€ -> è¯¦æƒ…é¡µå±•ç¤ºè¯„æµ‹åŽ†å² -> 审计留痕â€çš„闭环。 +- 剩余边界： + - 本轮尚未部署到 `192.168.3.90`；测试æœåŠ¡å™¨ä½¿ç”¨è¯¥èƒ½åŠ›å‰éœ€è¦æ‰§è¡Œ Alembic `20260521_0038` 并部署å‰åŽç«¯ã€‚ + - 当å‰è¿è¡Œä»æ˜¯åŒæ­¥è½»é‡è¯„测，适åˆæµ‹è¯•æœåŠ¡å™¨èµ„æºï¼›åŽç»­å¦‚æžœè¦è·‘真实大模型é€æ¡é»„金样本，å¯åœ¨è¿™å¼ è¿è¡Œè¡¨ä¸Šç»§ç»­æ‰©å±•å¼‚步队列和 worker。