235 lines
9.7 KiB
Python
235 lines
9.7 KiB
Python
import unittest
|
|
from datetime import datetime, timezone
|
|
|
|
from fastapi import HTTPException
|
|
|
|
from app.models.prompt_asset import PromptAsset
|
|
from app.models.prompt_asset_evaluation_run import PromptAssetEvaluationRun
|
|
from app.models.prompt_asset_version import PromptAssetVersion
|
|
from app.services.quality_eval_service import QualityEvalService
|
|
|
|
|
|
class StubQualityEvalRunRepository:
|
|
def __init__(self):
|
|
self.items = {}
|
|
|
|
def create(self, quality_eval_run):
|
|
quality_eval_run.created_at = quality_eval_run.created_at or datetime.now(timezone.utc)
|
|
quality_eval_run.started_at = quality_eval_run.started_at or datetime.now(timezone.utc)
|
|
self.items[quality_eval_run.id] = quality_eval_run
|
|
return quality_eval_run
|
|
|
|
def update(self, quality_eval_run):
|
|
self.items[quality_eval_run.id] = quality_eval_run
|
|
return quality_eval_run
|
|
|
|
def get_by_id(self, run_id, tenant_id=None):
|
|
item = self.items.get(run_id)
|
|
if item is None:
|
|
return None
|
|
if tenant_id is not None and item.tenant_id != tenant_id:
|
|
return None
|
|
return item
|
|
|
|
def list_paginated(self, tenant_id, page, page_size, *, source_kind=None, eval_domain=None, subject_type=None, status_filter=None):
|
|
items = [item for item in self.items.values() if item.tenant_id == tenant_id]
|
|
if source_kind:
|
|
items = [item for item in items if item.source_kind == source_kind]
|
|
if eval_domain:
|
|
items = [item for item in items if item.eval_domain == eval_domain]
|
|
if subject_type:
|
|
items = [item for item in items if item.subject_type == subject_type]
|
|
if status_filter:
|
|
items = [item for item in items if item.status == status_filter]
|
|
items = sorted(items, key=lambda item: (item.created_at, item.id), reverse=True)
|
|
start = (page - 1) * page_size
|
|
return items[start : start + page_size]
|
|
|
|
def count(self, tenant_id, *, source_kind=None, eval_domain=None, subject_type=None, status_filter=None):
|
|
return len(self.list_paginated(tenant_id, 1, 100000, source_kind=source_kind, eval_domain=eval_domain, subject_type=subject_type, status_filter=status_filter))
|
|
|
|
|
|
class StubQualityEvalItemRepository:
|
|
def __init__(self):
|
|
self.items = {}
|
|
|
|
def create(self, quality_eval_item):
|
|
quality_eval_item.created_at = quality_eval_item.created_at or datetime.now(timezone.utc)
|
|
self.items[quality_eval_item.id] = quality_eval_item
|
|
return quality_eval_item
|
|
|
|
def update(self, quality_eval_item):
|
|
self.items[quality_eval_item.id] = quality_eval_item
|
|
return quality_eval_item
|
|
|
|
def get_by_id(self, item_id, tenant_id=None):
|
|
item = self.items.get(item_id)
|
|
if item is None:
|
|
return None
|
|
if tenant_id is not None and item.tenant_id != tenant_id:
|
|
return None
|
|
return item
|
|
|
|
def list_by_run_id(self, tenant_id, run_id):
|
|
return [
|
|
item
|
|
for item in sorted(self.items.values(), key=lambda value: (value.created_at, value.item_key))
|
|
if item.tenant_id == tenant_id and item.quality_eval_run_id == run_id
|
|
]
|
|
|
|
|
|
class QualityEvalServiceTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.service = QualityEvalService(db=None)
|
|
self.service.run_repository = StubQualityEvalRunRepository()
|
|
self.service.item_repository = StubQualityEvalItemRepository()
|
|
self.admin_user = {"id": "user-admin", "tenant_id": "tenant-demo-001", "role": "admin"}
|
|
self.member_user = {"id": "user-member", "tenant_id": "tenant-demo-001", "role": "member"}
|
|
self.asset = PromptAsset(
|
|
id="prompt-001",
|
|
tenant_id="tenant-demo-001",
|
|
prompt_key="contract_generation_default",
|
|
prompt_type="contract_generation",
|
|
name="Contract generation default",
|
|
status="draft",
|
|
current_version_label="v1.0",
|
|
version_count=1,
|
|
created_at=datetime.now(timezone.utc),
|
|
updated_at=datetime.now(timezone.utc),
|
|
)
|
|
self.version = PromptAssetVersion(
|
|
id="promptver-001",
|
|
tenant_id="tenant-demo-001",
|
|
prompt_asset_id=self.asset.id,
|
|
prompt_key=self.asset.prompt_key,
|
|
prompt_type=self.asset.prompt_type,
|
|
version_label="v1.0",
|
|
prompt_content="Generate a contract from structured inputs.",
|
|
evaluation_status="passed",
|
|
evaluation_score=0.95,
|
|
evaluation_report_json={"sample_count": 2},
|
|
is_current=True,
|
|
is_published=False,
|
|
created_at=datetime.now(timezone.utc),
|
|
)
|
|
self.evaluation_run = PromptAssetEvaluationRun(
|
|
id="prompteval-001",
|
|
tenant_id=self.asset.tenant_id,
|
|
prompt_asset_id=self.asset.id,
|
|
prompt_asset_version_id=self.version.id,
|
|
prompt_key=self.asset.prompt_key,
|
|
prompt_type=self.asset.prompt_type,
|
|
suite_key="phase13_generation",
|
|
status="passed",
|
|
sample_count=2,
|
|
passed_samples=1,
|
|
failed_samples=1,
|
|
pass_rate=0.5,
|
|
quality_score=0.81,
|
|
report_json={"sample_count": 2},
|
|
triggered_by=self.admin_user["id"],
|
|
started_at=datetime.now(timezone.utc),
|
|
finished_at=datetime.now(timezone.utc),
|
|
created_at=datetime.now(timezone.utc),
|
|
)
|
|
|
|
def test_record_prompt_asset_evaluation_creates_run_and_items(self):
|
|
result = self.service.record_prompt_asset_evaluation(
|
|
self.admin_user,
|
|
self.asset,
|
|
self.version,
|
|
self.evaluation_run,
|
|
{
|
|
"suite_key": "phase13_generation",
|
|
"sample_count": 2,
|
|
"passed_samples": 1,
|
|
"failed_samples": 1,
|
|
"pass_rate": 0.5,
|
|
"quality_score": 0.81,
|
|
"passed": False,
|
|
"failed_items": [{"key": "sample_failed:GEN-002", "label": "Golden sample failed: GEN-002"}],
|
|
"sample_results": [
|
|
{
|
|
"sample_id": "GEN-001",
|
|
"sample_family": "phase13_generation",
|
|
"passed": True,
|
|
"score": 0.91,
|
|
"observed_output_summary": "ok",
|
|
"matched_expectations": ["expected_boundary"],
|
|
"issues": [],
|
|
},
|
|
{
|
|
"sample_id": "GEN-002",
|
|
"sample_family": "phase13_generation",
|
|
"passed": False,
|
|
"score": 0.42,
|
|
"observed_output_summary": "missing legal basis",
|
|
"matched_expectations": [],
|
|
"issues": ["missing_legal_basis"],
|
|
},
|
|
],
|
|
},
|
|
)
|
|
|
|
self.assertEqual(result["run"]["source_kind"], "prompt_asset_evaluation_run")
|
|
self.assertEqual(result["run"]["source_run_id"], self.evaluation_run.id)
|
|
self.assertEqual(result["run"]["eval_domain"], "prompt")
|
|
self.assertEqual(result["run"]["subject_id"], self.asset.id)
|
|
self.assertEqual(result["run"]["sample_count"], 2)
|
|
self.assertEqual(result["run"]["passed_samples"], 1)
|
|
self.assertEqual(result["run"]["failed_samples"], 1)
|
|
self.assertEqual(result["run"]["failure_reason"], "Golden sample failed: GEN-002")
|
|
self.assertEqual(len(result["items"]), 2)
|
|
self.assertEqual(result["items"][0]["item_key"], "GEN-001")
|
|
self.assertEqual(result["items"][1]["status"], "failed")
|
|
self.assertEqual(result["items"][1]["failure_reason"], "missing_legal_basis")
|
|
|
|
def test_list_and_detail_require_admin(self):
|
|
with self.assertRaises(HTTPException) as context:
|
|
self.service.list_quality_eval_runs(self.member_user)
|
|
self.assertEqual(context.exception.status_code, 403)
|
|
|
|
with self.assertRaises(HTTPException) as context:
|
|
self.service.get_quality_eval_run_detail(self.member_user, "prompteval-001")
|
|
self.assertEqual(context.exception.status_code, 403)
|
|
|
|
def test_list_and_detail_return_data_for_admin(self):
|
|
recorded = self.service.record_prompt_asset_evaluation(
|
|
self.admin_user,
|
|
self.asset,
|
|
self.version,
|
|
self.evaluation_run,
|
|
{
|
|
"suite_key": "phase13_generation",
|
|
"sample_count": 1,
|
|
"passed_samples": 1,
|
|
"failed_samples": 0,
|
|
"pass_rate": 1.0,
|
|
"quality_score": 0.99,
|
|
"passed": True,
|
|
"sample_results": [
|
|
{
|
|
"sample_id": "GEN-001",
|
|
"sample_family": "phase13_generation",
|
|
"passed": True,
|
|
"score": 0.99,
|
|
"observed_output_summary": "ok",
|
|
"matched_expectations": ["expected_boundary"],
|
|
"issues": [],
|
|
}
|
|
],
|
|
},
|
|
)
|
|
|
|
listed = self.service.list_quality_eval_runs(self.admin_user, page=1, page_size=10, eval_domain="prompt")
|
|
detail = self.service.get_quality_eval_run_detail(self.admin_user, recorded["run"]["id"])
|
|
|
|
self.assertEqual(listed["total"], 1)
|
|
self.assertEqual(listed["items"][0]["subject_key"], "contract_generation_default")
|
|
self.assertEqual(detail["run"]["status"], "passed")
|
|
self.assertEqual(detail["items"][0]["item_key"], "GEN-001")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|