334 lines
15 KiB
Python
334 lines
15 KiB
Python
import unittest
|
|
|
|
from fastapi import HTTPException
|
|
from fastapi.testclient import TestClient
|
|
|
|
from app.api.prompt_assets import get_audit_log_service, get_prompt_asset_service, require_current_user
|
|
from app.main import app
|
|
|
|
|
|
class StubAuditLogService:
|
|
def __init__(self):
|
|
self.events = []
|
|
|
|
def record_event(self, **kwargs):
|
|
self.events.append(kwargs)
|
|
return kwargs
|
|
|
|
|
|
class StubPromptAssetService:
|
|
def _require_admin(self, current_user):
|
|
if current_user["role"] != "admin":
|
|
raise HTTPException(status_code=403, detail="forbidden")
|
|
|
|
def _item(self, **overrides):
|
|
payload = {
|
|
"id": "prompt-123",
|
|
"tenant_id": "tenant-demo-001",
|
|
"prompt_key": "contract_generation_default",
|
|
"prompt_type": "contract_generation",
|
|
"name": "Contract generation default",
|
|
"description": "Default generation prompt",
|
|
"target_scene": "legal_document_generation",
|
|
"status": "draft",
|
|
"current_version_label": "v1.0",
|
|
"current_version_id": "promptver-123",
|
|
"published_version_id": None,
|
|
"published_at": None,
|
|
"published_by": None,
|
|
"rollback_from_version_id": None,
|
|
"latest_change_summary": "Initial prompt version",
|
|
"latest_evaluation_status": "passed",
|
|
"latest_evaluation_score": 0.95,
|
|
"latest_evaluation_report": {"sample_count": 3, "pass_rate": 0.95, "passed": True},
|
|
"version_count": 1,
|
|
"created_by": "user-admin",
|
|
"updated_by": "user-admin",
|
|
"created_at": "2026-05-21T00:00:00+00:00",
|
|
"updated_at": "2026-05-21T00:00:00+00:00",
|
|
}
|
|
payload.update(overrides)
|
|
return payload
|
|
|
|
def list_prompt_assets(self, current_user, page, page_size, prompt_type, status_filter):
|
|
self._require_admin(current_user)
|
|
return {
|
|
"items": [self._item()],
|
|
"page": page,
|
|
"page_size": page_size,
|
|
"total": 1,
|
|
"prompt_types": ["contract_generation", "contract_review"],
|
|
}
|
|
|
|
def get_prompt_asset_detail(self, current_user, asset_id):
|
|
self._require_admin(current_user)
|
|
return {
|
|
"asset": self._item(id=asset_id),
|
|
"versions": [
|
|
{
|
|
"id": "promptver-123",
|
|
"tenant_id": "tenant-demo-001",
|
|
"prompt_asset_id": asset_id,
|
|
"prompt_key": "contract_generation_default",
|
|
"prompt_type": "contract_generation",
|
|
"version_label": "v1.0",
|
|
"prompt_content": "Generate a contract.",
|
|
"system_prompt": None,
|
|
"variables": [],
|
|
"output_schema": None,
|
|
"model_config": None,
|
|
"evaluation_status": "passed",
|
|
"evaluation_score": 0.95,
|
|
"evaluation_report": {"sample_count": 3},
|
|
"change_summary": "Initial prompt version",
|
|
"rollback_from_version_id": None,
|
|
"is_current": True,
|
|
"is_published": False,
|
|
"created_by": "user-admin",
|
|
"created_by_name": "Admin",
|
|
"created_at": "2026-05-21T00:00:00+00:00",
|
|
"published_at": None,
|
|
"published_by": None,
|
|
}
|
|
],
|
|
"run_links": [],
|
|
"evaluation_runs": [
|
|
{
|
|
"id": "prompteval-123",
|
|
"tenant_id": "tenant-demo-001",
|
|
"prompt_asset_id": asset_id,
|
|
"prompt_asset_version_id": "promptver-123",
|
|
"prompt_key": "contract_generation_default",
|
|
"prompt_type": "contract_generation",
|
|
"suite_key": "phase13_generation",
|
|
"status": "passed",
|
|
"sample_count": 8,
|
|
"passed_samples": 8,
|
|
"failed_samples": 0,
|
|
"pass_rate": 1.0,
|
|
"quality_score": 0.95,
|
|
"report": {
|
|
"schema_version": "phase15-prompt-evaluation-report.v2",
|
|
"suite_key": "phase13_generation",
|
|
"evaluation_mode": "llm",
|
|
"sample_count": 8,
|
|
"passed_samples": 8,
|
|
"failed_samples": 0,
|
|
"sample_results": [{"sample_id": "GEN-SAMPLE-TPL-001", "passed": True}],
|
|
"passed": True,
|
|
},
|
|
"failure_reason": None,
|
|
"triggered_by": "user-admin",
|
|
"started_at": "2026-05-21T00:00:00+00:00",
|
|
"finished_at": "2026-05-21T00:00:01+00:00",
|
|
"created_at": "2026-05-21T00:00:00+00:00",
|
|
}
|
|
],
|
|
}
|
|
|
|
def create_prompt_asset(self, current_user, prompt_key, prompt_type, name, prompt_content, **kwargs):
|
|
self._require_admin(current_user)
|
|
return self._item(prompt_key=prompt_key, prompt_type=prompt_type, name=name)
|
|
|
|
def create_prompt_asset_version(self, current_user, asset_id, prompt_content, **kwargs):
|
|
self._require_admin(current_user)
|
|
return self._item(id=asset_id, current_version_label="v1.1", version_count=2)
|
|
|
|
def evaluate_current_prompt_asset(self, current_user, asset_id):
|
|
self._require_admin(current_user)
|
|
return {
|
|
"asset": self._item(id=asset_id, latest_evaluation_status="passed", latest_evaluation_score=0.95),
|
|
"version": {
|
|
"id": "promptver-123",
|
|
"tenant_id": "tenant-demo-001",
|
|
"prompt_asset_id": asset_id,
|
|
"prompt_key": "contract_generation_default",
|
|
"prompt_type": "contract_generation",
|
|
"version_label": "v1.0",
|
|
"prompt_content": "Generate a contract.",
|
|
"system_prompt": None,
|
|
"variables": [],
|
|
"output_schema": None,
|
|
"model_config": None,
|
|
"evaluation_status": "passed",
|
|
"evaluation_score": 0.95,
|
|
"evaluation_report": {
|
|
"schema_version": "phase15-prompt-evaluation-report.v2",
|
|
"suite_key": "phase13_generation",
|
|
"evaluation_mode": "llm",
|
|
"sample_count": 8,
|
|
"passed_samples": 8,
|
|
"failed_samples": 0,
|
|
"sample_results": [{"sample_id": "GEN-SAMPLE-TPL-001", "passed": True}],
|
|
"passed": True,
|
|
},
|
|
"change_summary": "Initial prompt version",
|
|
"rollback_from_version_id": None,
|
|
"is_current": True,
|
|
"is_published": False,
|
|
"created_by": "user-admin",
|
|
"created_by_name": "Admin",
|
|
"created_at": "2026-05-21T00:00:00+00:00",
|
|
"published_at": None,
|
|
"published_by": None,
|
|
},
|
|
"evaluation_report": {
|
|
"schema_version": "phase15-prompt-evaluation-report.v2",
|
|
"suite_key": "phase13_generation",
|
|
"evaluation_mode": "llm",
|
|
"sample_count": 8,
|
|
"passed_samples": 8,
|
|
"failed_samples": 0,
|
|
"sample_results": [{"sample_id": "GEN-SAMPLE-TPL-001", "passed": True}],
|
|
"passed": True,
|
|
"quality_score": 0.95,
|
|
},
|
|
"evaluation_run": {
|
|
"id": "prompteval-123",
|
|
"tenant_id": "tenant-demo-001",
|
|
"prompt_asset_id": asset_id,
|
|
"prompt_asset_version_id": "promptver-123",
|
|
"prompt_key": "contract_generation_default",
|
|
"prompt_type": "contract_generation",
|
|
"suite_key": "phase13_generation",
|
|
"status": "passed",
|
|
"sample_count": 8,
|
|
"passed_samples": 8,
|
|
"failed_samples": 0,
|
|
"pass_rate": 1.0,
|
|
"quality_score": 0.95,
|
|
"report": {
|
|
"schema_version": "phase15-prompt-evaluation-report.v2",
|
|
"suite_key": "phase13_generation",
|
|
"evaluation_mode": "llm",
|
|
"sample_count": 8,
|
|
"passed_samples": 8,
|
|
"failed_samples": 0,
|
|
"sample_results": [{"sample_id": "GEN-SAMPLE-TPL-001", "passed": True}],
|
|
"passed": True,
|
|
},
|
|
"failure_reason": None,
|
|
"triggered_by": current_user["id"],
|
|
"started_at": "2026-05-21T00:00:00+00:00",
|
|
"finished_at": "2026-05-21T00:00:01+00:00",
|
|
"created_at": "2026-05-21T00:00:00+00:00",
|
|
},
|
|
}
|
|
|
|
def publish_prompt_asset(self, current_user, asset_id, **kwargs):
|
|
self._require_admin(current_user)
|
|
return self._item(id=asset_id, status="published", published_version_id="promptver-123", published_by=current_user["id"], published_at="2026-05-21T01:00:00+00:00")
|
|
|
|
def unpublish_prompt_asset(self, current_user, asset_id):
|
|
self._require_admin(current_user)
|
|
return self._item(id=asset_id, status="inactive")
|
|
|
|
def rollback_prompt_asset(self, current_user, asset_id, target_version_label, change_summary=None):
|
|
self._require_admin(current_user)
|
|
return self._item(id=asset_id, current_version_label="v1.2", rollback_from_version_id="promptver-123", status="published")
|
|
|
|
def record_prompt_run_link(self, current_user, asset_id, run_type, **kwargs):
|
|
self._require_admin(current_user)
|
|
return {
|
|
"id": "promptrun-123",
|
|
"tenant_id": "tenant-demo-001",
|
|
"prompt_asset_id": asset_id,
|
|
"prompt_asset_version_id": "promptver-123",
|
|
"prompt_type": "contract_generation",
|
|
"run_type": run_type,
|
|
"run_id": kwargs.get("run_id"),
|
|
"resource_type": kwargs.get("resource_type"),
|
|
"resource_id": kwargs.get("resource_id"),
|
|
"workflow_run_id": kwargs.get("workflow_run_id"),
|
|
"task_run_id": kwargs.get("task_run_id"),
|
|
"trace": {"provider": "shared-ai-platform"},
|
|
"created_by": current_user["id"],
|
|
"created_at": "2026-05-21T01:00:00+00:00",
|
|
}
|
|
|
|
|
|
class PromptAssetsApiTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.audit_log_service = StubAuditLogService()
|
|
app.dependency_overrides[get_prompt_asset_service] = lambda: StubPromptAssetService()
|
|
app.dependency_overrides[get_audit_log_service] = lambda: self.audit_log_service
|
|
self.client = TestClient(app)
|
|
|
|
def tearDown(self):
|
|
app.dependency_overrides.clear()
|
|
|
|
def _as_admin(self):
|
|
app.dependency_overrides[require_current_user] = lambda: {
|
|
"id": "user-admin",
|
|
"tenant_id": "tenant-demo-001",
|
|
"role": "admin",
|
|
}
|
|
|
|
def test_admin_can_list_and_view_prompt_assets(self):
|
|
self._as_admin()
|
|
|
|
list_response = self.client.get("/api/v1/prompt-assets")
|
|
detail_response = self.client.get("/api/v1/prompt-assets/prompt-123")
|
|
|
|
self.assertEqual(list_response.status_code, 200)
|
|
self.assertEqual(list_response.json()["data"]["items"][0]["prompt_key"], "contract_generation_default")
|
|
self.assertEqual(detail_response.status_code, 200)
|
|
self.assertEqual(detail_response.json()["data"]["versions"][0]["version_label"], "v1.0")
|
|
|
|
def test_admin_can_create_publish_rollback_and_record_run_link(self):
|
|
self._as_admin()
|
|
|
|
create_response = self.client.post(
|
|
"/api/v1/prompt-assets",
|
|
data={
|
|
"prompt_key": "contract_generation_default",
|
|
"prompt_type": "contract_generation",
|
|
"name": "Contract generation default",
|
|
"prompt_content": "Generate a contract.",
|
|
"evaluation_report_json": '{"sample_count":3,"pass_rate":0.95,"passed":true}',
|
|
},
|
|
)
|
|
version_response = self.client.post("/api/v1/prompt-assets/prompt-123/versions", data={"prompt_content": "Generate a better contract."})
|
|
evaluation_response = self.client.post("/api/v1/prompt-assets/prompt-123/evaluations/run")
|
|
publish_response = self.client.post("/api/v1/prompt-assets/prompt-123/publish", data={"evaluation_report_json": '{"sample_count":3,"passed":true}'})
|
|
rollback_response = self.client.post("/api/v1/prompt-assets/prompt-123/rollback", data={"target_version_label": "v1.0"})
|
|
run_link_response = self.client.post(
|
|
"/api/v1/prompt-assets/prompt-123/run-links",
|
|
data={"run_type": "contract_generation", "run_id": "run-001", "resource_type": "generated_contract", "resource_id": "contract-001"},
|
|
)
|
|
|
|
self.assertEqual(create_response.status_code, 200)
|
|
self.assertEqual(version_response.json()["data"]["current_version_label"], "v1.1")
|
|
self.assertEqual(evaluation_response.json()["data"]["version"]["evaluation_status"], "passed")
|
|
self.assertEqual(evaluation_response.json()["data"]["evaluation_run"]["status"], "passed")
|
|
self.assertEqual(evaluation_response.json()["data"]["evaluation_report"]["evaluation_mode"], "llm")
|
|
self.assertEqual(publish_response.json()["data"]["status"], "published")
|
|
self.assertEqual(rollback_response.json()["data"]["current_version_label"], "v1.2")
|
|
self.assertEqual(run_link_response.json()["data"]["prompt_asset_version_id"], "promptver-123")
|
|
self.assertEqual([event["action"] for event in self.audit_log_service.events], [
|
|
"create_prompt_asset",
|
|
"create_prompt_asset_version",
|
|
"run_prompt_asset_evaluation",
|
|
"publish_prompt_asset",
|
|
"rollback_prompt_asset",
|
|
"record_prompt_asset_run_link",
|
|
])
|
|
self.assertEqual(self.audit_log_service.events[2]["detail_json"]["evaluation_mode"], "llm")
|
|
self.assertEqual(self.audit_log_service.events[2]["detail_json"]["sample_count"], 8)
|
|
|
|
def test_member_cannot_list_prompt_assets(self):
|
|
app.dependency_overrides[require_current_user] = lambda: {
|
|
"id": "user-member",
|
|
"tenant_id": "tenant-demo-001",
|
|
"role": "member",
|
|
}
|
|
|
|
response = self.client.get("/api/v1/prompt-assets")
|
|
|
|
self.assertEqual(response.status_code, 403)
|
|
self.assertEqual(response.json()["message"], "forbidden")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|