286 lines
13 KiB
Python
286 lines
13 KiB
Python
import copy
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Optional
|
|
from uuid import uuid4
|
|
|
|
from fastapi import HTTPException, status
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.models.ai_quality_eval_item import AiQualityEvalItem
|
|
from app.models.ai_quality_eval_run import AiQualityEvalRun
|
|
from app.models.prompt_asset import PromptAsset
|
|
from app.models.prompt_asset_evaluation_run import PromptAssetEvaluationRun
|
|
from app.models.prompt_asset_version import PromptAssetVersion
|
|
from app.repositories.ai_quality_eval_item_repository import AiQualityEvalItemRepository
|
|
from app.repositories.ai_quality_eval_run_repository import AiQualityEvalRunRepository
|
|
|
|
|
|
class QualityEvalService:
|
|
def __init__(self, db: Session):
|
|
self.db = db
|
|
self.run_repository = AiQualityEvalRunRepository(db)
|
|
self.item_repository = AiQualityEvalItemRepository(db)
|
|
|
|
def record_prompt_asset_evaluation(
|
|
self,
|
|
current_user: dict[str, Any],
|
|
asset: PromptAsset,
|
|
version: PromptAssetVersion,
|
|
evaluation_run: PromptAssetEvaluationRun,
|
|
report: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
self._require_admin(current_user)
|
|
payload = report if isinstance(report, dict) else {}
|
|
now = self._now()
|
|
sample_results = payload.get("sample_results") if isinstance(payload.get("sample_results"), list) else []
|
|
run = self.run_repository.create(
|
|
AiQualityEvalRun(
|
|
id=f"qualityeval-{uuid4().hex[:12]}",
|
|
tenant_id=asset.tenant_id,
|
|
source_kind="prompt_asset_evaluation_run",
|
|
source_run_id=evaluation_run.id,
|
|
eval_domain="prompt",
|
|
subject_type="prompt_asset",
|
|
subject_id=asset.id,
|
|
subject_key=asset.prompt_key,
|
|
subject_name=asset.name,
|
|
suite_key=str(evaluation_run.suite_key or payload.get("suite_key") or "prompt_suite"),
|
|
status=str(evaluation_run.status or payload.get("status") or "running"),
|
|
sample_count=int(evaluation_run.sample_count or payload.get("sample_count") or len(sample_results) or 0),
|
|
passed_samples=int(evaluation_run.passed_samples or payload.get("passed_samples") or self._count_passed_samples(sample_results)),
|
|
failed_samples=int(evaluation_run.failed_samples or payload.get("failed_samples") or self._count_failed_samples(sample_results)),
|
|
pass_rate=self._optional_float(evaluation_run.pass_rate if evaluation_run.pass_rate is not None else payload.get("pass_rate")),
|
|
quality_score=self._optional_float(
|
|
evaluation_run.quality_score if evaluation_run.quality_score is not None else payload.get("quality_score")
|
|
),
|
|
report_json=self._clone_json(payload),
|
|
failure_reason=self._build_failure_reason(payload),
|
|
triggered_by=evaluation_run.triggered_by or current_user.get("id"),
|
|
started_at=evaluation_run.started_at or now,
|
|
finished_at=evaluation_run.finished_at or now,
|
|
created_at=now,
|
|
)
|
|
)
|
|
|
|
items: list[AiQualityEvalItem] = []
|
|
for index, sample in enumerate(sample_results):
|
|
if not isinstance(sample, dict):
|
|
continue
|
|
items.append(self.item_repository.create(self._build_item(run, sample, index, now)))
|
|
|
|
run.sample_count = int(run.sample_count or 0)
|
|
run.passed_samples = int(run.passed_samples or 0)
|
|
run.failed_samples = int(run.failed_samples or 0)
|
|
run.pass_rate = self._optional_float(run.pass_rate)
|
|
run.quality_score = self._optional_float(run.quality_score)
|
|
run.report_json = self._clone_json(payload)
|
|
run.failure_reason = self._build_failure_reason(payload)
|
|
run.triggered_by = evaluation_run.triggered_by or current_user.get("id")
|
|
run.started_at = evaluation_run.started_at or now
|
|
run.finished_at = evaluation_run.finished_at or now
|
|
run = self.run_repository.update(run)
|
|
return {
|
|
"run": self._to_run_item(run),
|
|
"items": [self._to_item(item) for item in items],
|
|
}
|
|
|
|
def list_quality_eval_runs(
|
|
self,
|
|
current_user: dict[str, Any],
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
*,
|
|
source_kind: Optional[str] = None,
|
|
eval_domain: Optional[str] = None,
|
|
subject_type: Optional[str] = None,
|
|
status_filter: Optional[str] = None,
|
|
) -> dict[str, Any]:
|
|
self._require_admin(current_user)
|
|
tenant_id = str(current_user["tenant_id"])
|
|
normalized_page = max(page, 1)
|
|
normalized_page_size = max(min(page_size, 100), 1)
|
|
items = self.run_repository.list_paginated(
|
|
tenant_id,
|
|
normalized_page,
|
|
normalized_page_size,
|
|
source_kind=self._optional_text(source_kind),
|
|
eval_domain=self._optional_text(eval_domain),
|
|
subject_type=self._optional_text(subject_type),
|
|
status_filter=self._optional_text(status_filter),
|
|
)
|
|
return {
|
|
"items": [self._to_run_item(item) for item in items],
|
|
"page": normalized_page,
|
|
"page_size": normalized_page_size,
|
|
"total": self.run_repository.count(
|
|
tenant_id,
|
|
source_kind=self._optional_text(source_kind),
|
|
eval_domain=self._optional_text(eval_domain),
|
|
subject_type=self._optional_text(subject_type),
|
|
status_filter=self._optional_text(status_filter),
|
|
),
|
|
}
|
|
|
|
def get_quality_eval_run_detail(self, current_user: dict[str, Any], run_id: str) -> dict[str, Any]:
|
|
self._require_admin(current_user)
|
|
tenant_id = str(current_user["tenant_id"])
|
|
run = self.run_repository.get_by_id(run_id, tenant_id)
|
|
if run is None:
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="quality eval run not found")
|
|
return {
|
|
"run": self._to_run_item(run),
|
|
"items": [self._to_item(item) for item in self.item_repository.list_by_run_id(tenant_id, run.id)],
|
|
}
|
|
|
|
def _build_item(self, run: AiQualityEvalRun, sample: dict[str, Any], index: int, now: datetime) -> AiQualityEvalItem:
|
|
item_key = self._optional_text(sample.get("sample_id")) or self._optional_text(sample.get("id")) or f"sample-{index + 1:03d}"
|
|
item_name = (
|
|
self._optional_text(sample.get("sample_name"))
|
|
or self._optional_text(sample.get("sample_family"))
|
|
or self._optional_text(sample.get("title"))
|
|
or item_key
|
|
)
|
|
passed = self._sample_passed(sample)
|
|
status_value = "passed" if passed else "failed"
|
|
return AiQualityEvalItem(
|
|
id=f"qualityitem-{uuid4().hex[:12]}",
|
|
tenant_id=run.tenant_id,
|
|
quality_eval_run_id=run.id,
|
|
item_key=item_key,
|
|
item_name=item_name,
|
|
status=status_value,
|
|
score=self._optional_float(sample.get("score")),
|
|
expected_json=self._build_expected_payload(sample, item_key),
|
|
actual_json=self._build_actual_payload(sample, status_value),
|
|
evidence_json=self._clone_json(sample),
|
|
failure_reason=self._build_item_failure_reason(sample),
|
|
created_at=now,
|
|
)
|
|
|
|
def _build_expected_payload(self, sample: dict[str, Any], item_key: str) -> Optional[dict[str, Any]]:
|
|
expected = sample.get("expected")
|
|
if isinstance(expected, (dict, list)):
|
|
return self._clone_json(expected)
|
|
payload = {
|
|
"sample_id": item_key,
|
|
"sample_family": sample.get("sample_family"),
|
|
"input_summary": sample.get("input_summary"),
|
|
"expected_summary": sample.get("expected_summary"),
|
|
}
|
|
payload = {key: value for key, value in payload.items() if value not in (None, "", [], {})}
|
|
return payload or None
|
|
|
|
def _build_actual_payload(self, sample: dict[str, Any], status_value: str) -> dict[str, Any]:
|
|
payload = {
|
|
"status": status_value,
|
|
"passed": sample.get("passed"),
|
|
"score": self._optional_float(sample.get("score")),
|
|
"evaluator_mode": sample.get("evaluator_mode"),
|
|
"observed_output_summary": sample.get("observed_output_summary"),
|
|
"matched_expectations": self._clone_json(sample.get("matched_expectations")),
|
|
"issues": self._clone_json(sample.get("issues")),
|
|
}
|
|
return {key: value for key, value in payload.items() if value is not None}
|
|
|
|
def _build_item_failure_reason(self, sample: dict[str, Any]) -> Optional[str]:
|
|
if sample.get("passed") is True or str(sample.get("status") or "").lower() in {"pass", "passed"}:
|
|
return None
|
|
issues = sample.get("issues")
|
|
if isinstance(issues, list) and issues:
|
|
return "; ".join(str(issue) for issue in issues if issue is not None)[:500] or "quality evaluation failed"
|
|
observed = sample.get("observed_output_summary")
|
|
if observed:
|
|
return str(observed)[:500]
|
|
return "quality evaluation failed"
|
|
|
|
def _build_failure_reason(self, report: dict[str, Any]) -> Optional[str]:
|
|
failed_items = report.get("failed_items")
|
|
if isinstance(failed_items, list) and failed_items:
|
|
first = failed_items[0]
|
|
if isinstance(first, dict):
|
|
return str(first.get("label") or first.get("key") or "quality evaluation failed")[:500]
|
|
if report.get("passed") is True:
|
|
return None
|
|
return "quality evaluation failed"
|
|
|
|
def _to_run_item(self, run: AiQualityEvalRun) -> dict[str, Any]:
|
|
return {
|
|
"id": run.id,
|
|
"tenant_id": run.tenant_id,
|
|
"source_kind": run.source_kind,
|
|
"source_run_id": run.source_run_id,
|
|
"eval_domain": run.eval_domain,
|
|
"subject_type": run.subject_type,
|
|
"subject_id": run.subject_id,
|
|
"subject_key": run.subject_key,
|
|
"subject_name": run.subject_name,
|
|
"suite_key": run.suite_key,
|
|
"status": run.status,
|
|
"sample_count": run.sample_count,
|
|
"passed_samples": run.passed_samples,
|
|
"failed_samples": run.failed_samples,
|
|
"pass_rate": run.pass_rate,
|
|
"quality_score": run.quality_score,
|
|
"report": run.report_json,
|
|
"failure_reason": run.failure_reason,
|
|
"triggered_by": run.triggered_by,
|
|
"started_at": run.started_at.isoformat(),
|
|
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
|
"created_at": run.created_at.isoformat(),
|
|
}
|
|
|
|
def _to_item(self, item: AiQualityEvalItem) -> dict[str, Any]:
|
|
return {
|
|
"id": item.id,
|
|
"tenant_id": item.tenant_id,
|
|
"quality_eval_run_id": item.quality_eval_run_id,
|
|
"item_key": item.item_key,
|
|
"item_name": item.item_name,
|
|
"status": item.status,
|
|
"score": item.score,
|
|
"expected": item.expected_json,
|
|
"actual": item.actual_json,
|
|
"evidence": item.evidence_json,
|
|
"failure_reason": item.failure_reason,
|
|
"created_at": item.created_at.isoformat(),
|
|
}
|
|
|
|
def _count_passed_samples(self, sample_results: list[dict[str, Any]]) -> int:
|
|
return sum(1 for sample in sample_results if self._sample_passed(sample))
|
|
|
|
def _count_failed_samples(self, sample_results: list[dict[str, Any]]) -> int:
|
|
return sum(1 for sample in sample_results if not self._sample_passed(sample))
|
|
|
|
def _sample_passed(self, sample: dict[str, Any]) -> bool:
|
|
return sample.get("passed") is True or str(sample.get("status") or "").lower() in {"pass", "passed"}
|
|
|
|
def _optional_text(self, value: Optional[str], max_length: Optional[int] = None) -> Optional[str]:
|
|
if value is None:
|
|
return None
|
|
normalized = str(value).strip()
|
|
if not normalized:
|
|
return None
|
|
if max_length is not None:
|
|
return normalized[:max_length]
|
|
return normalized
|
|
|
|
def _optional_float(self, value: Any) -> Optional[float]:
|
|
if value is None or value == "":
|
|
return None
|
|
try:
|
|
return float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
def _clone_json(self, value: Any) -> Any:
|
|
if value is None:
|
|
return None
|
|
return copy.deepcopy(value)
|
|
|
|
def _now(self) -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
def _require_admin(self, current_user: dict[str, Any]) -> None:
|
|
if current_user.get("role") != "admin":
|
|
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="forbidden")
|