Codex
97 lines
3.9 KiB
Python
97 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from src.services.research_council import ResearchCouncilService
|
|
|
|
|
|
def main(env=None) -> int:
|
|
env = os.environ if env is None else env
|
|
instrument_id = int(env.get("RESEARCH_COUNCIL_INSTRUMENT_ID", "1"))
|
|
limit = int(env.get("RESEARCH_COUNCIL_EVAL_LIMIT", "20"))
|
|
output_json = _get_bool(env, "RESEARCH_COUNCIL_OUTPUT_JSON", False)
|
|
runs = ResearchCouncilService().get_recent_runs(instrument_id=instrument_id, limit=limit)
|
|
payload = build_eval_summary(instrument_id=instrument_id, runs=runs)
|
|
payload["passed"] = bool(payload.get("run_count"))
|
|
_print_payload(payload, output_json=output_json)
|
|
return 0 if payload["passed"] else 2
|
|
|
|
|
|
def build_eval_summary(*, instrument_id: int, runs: list[dict]) -> dict:
|
|
if not runs:
|
|
return {
|
|
"instrument_id": instrument_id,
|
|
"run_count": 0,
|
|
"error": "no_research_council_runs",
|
|
}
|
|
baseline_counts = []
|
|
follow_up_counts = []
|
|
question_counts = []
|
|
disagreement_count = 0
|
|
lesson_usage_count = 0
|
|
session_scores: dict[str, list[float]] = {}
|
|
setup_scores: dict[str, list[float]] = {}
|
|
|
|
for row in runs:
|
|
payload = dict(row.get("payload") or {})
|
|
context = dict(payload.get("context") or {})
|
|
decision = dict(payload.get("decision") or {})
|
|
candidate = dict(context.get("latest_candidate") or {})
|
|
trust_policy = dict(context.get("trust_policy") or {})
|
|
risk_result = dict(context.get("risk_result") or {})
|
|
baseline = len(list(candidate.get("missing_confirmations") or [])) + len(list(trust_policy.get("blocking_reasons") or [])) + len(list(risk_result.get("reasons") or []))
|
|
baseline_counts.append(baseline)
|
|
follow_up_counts.append(len(list(decision.get("follow_up_checks") or [])))
|
|
question_counts.append(len(list(decision.get("operator_questions") or [])))
|
|
disagreement_count += 1 if list(decision.get("disagreement_points") or []) else 0
|
|
lessons = list(context.get("compact_lessons") or [])
|
|
if lessons:
|
|
lesson_usage_count += 1
|
|
session_code = str(candidate.get("session_code") or "UNKNOWN")
|
|
setup_type = str(candidate.get("model_code") or "UNKNOWN")
|
|
session_scores.setdefault(session_code, []).append(float(decision.get("committee_confidence") or 0.0))
|
|
setup_scores.setdefault(setup_type, []).append(float(decision.get("committee_confidence") or 0.0))
|
|
|
|
return {
|
|
"instrument_id": instrument_id,
|
|
"run_count": len(runs),
|
|
"avg_baseline_issue_count": _avg(baseline_counts),
|
|
"avg_follow_up_check_count": _avg(follow_up_counts),
|
|
"avg_operator_question_count": _avg(question_counts),
|
|
"guidance_delta": round(_avg(follow_up_counts) - _avg(baseline_counts), 4),
|
|
"disagreement_hint_rate": round(disagreement_count / len(runs), 4),
|
|
"lesson_usage_rate": round(lesson_usage_count / len(runs), 4),
|
|
"session_confidence": {key: round(_avg(values), 4) for key, values in session_scores.items()},
|
|
"setup_confidence": {key: round(_avg(values), 4) for key, values in setup_scores.items()},
|
|
}
|
|
|
|
|
|
def _avg(values: list[float]) -> float:
|
|
return round(sum(values) / len(values), 4) if values else 0.0
|
|
|
|
|
|
def _print_payload(payload: dict, *, output_json: bool) -> None:
|
|
if output_json:
|
|
print(json.dumps(payload, ensure_ascii=False, sort_keys=True, default=str))
|
|
return
|
|
print(f"Research council eval for instrument {payload.get('instrument_id')}")
|
|
print(f"Runs: {payload.get('run_count')}")
|
|
|
|
|
|
def _get_bool(env, key: str, default: bool) -> bool:
|
|
value = env.get(key)
|
|
if value is None or value == "":
|
|
return default
|
|
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|