367 lines
16 KiB
Python
367 lines
16 KiB
Python
"""Verify and aggregate the TP06.8 context-optimization holdout.
|
|
|
|
The candidate holdout is behaviorally independent from the baseline suite, so
|
|
latency comparisons are explicitly non-paired. Raw provider answers remain in
|
|
case artifacts; this evidence contains bounded metrics and hashes only.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
from statistics import median
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
for path in (
|
|
"yuqei-ai-sdk-python/src",
|
|
"yuqei-ai-platform/services/ai-platform-api/src",
|
|
"yuqei-infra/scripts",
|
|
):
|
|
sys.path.insert(0, str(ROOT / path))
|
|
|
|
from evaluation_artifacts import draft_digest, write_evidence
|
|
|
|
|
|
def _load_review_runner():
|
|
path = ROOT / "yuqei-infra/scripts/evaluate-scene-reviews.py"
|
|
spec = importlib.util.spec_from_file_location("tp06_8_review_runner", path)
|
|
if spec is None or spec.loader is None:
|
|
raise RuntimeError("Scene-review evaluator could not be loaded.")
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _file_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _portable(path: Path) -> str:
|
|
try:
|
|
return path.resolve().relative_to(ROOT).as_posix()
|
|
except ValueError:
|
|
return path.as_posix()
|
|
|
|
|
|
def _percentile(values: list[float], percentile: float) -> float:
|
|
ordered = sorted(values)
|
|
if not ordered:
|
|
return 0.0
|
|
if len(ordered) == 1:
|
|
return ordered[0]
|
|
position = (len(ordered) - 1) * percentile
|
|
lower = int(position)
|
|
upper = min(lower + 1, len(ordered) - 1)
|
|
fraction = position - lower
|
|
return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction
|
|
|
|
|
|
def _rounded(value: float) -> float:
|
|
return round(float(value), 3)
|
|
|
|
|
|
def _metric(values: list[float]) -> dict[str, float]:
|
|
return {
|
|
"minimum": _rounded(min(values)),
|
|
"median": _rounded(median(values)),
|
|
"p95": _rounded(_percentile(values, 0.95)),
|
|
"maximum": _rounded(max(values)),
|
|
}
|
|
|
|
|
|
def _artifact_paths(root: Path) -> dict[str, Path]:
|
|
paths: dict[str, Path] = {}
|
|
for part in sorted(root.glob("run-part-*")):
|
|
for path in sorted(part.glob("REVIEW-*.json")):
|
|
if path.stem in paths:
|
|
raise ValueError(f"Duplicate candidate artifact: {path.stem}")
|
|
paths[path.stem] = path
|
|
return paths
|
|
|
|
|
|
def _baseline_metrics(root: Path, evidence_path: Path) -> dict:
|
|
evidence = json.loads(evidence_path.read_text(encoding="utf-8"))
|
|
expected = evidence.get("medium") or {}
|
|
directory = root / "medium-run-02"
|
|
paths = sorted(directory.glob("REVIEW-*.json"))
|
|
if (
|
|
evidence.get("schema_version") != "tp06-7-reasoning-effort-quality-evidence.v2"
|
|
or expected.get("sample_count") != 12
|
|
or len(paths) != 12
|
|
):
|
|
raise ValueError("TP06.7 medium baseline is incomplete.")
|
|
records = []
|
|
canonical_hashes = []
|
|
for path in paths:
|
|
artifact = json.loads(path.read_text(encoding="utf-8"))
|
|
telemetry = artifact["job"]["result"]["validation_evidence"]["execution_telemetry"]
|
|
audit = artifact.get("audit") or {}
|
|
canonical_hashes.append(draft_digest(artifact))
|
|
records.append({
|
|
"elapsed_seconds": float(artifact["elapsed_seconds"]),
|
|
"prompt_tokens": int(audit.get("prompt_tokens") or 0),
|
|
"completion_tokens": int(audit.get("completion_tokens") or 0),
|
|
"rendered_prompt_characters": int(
|
|
telemetry["input_scale"]["rendered_prompt_characters"]
|
|
),
|
|
})
|
|
if draft_digest(sorted(canonical_hashes)) != expected.get("artifact_canonical_set_sha256"):
|
|
raise ValueError("TP06.7 medium baseline artifact set changed.")
|
|
return {
|
|
"kind": "verified_tp06_7_medium_baseline",
|
|
"evidence_file": _portable(evidence_path),
|
|
"evidence_raw_sha256": _file_sha256(evidence_path),
|
|
"artifact_canonical_set_sha256": expected["artifact_canonical_set_sha256"],
|
|
"sample_count": 12,
|
|
"technical_passed": expected["technical_passed"],
|
|
"expectation_passed": expected["expectation_passed"],
|
|
"latency_seconds": _metric([item["elapsed_seconds"] for item in records]),
|
|
"prompt_tokens": _metric([item["prompt_tokens"] for item in records]),
|
|
"completion_tokens": _metric([item["completion_tokens"] for item in records]),
|
|
"rendered_prompt_characters": _metric([
|
|
item["rendered_prompt_characters"] for item in records
|
|
]),
|
|
}
|
|
|
|
|
|
def build(
|
|
root: Path,
|
|
baseline_root: Path,
|
|
baseline_evidence: Path,
|
|
output: Path,
|
|
) -> dict:
|
|
if output.exists():
|
|
raise FileExistsError(f"Evidence already exists: {output}")
|
|
runner = _load_review_runner()
|
|
frozen_path = root / "frozen-v1.json"
|
|
frozen = json.loads(frozen_path.read_text(encoding="utf-8"))
|
|
if (
|
|
frozen.get("schema_version") != runner.CONTEXT_OPTIMIZATION_HOLDOUT_VERSION
|
|
or frozen.get("model") != "gpt-5.6-sol"
|
|
or frozen.get("codex_cli_reasoning_effort") != "medium"
|
|
):
|
|
raise ValueError("Invalid context optimization frozen suite.")
|
|
expected_ids = [item["id"] for item in frozen["cases"]]
|
|
paths = _artifact_paths(root)
|
|
if set(paths) != set(expected_ids) or len(paths) != 12:
|
|
raise ValueError("Context optimization run must contain exactly 12 cases.")
|
|
combined = root / "run-01"
|
|
if combined.exists():
|
|
raise FileExistsError(f"Combined directory already exists: {combined}")
|
|
combined.mkdir(parents=True)
|
|
|
|
frozen_sha256 = draft_digest(frozen)
|
|
by_id = {item["id"]: item for item in frozen["cases"]}
|
|
records = []
|
|
canonical_hashes = []
|
|
raw_hashes = []
|
|
for case_id in expected_ids:
|
|
path = paths[case_id]
|
|
artifact = json.loads(path.read_text(encoding="utf-8"))
|
|
result = (artifact.get("job") or {}).get("result") or {}
|
|
evidence = result.get("validation_evidence") or {}
|
|
telemetry = evidence.get("execution_telemetry") or {}
|
|
runtime_context = evidence.get("runtime_context_policy") or {}
|
|
configuration = artifact.get("execution_configuration") or {}
|
|
assessment = runner.assess_review(
|
|
frozen, by_id[case_id], artifact.get("job") or {}, artifact.get("audit")
|
|
)
|
|
if (
|
|
artifact.get("case_id") != case_id
|
|
or artifact.get("real_model_call") is not True
|
|
or artifact.get("frozen_file_sha256") != frozen_sha256
|
|
or configuration.get("model") != "gpt-5.6-sol"
|
|
or configuration.get("codex_cli_reasoning_effort") != "medium"
|
|
or artifact.get("assessment") != assessment
|
|
or runtime_context.get("policy_version")
|
|
!= "contract-review-runtime-context.v1"
|
|
or runtime_context.get("model_source_projection")
|
|
!= "legal-decision-source.v1"
|
|
or runtime_context.get("output_schema_delivery")
|
|
!= "provider_json_schema"
|
|
or runtime_context.get("supplemental_request_summary") is not False
|
|
or telemetry.get("contains_raw_contract_text") is not False
|
|
or telemetry.get("contains_raw_prompt") is not False
|
|
or not artifact.get("synthetic_provider_responses")
|
|
):
|
|
raise ValueError(f"Invalid context optimization artifact: {case_id}")
|
|
canonical_hash = draft_digest(artifact)
|
|
raw_hash = _file_sha256(path)
|
|
canonical_hashes.append(canonical_hash)
|
|
raw_hashes.append(raw_hash)
|
|
shutil.copy2(path, combined / path.name)
|
|
audit = artifact.get("audit") or {}
|
|
scale = telemetry.get("input_scale") or {}
|
|
retry = evidence.get("provider_validation_repair") or {}
|
|
records.append({
|
|
"case_id": case_id,
|
|
"scene_id": by_id[case_id]["scene_id"],
|
|
"variant": by_id[case_id]["variant"],
|
|
"elapsed_seconds": _rounded(artifact["elapsed_seconds"]),
|
|
"prompt_tokens": int(audit.get("prompt_tokens") or 0),
|
|
"completion_tokens": int(audit.get("completion_tokens") or 0),
|
|
"total_tokens": int(audit.get("total_tokens") or 0),
|
|
"rendered_prompt_characters": int(scale["rendered_prompt_characters"]),
|
|
"model_source_context_characters": int(
|
|
scale["model_source_context_characters"]
|
|
),
|
|
"model_citation_count": int(scale["model_citation_count"]),
|
|
"provider_seconds": _rounded(
|
|
(telemetry.get("timings_ms", {}).get("initial_provider") or 0) / 1000
|
|
),
|
|
"technical_passed": assessment["technical_passed"],
|
|
"expectation_passed": assessment["expectation_passed"],
|
|
"review_completed": assessment["review_completed"],
|
|
"review_status": assessment["review_status"],
|
|
"fallback_used": bool((artifact.get("job") or {}).get("fallback_used")),
|
|
"awaiting_human": (artifact.get("job") or {}).get("status") == "awaiting_human",
|
|
"validation_repair_attempted": retry.get("attempted") is True,
|
|
"integrity_errors": assessment["integrity_errors"],
|
|
"artifact_canonical_sha256": canonical_hash,
|
|
"artifact_raw_sha256": raw_hash,
|
|
})
|
|
write_evidence(combined / "summary.json", {
|
|
"schema_version": "tp06-8-context-optimization-run-summary.v1",
|
|
"samples": records,
|
|
"artifact_canonical_set_sha256": draft_digest(sorted(canonical_hashes)),
|
|
"artifact_raw_set_sha256": draft_digest(sorted(raw_hashes)),
|
|
})
|
|
|
|
baseline = _baseline_metrics(baseline_root, baseline_evidence)
|
|
candidate = {
|
|
"kind": "fresh_independent_context_optimization_holdout",
|
|
"frozen_file": _portable(frozen_path),
|
|
"frozen_canonical_sha256": frozen_sha256,
|
|
"frozen_raw_sha256": _file_sha256(frozen_path),
|
|
"request_set_sha256": frozen["holdout"]["request_set_sha256"],
|
|
"fixture_sha256": frozen["fixture_sha256"],
|
|
"prompt_version": frozen["holdout"]["prompt_version"],
|
|
"sample_count": len(records),
|
|
"technical_passed": sum(item["technical_passed"] for item in records),
|
|
"expectation_passed": sum(item["expectation_passed"] for item in records),
|
|
"fallback_count": sum(item["fallback_used"] for item in records),
|
|
"awaiting_human_count": sum(item["awaiting_human"] for item in records),
|
|
"validation_repair_count": sum(
|
|
item["validation_repair_attempted"] for item in records
|
|
),
|
|
"latency_seconds": _metric([item["elapsed_seconds"] for item in records]),
|
|
"provider_seconds": _metric([item["provider_seconds"] for item in records]),
|
|
"prompt_tokens": _metric([item["prompt_tokens"] for item in records]),
|
|
"completion_tokens": _metric([item["completion_tokens"] for item in records]),
|
|
"rendered_prompt_characters": _metric([
|
|
item["rendered_prompt_characters"] for item in records
|
|
]),
|
|
"model_source_context_characters": _metric([
|
|
item["model_source_context_characters"] for item in records
|
|
]),
|
|
"artifact_canonical_set_sha256": draft_digest(sorted(canonical_hashes)),
|
|
"artifact_raw_set_sha256": draft_digest(sorted(raw_hashes)),
|
|
"records": records,
|
|
}
|
|
prompt_token_reduction = (
|
|
1 - candidate["prompt_tokens"]["median"] / baseline["prompt_tokens"]["median"]
|
|
) * 100
|
|
prompt_character_reduction = (
|
|
1 - candidate["rendered_prompt_characters"]["median"]
|
|
/ baseline["rendered_prompt_characters"]["median"]
|
|
) * 100
|
|
latency_change = (
|
|
candidate["latency_seconds"]["median"]
|
|
/ baseline["latency_seconds"]["median"] - 1
|
|
) * 100
|
|
quality_gate_met = (
|
|
candidate["technical_passed"] == candidate["sample_count"]
|
|
and candidate["expectation_passed"] == candidate["sample_count"]
|
|
and candidate["fallback_count"] == 0
|
|
and candidate["awaiting_human_count"] == 0
|
|
)
|
|
context_gate_met = prompt_token_reduction >= 25
|
|
evidence = {
|
|
"schema_version": "tp06-8-context-optimization-quality-evidence.v1",
|
|
"title": "TP06.8 gpt-5.6-sol medium 上下文优化质量与耗时证据",
|
|
"model": "gpt-5.6-sol",
|
|
"codex_cli_reasoning_effort": "medium",
|
|
"comparison_kind": "non_paired_fresh_holdout_vs_verified_prior_baseline",
|
|
"paired_latency_claim_prohibited": True,
|
|
"real_model_calls": 12,
|
|
"baseline": baseline,
|
|
"candidate": candidate,
|
|
"comparison": {
|
|
"median_prompt_token_reduction_percent": _rounded(prompt_token_reduction),
|
|
"median_rendered_prompt_character_reduction_percent": _rounded(
|
|
prompt_character_reduction
|
|
),
|
|
"median_latency_change_percent": _rounded(latency_change),
|
|
"candidate_latency_target_seconds": 180,
|
|
"candidate_latency_target_met": (
|
|
candidate["latency_seconds"]["median"] < 180
|
|
),
|
|
},
|
|
"decision": {
|
|
"quality_gate_met": quality_gate_met,
|
|
"context_reduction_gate_met": context_gate_met,
|
|
"adopt_runtime_context_policy": quality_gate_met and context_gate_met,
|
|
"adopt_prompt_version": (
|
|
candidate["prompt_version"] if quality_gate_met and context_gate_met else None
|
|
),
|
|
"retain_medium_reasoning_effort": True,
|
|
"latency_target_is_separate_gate": True,
|
|
"latency_target_met": candidate["latency_seconds"]["median"] < 180,
|
|
"failed_result_selection_or_hidden_retry": False,
|
|
},
|
|
"limitations": [
|
|
"candidate_and_baseline_inputs_are_not_identical_so_latency_is_not_paired",
|
|
"synthetic_holdout_does_not_establish_legal_correctness",
|
|
"source_applicability_remains_unverified",
|
|
"public_release_requires_separate_release_gate",
|
|
],
|
|
"live_database_writes": 0,
|
|
"live_prompt_changes": 0,
|
|
"public_release_authorized": False,
|
|
"not_legal_approval": True,
|
|
"contains_raw_contract_text": False,
|
|
"contains_raw_prompt": False,
|
|
"contains_raw_source_quotes": False,
|
|
}
|
|
write_evidence(output, evidence)
|
|
return evidence
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--root", required=True, type=Path)
|
|
parser.add_argument("--baseline-root", required=True, type=Path)
|
|
parser.add_argument("--baseline-evidence", required=True, type=Path)
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
args = parser.parse_args()
|
|
evidence = build(
|
|
args.root.resolve(),
|
|
args.baseline_root.resolve(),
|
|
args.baseline_evidence.resolve(),
|
|
args.output.resolve(),
|
|
)
|
|
print(json.dumps({
|
|
"output": _portable(args.output),
|
|
"technical_passed": evidence["candidate"]["technical_passed"],
|
|
"expectation_passed": evidence["candidate"]["expectation_passed"],
|
|
"sample_count": evidence["candidate"]["sample_count"],
|
|
"median_prompt_token_reduction_percent": evidence["comparison"]["median_prompt_token_reduction_percent"],
|
|
"median_latency_change_percent": evidence["comparison"]["median_latency_change_percent"],
|
|
"adopt_runtime_context_policy": evidence["decision"]["adopt_runtime_context_policy"],
|
|
"latency_target_met": evidence["decision"]["latency_target_met"],
|
|
}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|