1386 lines
70 KiB
Python
1386 lines
70 KiB
Python
"""W4 execution over existing isolated business/AI flows, not a new service.
|
|
|
|
Real calls require the explicit --execute switch and a matching frozen plan.
|
|
Each case/operation is attempted at most once per ledger. Resume only starts
|
|
previously untouched cases; unknown or failed work is never automatically sent
|
|
again. Local hashes support reproducibility, not adversarial authentication.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import re
|
|
import tempfile
|
|
from contextlib import contextmanager
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from time import perf_counter
|
|
from uuid import uuid4
|
|
|
|
from multiscene_call_ledger import CallLedger, CallLedgerError, OPERATIONS
|
|
|
|
|
|
HISTORICAL_USAGE_SCHEMA = "rc05-historical-provider-usage.v3"
|
|
MINIMUM_KNOWN_HISTORICAL_PROVIDER_CALLS = 4
|
|
REAL_PROVIDER_NAME = "codex-cli-development"
|
|
REAL_PROVIDER_MODEL = "gpt-5.6-sol"
|
|
REAL_RUN_REGISTRY_ENV = "YUQEI_RC05_RUN_REGISTRY_DIR"
|
|
_AFTER_FILES = {
|
|
"draft": ("01-draft-after.json",),
|
|
"semantic_review": ("02-semantic-after.json",),
|
|
"repair_draft": ("03-repair-after.json",),
|
|
"repair_review": ("04-post-review-after.json",),
|
|
"specialist_review": ("02-specialist-after.json", "02-after.json"),
|
|
"intake": ("02-after.json",),
|
|
}
|
|
_JOB_OPERATIONS = {
|
|
"contract_draft": {"draft", "repair_draft"},
|
|
"contract_draft_semantic_review": {"semantic_review", "repair_review"},
|
|
"contract_review": {"specialist_review"},
|
|
"contract_intake": {"intake"},
|
|
}
|
|
|
|
|
|
def _after_payload_measurements(payload) -> tuple[dict[str, object] | None, str | None]:
|
|
if not isinstance(payload, dict):
|
|
return None, "invalid_after_evidence"
|
|
elapsed = payload.get("execution_elapsed_seconds")
|
|
if not isinstance(elapsed, (int, float)) or isinstance(elapsed, bool) or elapsed < 0:
|
|
return None, "invalid_operation_elapsed_seconds"
|
|
job = payload.get("job") if isinstance(payload.get("job"), dict) else {}
|
|
audit = payload.get("audit") if isinstance(payload.get("audit"), dict) else {}
|
|
job_status = job.get("status")
|
|
if not isinstance(job_status, str) or not job_status:
|
|
return None, "missing_business_job_status"
|
|
provider_status = job.get("provider_status")
|
|
if provider_status is not None and provider_status not in {"succeeded", "failed", "cancelled"}:
|
|
return None, "invalid_provider_status"
|
|
timeout_fields = (
|
|
job.get("error_code"), job.get("error_message"),
|
|
audit.get("provider_error"), audit.get("provider_error_code"),
|
|
)
|
|
timed_out = any(
|
|
isinstance(value, str)
|
|
and ("timeout" in value.casefold() or "timed out" in value.casefold())
|
|
for value in timeout_fields
|
|
)
|
|
return {
|
|
"elapsed_seconds": round(float(elapsed), 3),
|
|
"timed_out": timed_out,
|
|
"business_status": job_status,
|
|
"provider_status": provider_status,
|
|
"job_operation": job.get("operation"),
|
|
}, None
|
|
|
|
|
|
def _evidence_path(runner, value: object) -> Path:
|
|
if not isinstance(value, str) or not value or len(value) > 2048 or "\x00" in value:
|
|
raise ValueError("historical evidence path is invalid")
|
|
supplied = Path(value)
|
|
if supplied.is_absolute():
|
|
path = supplied.absolute()
|
|
else:
|
|
if any(part in {".", ".."} for part in supplied.parts):
|
|
raise ValueError("historical repository evidence path is not normalized")
|
|
path = (runner.ROOT / supplied).absolute()
|
|
if not path.resolve().is_relative_to(Path(runner.ROOT).resolve()):
|
|
raise ValueError("historical repository evidence escapes the repository")
|
|
current = path
|
|
while True:
|
|
is_junction = getattr(current, "is_junction", lambda: False)
|
|
if current.is_symlink() or is_junction():
|
|
raise ValueError("historical evidence path must not traverse a link or junction")
|
|
if current.parent == current:
|
|
break
|
|
current = current.parent
|
|
if not path.is_file():
|
|
raise ValueError("historical evidence path must be a regular file")
|
|
return path
|
|
|
|
|
|
def _canonical_digest(runner, value: object) -> str:
|
|
return runner.sha256(json.dumps(
|
|
value, ensure_ascii=False, sort_keys=True, separators=(",", ":"),
|
|
allow_nan=False,
|
|
).encode("utf-8"))
|
|
|
|
|
|
def _strict_json(raw: bytes, label: str):
|
|
def pairs(items):
|
|
result = {}
|
|
for key, value in items:
|
|
if key in result:
|
|
raise ValueError(f"{label} contains a duplicate JSON key")
|
|
result[key] = value
|
|
return result
|
|
|
|
def constant(value):
|
|
raise ValueError(f"{label} contains non-finite JSON value {value}")
|
|
|
|
try:
|
|
return json.loads(raw, object_pairs_hook=pairs, parse_constant=constant)
|
|
except UnicodeError as error:
|
|
raise ValueError(f"{label} is not valid UTF-8 JSON") from error
|
|
|
|
|
|
def _json_artifact(runner, path_value: object, sha_value: object, label: str):
|
|
if (not isinstance(sha_value, str)
|
|
or re.fullmatch(r"[0-9a-f]{64}", sha_value) is None):
|
|
raise ValueError(f"historical {label} SHA-256 is invalid")
|
|
path = _evidence_path(runner, path_value)
|
|
raw = path.read_bytes()
|
|
if runner.sha256(raw) != sha_value:
|
|
raise ValueError(f"historical {label} SHA-256 mismatch")
|
|
try:
|
|
value = _strict_json(raw, f"historical {label}")
|
|
except ValueError as error:
|
|
raise ValueError(f"historical {label} is not valid JSON: {error}") from error
|
|
if not isinstance(value, dict):
|
|
raise ValueError(f"historical {label} must contain a JSON object")
|
|
return path, value
|
|
|
|
|
|
def _checked_get(flow, suffix):
|
|
response = flow.client.get(
|
|
f"{flow.drafting_prefix}/{flow.workspace['id']}{suffix}", headers=flow.member)
|
|
if response.status_code != 200:
|
|
raise RuntimeError(f"evaluation read failed: {response.status_code}")
|
|
return response.json()
|
|
|
|
|
|
def run_scene_case(runner, case, out, factory, state, *, ledger):
|
|
from multiscene_supplemental_cases import install_executors
|
|
|
|
started = perf_counter()
|
|
out.mkdir(parents=True)
|
|
runner.write_x(out / "attempt.json", {"case_id": case["case_id"],
|
|
"started_at": datetime.now(UTC).isoformat(), "automatic_retry": False,
|
|
"provider_call_budget": case["provider_call_budget"]})
|
|
with tempfile.TemporaryDirectory(prefix="w4-scene-") as temporary:
|
|
with runner.isolated_flow(Path(temporary)) as flow:
|
|
flow.member = {**flow.member, "X-User-Id": "user-current-admin"}
|
|
flow.drafting_prefix = "/api/v1/template-evaluations"
|
|
settings = runner.AiPlatformSettings(_env_file=None, environment="test",
|
|
repository_backend="memory", provider_call_mode="codex_cli",
|
|
provider_fallback_enabled=False, model_job_dispatch_mode="worker",
|
|
model_job_max_attempts=1, contract_review_provider_max_attempts=1,
|
|
model_job_manual_sample_rate_percent=0, model_job_high_risk_review_required=False,
|
|
codex_cli_model=runner.MODEL, codex_cli_reasoning_effort=runner.REASONING,
|
|
codex_cli_timeout_seconds=runner.TIMEOUT,
|
|
codex_cli_working_dir=str(runner.ROOT / ".runtime/tp06-24-3"))
|
|
adapter = runner.CountingAdapter(factory(settings), state,
|
|
case["provider_call_budget"], ledger=ledger)
|
|
adapter.frozen_case = case
|
|
runner.register_contract_drafting_executor(flow.ai_app, settings=settings,
|
|
store=flow.ai_repository, provider_adapter=adapter, context_mode="full-v1")
|
|
install_executors(flow, settings, adapter)
|
|
contracts_before = flow.repository.list_contracts().total
|
|
response = flow.client.post(flow.drafting_prefix,
|
|
headers={**flow.member, "Idempotency-Key": f"w4-{case['case_id']}"}, json={
|
|
"template_id": case["template_id"], "title": case["title"],
|
|
"values": case["values"], "instructions": case["instructions"],
|
|
"expected_template_content_sha256": case["template_content_sha256"],
|
|
"synthetic_only": True, **case["scene_request_context"],
|
|
})
|
|
if response.status_code != 201:
|
|
raise RuntimeError(f"evaluation workspace failed: {response.status_code}")
|
|
flow.workspace = response.json()
|
|
runner._post(flow, "/candidates", {
|
|
"expected_revision": flow.workspace["revision"],
|
|
"idempotency_key": f"w4-candidate-{case['case_id']}",
|
|
**case["candidate_policy"],
|
|
}, expected=202)
|
|
candidate = flow.workspace["candidates"][-1]
|
|
runner._run_job(flow, adapter, case["case_id"], "draft",
|
|
candidate["job"]["id"], out, 1)
|
|
runner._post(flow, f"/candidates/{candidate['id']}/refresh")
|
|
candidate = flow.workspace["candidates"][-1]
|
|
output = candidate.get("output") or {}
|
|
profile = case["expectation_profile"]
|
|
output_matches = (
|
|
output.get("status") == profile["expected_status"]
|
|
and set(profile["expected_missing_fields"]) == {
|
|
item["field"] for item in output.get("missing_fields", [])})
|
|
if not output_matches:
|
|
runner.write_x(out / "draft-expectation-failure.json", {
|
|
"expected": profile, "candidate": candidate})
|
|
raise RuntimeError("draft output differs from frozen scenario expectation")
|
|
runner._post(flow, f"/reviews/{candidate['id']}/start",
|
|
{"expected_body_sha256": candidate["body_sha256"]}, expected=202)
|
|
candidate = flow.workspace["candidates"][-1]
|
|
if not candidate.get("review_job"):
|
|
raise RuntimeError("specialist review admission did not produce a job")
|
|
reviewed = runner._run_job(flow, adapter, case["case_id"], "specialist",
|
|
candidate["review_job"]["id"], out, 2,
|
|
acceptable_statuses=("succeeded", "awaiting_human", "resolved"))
|
|
runner._post(flow, f"/reviews/{candidate['id']}/refresh")
|
|
candidate = flow.workspace["candidates"][-1]
|
|
evidence = _checked_get(flow, f"/candidates/{candidate['id']}/evidence")
|
|
history = _checked_get(flow, "/evaluation-history")
|
|
runner.write_x(out / "evaluation-evidence.json", evidence)
|
|
runner.write_x(out / "evaluation-history.json", history)
|
|
matches = [item for item in history.get("items", [])
|
|
if item.get("candidate_id") == candidate["id"]]
|
|
if not matches:
|
|
raise RuntimeError("completed scene has no actual evaluation checkpoint")
|
|
assessment = matches[0]
|
|
snapshot = _checked_get(flow, f"/evaluation-history/{assessment['id']}")
|
|
runner.write_x(out / "evaluation-checkpoint.json", snapshot)
|
|
technical = (output_matches and reviewed.provider_status == "succeeded"
|
|
and not candidate.get("review_validation_errors")
|
|
and flow.repository.list_contracts().total == contracts_before
|
|
and evidence["contract_writes"] == 0 and adapter.calls == 2)
|
|
review_expectation = assessment.get("review_expectation") or {}
|
|
# Existing scenario assessor, not a runner-local unconditional verdict.
|
|
quality = (assessment.get("draft_passed") is True
|
|
and not assessment.get("review_errors")
|
|
and review_expectation.get("expectation_passed") is True)
|
|
real = factory is runner.CodexCliDevelopmentProviderAdapter
|
|
result = {"case_id": case["case_id"], "status": "completed",
|
|
"elapsed_seconds": round(perf_counter() - started, 3),
|
|
"passed": technical and (quality if real else True),
|
|
"technical_passed": technical, "content_checks_passed": quality,
|
|
"automated_quality_passed": quality,
|
|
"automated_model_quality_release_passed": quality,
|
|
"content_quality_release_passed": quality,
|
|
"human_calibration_completed": False,
|
|
"legal_approval_completed": False,
|
|
"real_model_quality_verified": real and quality,
|
|
"provider_call_count": adapter.calls,
|
|
"document_kind": case["document_kind"], "mode": "scene_evaluation",
|
|
"assessment": assessment, "contract_writes_allowed": False,
|
|
"member_delivery_verified": False, "not_legal_approval": True}
|
|
runner.write_x(out / "result.json", result)
|
|
return result
|
|
|
|
|
|
def _safe_case_artifact(out: Path, path: Path) -> Path:
|
|
root = Path(out).absolute().resolve()
|
|
artifact = path.absolute()
|
|
if not artifact.resolve().is_relative_to(root):
|
|
raise ValueError("case artifact escapes the evidence output")
|
|
current = artifact
|
|
while True:
|
|
is_junction = getattr(current, "is_junction", lambda: False)
|
|
if current.is_symlink() or is_junction():
|
|
raise ValueError("case artifact traverses a link or junction")
|
|
if current == root:
|
|
break
|
|
if current.parent == current:
|
|
raise ValueError("case artifact escapes the evidence output")
|
|
current = current.parent
|
|
if not artifact.is_file():
|
|
raise ValueError("case artifact is not a regular file")
|
|
return artifact
|
|
|
|
|
|
def _case_evidence_hashes(runner, out: Path, ledger: CallLedger, case_id: str) -> dict[str, str]:
|
|
evidence: dict[str, str] = {}
|
|
for record in ledger.attempt_records():
|
|
if record["case_id"] != case_id:
|
|
continue
|
|
names = _AFTER_FILES[str(record["operation"])]
|
|
candidates = [out / case_id / name for name in names
|
|
if (out / case_id / name).exists()]
|
|
if len(candidates) > 1:
|
|
raise ValueError("case has ambiguous operation evidence")
|
|
if not candidates:
|
|
continue
|
|
artifact = _safe_case_artifact(out, candidates[0])
|
|
if artifact.name in evidence:
|
|
raise ValueError("case operation evidence names are not unique")
|
|
evidence[artifact.name] = runner.sha256(artifact.read_bytes())
|
|
return dict(sorted(evidence.items()))
|
|
|
|
|
|
def _bind_case_outcome(
|
|
runner, out: Path, ledger: CallLedger, case: dict, outcome_file: str,
|
|
) -> None:
|
|
case_id = case["case_id"]
|
|
artifact = _safe_case_artifact(out, out / case_id / outcome_file)
|
|
ledger.bind_case_outcome(
|
|
case_id,
|
|
outcome_file,
|
|
runner.sha256(artifact.read_bytes()),
|
|
_case_evidence_hashes(runner, out, ledger, case_id),
|
|
case["operations"],
|
|
)
|
|
|
|
|
|
def _existing_case(runner, out, case: dict, snapshot, ledger: CallLedger):
|
|
"""A partially written case is unknown, even before its first model call."""
|
|
case_id = case["case_id"]
|
|
directory = out / case_id
|
|
counts = snapshot["cases"][case_id]
|
|
candidates = [directory / filename for filename in ("result.json", "failure.json")
|
|
if (directory / filename).exists()]
|
|
if len(candidates) == 1:
|
|
path = candidates[0]
|
|
try:
|
|
artifact = _safe_case_artifact(out, path)
|
|
row = _strict_json(artifact.read_bytes(), "case result evidence")
|
|
if not isinstance(row, dict):
|
|
raise ValueError("case result must be an object")
|
|
calls = row.get("provider_call_count")
|
|
if (row.get("case_id") != case_id
|
|
or not isinstance(calls, int) or isinstance(calls, bool)
|
|
or calls != counts["attempted"]
|
|
or not isinstance(row.get("passed"), bool)
|
|
or (artifact.name == "result.json" and row.get("status") != "completed")
|
|
or (artifact.name == "failure.json"
|
|
and (row.get("status") != "stopped" or row.get("passed") is not False))):
|
|
raise ValueError("case result shape or counts are invalid")
|
|
ledger.verify_case_outcome(
|
|
case_id,
|
|
artifact.name,
|
|
runner.sha256(artifact.read_bytes()),
|
|
_case_evidence_hashes(runner, out, ledger, case_id),
|
|
case["operations"],
|
|
)
|
|
return row
|
|
except (OSError, UnicodeError, ValueError, RuntimeError):
|
|
pass
|
|
if directory.exists() or counts["attempted"]:
|
|
return {"case_id": case_id, "status": "unknown", "passed": False,
|
|
"provider_call_count": counts["attempted"], "automatic_retry": False}
|
|
return {"case_id": case_id, "status": "not_attempted", "passed": False,
|
|
"provider_call_count": 0}
|
|
|
|
|
|
def load_historical_usage(runner, path: Path) -> dict[str, object]:
|
|
"""Validate a real-provider receipt down to its immutable ledger chain."""
|
|
receipt = _evidence_path(runner, str(Path(path).absolute()))
|
|
raw = receipt.read_bytes()
|
|
try:
|
|
payload = _strict_json(raw, "historical usage receipt")
|
|
except ValueError as error:
|
|
raise ValueError("historical usage receipt is not valid JSON") from error
|
|
expected = {"schema_version", "attempts"}
|
|
if not isinstance(payload, dict) or set(payload) != expected:
|
|
raise ValueError("historical usage receipt shape is invalid")
|
|
if payload["schema_version"] != HISTORICAL_USAGE_SCHEMA:
|
|
raise ValueError("historical usage receipt schema is invalid")
|
|
attempts = payload["attempts"]
|
|
if not isinstance(attempts, list) or len(attempts) < MINIMUM_KNOWN_HISTORICAL_PROVIDER_CALLS:
|
|
raise ValueError("RC-05 receipt omits one or more of the four known historical calls")
|
|
if len(attempts) > 280:
|
|
raise ValueError("historical usage exceeds the global provider-call ceiling")
|
|
normalized = []
|
|
attempt_ids: set[str] = set()
|
|
claim_hashes: set[str] = set()
|
|
expected_attempt_keys = {
|
|
"attempt_id", "plan_sha256", "case_id", "operation",
|
|
"provider_status", "business_status", "elapsed_seconds", "timed_out",
|
|
"request_sha256", "response_sha256", "provider", "model",
|
|
"job_id", "audit_run_id",
|
|
"identity_path", "identity_file_sha256", "ledger_identity_sha256",
|
|
"claim_path", "claim_sha256", "finish_path", "finish_sha256",
|
|
"before_path", "before_sha256", "after_path", "after_sha256",
|
|
"disposition",
|
|
}
|
|
for attempt in attempts:
|
|
if not isinstance(attempt, dict) or set(attempt) != expected_attempt_keys:
|
|
raise ValueError("historical attempt shape is invalid")
|
|
attempt_id = attempt["attempt_id"]
|
|
plan_sha256 = attempt["plan_sha256"]
|
|
case_id = attempt["case_id"]
|
|
operation = attempt["operation"]
|
|
provider_status = attempt["provider_status"]
|
|
business_status = attempt["business_status"]
|
|
elapsed_seconds = attempt["elapsed_seconds"]
|
|
timed_out = attempt["timed_out"]
|
|
if (not isinstance(attempt_id, str)
|
|
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", attempt_id) is None):
|
|
raise ValueError("historical attempt ID is invalid")
|
|
if attempt_id in attempt_ids:
|
|
raise ValueError("historical attempt IDs must be unique")
|
|
if operation not in OPERATIONS:
|
|
raise ValueError("historical attempt operation is invalid")
|
|
if (not isinstance(case_id, str)
|
|
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}", case_id) is None):
|
|
raise ValueError("historical attempt case ID is invalid")
|
|
for label in ("plan_sha256", "request_sha256", "response_sha256",
|
|
"ledger_identity_sha256"):
|
|
value = attempt[label]
|
|
if not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None:
|
|
raise ValueError(f"historical {label} is invalid")
|
|
if provider_status not in {"succeeded", "failed", "cancelled"}:
|
|
raise ValueError("historical provider status is invalid")
|
|
if business_status not in {
|
|
"succeeded", "failed", "cancelled", "awaiting_human", "resolved",
|
|
}:
|
|
raise ValueError("historical business status is invalid")
|
|
if (not isinstance(elapsed_seconds, (int, float))
|
|
or isinstance(elapsed_seconds, bool) or elapsed_seconds < 0):
|
|
raise ValueError("historical elapsed seconds is invalid")
|
|
if not isinstance(timed_out, bool):
|
|
raise ValueError("historical timeout flag is invalid")
|
|
if timed_out and provider_status != "failed":
|
|
raise ValueError("historical timeout requires failed provider status")
|
|
if attempt["provider"] != REAL_PROVIDER_NAME or attempt["model"] != REAL_PROVIDER_MODEL:
|
|
raise ValueError("historical attempt is not the frozen real provider/model")
|
|
for label in ("job_id", "audit_run_id"):
|
|
if (not isinstance(attempt[label], str) or not attempt[label]
|
|
or len(attempt[label]) > 256 or "\x00" in attempt[label]):
|
|
raise ValueError(f"historical {label} is invalid")
|
|
disposition = attempt["disposition"]
|
|
if not isinstance(disposition, dict) or set(disposition) != {
|
|
"resolution_status", "release_eligible", "reason",
|
|
"replacement_case_id", "replacement_operation",
|
|
}:
|
|
raise ValueError("historical attempt disposition shape is invalid")
|
|
if disposition["resolution_status"] not in {"resolved", "requires_replacement"}:
|
|
raise ValueError("historical attempt disposition status is invalid")
|
|
if not isinstance(disposition["release_eligible"], bool):
|
|
raise ValueError("historical attempt release eligibility is invalid")
|
|
if (not isinstance(disposition["reason"], str)
|
|
or not disposition["reason"].strip() or len(disposition["reason"]) > 1000):
|
|
raise ValueError("historical attempt disposition reason is invalid")
|
|
successful_provider_business = (
|
|
provider_status == "succeeded"
|
|
and business_status in {"succeeded", "resolved"}
|
|
and not timed_out
|
|
)
|
|
if disposition["release_eligible"] and not successful_provider_business:
|
|
raise ValueError(
|
|
"historical provider/business failure cannot be release eligible"
|
|
)
|
|
if (not successful_provider_business
|
|
and disposition["resolution_status"] != "requires_replacement"):
|
|
raise ValueError("historical provider/business failure requires replacement")
|
|
if disposition["release_eligible"] and disposition["resolution_status"] != "resolved":
|
|
raise ValueError("historical release eligibility requires resolved disposition")
|
|
replacement_case_id = disposition["replacement_case_id"]
|
|
replacement_operation = disposition["replacement_operation"]
|
|
if disposition["resolution_status"] == "requires_replacement":
|
|
if (not isinstance(replacement_case_id, str)
|
|
or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}",
|
|
replacement_case_id) is None
|
|
or replacement_operation not in OPERATIONS
|
|
or disposition["release_eligible"]):
|
|
raise ValueError("historical replacement disposition is invalid")
|
|
elif replacement_case_id is not None or replacement_operation is not None:
|
|
raise ValueError("resolved historical disposition cannot declare replacement")
|
|
|
|
identity_path, identity = _json_artifact(
|
|
runner, attempt["identity_path"], attempt["identity_file_sha256"],
|
|
"ledger identity",
|
|
)
|
|
if (identity.get("schema_version") != "multiscene-call-ledger.v1"
|
|
or identity.get("real_provider") is not True
|
|
or identity.get("plan_sha256") != plan_sha256
|
|
or not isinstance(identity.get("case_limits"), dict)
|
|
or case_id not in identity["case_limits"]
|
|
or not isinstance(identity.get("operation_limits"), dict)
|
|
or identity["operation_limits"].get(operation, 0) < 1):
|
|
raise ValueError("historical ledger identity is not a matching real run")
|
|
ledger_identity_sha256 = _canonical_digest(runner, identity)
|
|
if ledger_identity_sha256 != attempt["ledger_identity_sha256"]:
|
|
raise ValueError("historical ledger identity digest mismatch")
|
|
|
|
claim_path, claim = _json_artifact(
|
|
runner, attempt["claim_path"], attempt["claim_sha256"], "claim")
|
|
finish_path, finish = _json_artifact(
|
|
runner, attempt["finish_path"], attempt["finish_sha256"], "finish")
|
|
expected_stem = f"{case_id}--{operation}"
|
|
if (claim_path.name != f"{expected_stem}.claim.json"
|
|
or finish_path.name != f"{expected_stem}.finish.json"
|
|
or claim_path.parent != finish_path.parent
|
|
or identity_path.parent / "attempts" != claim_path.parent):
|
|
raise ValueError("historical claim/finish do not belong to the declared ledger")
|
|
if attempt["claim_sha256"] in claim_hashes:
|
|
raise ValueError("historical claim evidence must be unique")
|
|
if (claim.get("schema_version") != "multiscene-call-claim.v1"
|
|
or claim.get("ledger_identity_sha256") != ledger_identity_sha256
|
|
or claim.get("case_id") != case_id or claim.get("operation") != operation
|
|
or claim.get("request_sha256") != attempt["request_sha256"]
|
|
or claim.get("attempt_number") != 1):
|
|
raise ValueError("historical claim does not bind the declared attempt")
|
|
if (finish.get("schema_version") != "multiscene-call-finish.v1"
|
|
or finish.get("ledger_identity_sha256") != ledger_identity_sha256
|
|
or finish.get("claim_id") != claim.get("claim_id")
|
|
or finish.get("case_id") != case_id or finish.get("operation") != operation
|
|
or finish.get("status") != provider_status
|
|
or finish.get("response_sha256") != attempt["response_sha256"]):
|
|
raise ValueError("historical finish does not bind the declared attempt")
|
|
|
|
before_path, before = _json_artifact(
|
|
runner, attempt["before_path"], attempt["before_sha256"], "before evidence")
|
|
after_path, evidence_payload = _json_artifact(
|
|
runner, attempt["after_path"], attempt["after_sha256"], "after evidence")
|
|
run_root = identity_path.parent.parent.resolve()
|
|
if (before_path.parent != after_path.parent
|
|
or not before_path.resolve().is_relative_to(run_root)
|
|
or not after_path.resolve().is_relative_to(run_root)):
|
|
raise ValueError("historical before/after do not belong to the declared run")
|
|
before_job = before.get("job") if isinstance(before.get("job"), dict) else {}
|
|
before_job_operation = before_job.get("operation")
|
|
if (before.get("request_sha256") != attempt["request_sha256"]
|
|
or _canonical_digest(runner, before.get("request")) != attempt["request_sha256"]
|
|
or before_job.get("id") != attempt["job_id"]
|
|
or before_job_operation not in _JOB_OPERATIONS
|
|
or operation not in _JOB_OPERATIONS[before_job_operation]):
|
|
raise ValueError("historical before evidence does not bind request/operation")
|
|
measurements, measurement_error = _after_payload_measurements(evidence_payload)
|
|
if measurement_error is not None:
|
|
raise ValueError(f"historical attempt evidence is invalid: {measurement_error}")
|
|
if (measurements["provider_status"] != provider_status
|
|
or measurements["business_status"] != business_status
|
|
or measurements["elapsed_seconds"] != round(float(elapsed_seconds), 3)
|
|
or measurements["timed_out"] is not timed_out):
|
|
raise ValueError("historical attempt declaration differs from evidence")
|
|
job_operation = measurements["job_operation"]
|
|
if job_operation not in _JOB_OPERATIONS or operation not in _JOB_OPERATIONS[job_operation]:
|
|
raise ValueError("historical attempt operation differs from evidence")
|
|
job = evidence_payload.get("job") if isinstance(evidence_payload.get("job"), dict) else {}
|
|
audit = evidence_payload.get("audit") if isinstance(evidence_payload.get("audit"), dict) else {}
|
|
if (job.get("id") != attempt["job_id"]
|
|
or job.get("audit_run_id") != attempt["audit_run_id"]
|
|
or audit.get("id") != attempt["audit_run_id"]
|
|
or audit.get("provider") != attempt["provider"]
|
|
or audit.get("model") != attempt["model"]
|
|
or audit.get("provider_status") != provider_status):
|
|
raise ValueError("historical after evidence lacks matching real provider invocation")
|
|
execution_evidence = (evidence_payload.get("evidence")
|
|
if isinstance(evidence_payload.get("evidence"), dict) else {})
|
|
output = evidence_payload.get("output")
|
|
if operation in {"draft", "repair_draft"}:
|
|
if execution_evidence.get("input_sha256") != attempt["request_sha256"]:
|
|
raise ValueError("historical draft evidence request digest mismatch")
|
|
if provider_status == "succeeded":
|
|
identity_evidence = execution_evidence.get("execution_identity")
|
|
if (not isinstance(identity_evidence, dict) or output is None
|
|
or identity_evidence.get("output_sha256")
|
|
!= _canonical_digest(runner, output)):
|
|
raise ValueError("historical draft output digest mismatch")
|
|
elif timed_out and (output is not None
|
|
or attempt["response_sha256"] != runner.sha256(b"")):
|
|
raise ValueError("historical timed-out draft has unexpected output")
|
|
elif operation in {"semantic_review", "repair_review"}:
|
|
wire_raw = execution_evidence.get("wire_output_raw")
|
|
if (execution_evidence.get("request_sha256") != attempt["request_sha256"]
|
|
or not isinstance(wire_raw, str)
|
|
or execution_evidence.get("wire_output_sha256")
|
|
!= runner.sha256(wire_raw.encode("utf-8"))
|
|
or execution_evidence.get("wire_output_sha256")
|
|
!= attempt["response_sha256"]
|
|
or execution_evidence.get("output_sha256")
|
|
!= _canonical_digest(runner, output)):
|
|
raise ValueError("historical semantic response digest mismatch")
|
|
attempt_ids.add(attempt_id)
|
|
claim_hashes.add(attempt["claim_sha256"])
|
|
normalized.append({**attempt,
|
|
"elapsed_seconds": round(float(elapsed_seconds), 3),
|
|
"disposition": dict(disposition),
|
|
})
|
|
provider_call_count = len(normalized)
|
|
return {
|
|
"provider_call_count": provider_call_count,
|
|
# Conservatively reserve the same number from the optional repair pool.
|
|
"repair_budget_charged": min(provider_call_count, 16),
|
|
"attempts": normalized,
|
|
"receipt_sha256": runner.sha256(raw),
|
|
}
|
|
|
|
|
|
def _percentile(values: list[float], quantile: float) -> float | None:
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
position = (len(ordered) - 1) * quantile
|
|
lower = math.floor(position)
|
|
upper = math.ceil(position)
|
|
if lower == upper:
|
|
return round(ordered[lower], 3)
|
|
result = ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
|
|
return round(result, 3)
|
|
|
|
|
|
def _operation_after_evidence(
|
|
out: Path, record: dict[str, object], *, real_provider: bool,
|
|
) -> tuple[dict[str, object] | None, str | None]:
|
|
case_id = str(record["case_id"])
|
|
operation = str(record["operation"])
|
|
candidates = [out / case_id / name for name in _AFTER_FILES[operation]
|
|
if (out / case_id / name).exists()]
|
|
if len(candidates) != 1 or candidates[0].is_symlink() or not candidates[0].is_file():
|
|
return None, "missing_or_ambiguous_after_evidence"
|
|
try:
|
|
payload = _strict_json(candidates[0].read_bytes(), "current after evidence")
|
|
except (OSError, UnicodeError, ValueError):
|
|
return None, "invalid_after_evidence"
|
|
measurements, error = _after_payload_measurements(payload)
|
|
if error is not None:
|
|
return None, error
|
|
if (payload.get("case_id") != case_id or payload.get("operation") != operation
|
|
or payload.get("request_sha256") != record["request_sha256"]
|
|
or payload.get("response_sha256") != record["response_sha256"]
|
|
or payload.get("provider_invocation_count") != 1):
|
|
return None, "after_evidence_ledger_binding_mismatch"
|
|
job_operation = measurements["job_operation"]
|
|
if job_operation not in _JOB_OPERATIONS or operation not in _JOB_OPERATIONS[job_operation]:
|
|
return None, "after_evidence_operation_mismatch"
|
|
if real_provider:
|
|
job = payload.get("job") if isinstance(payload.get("job"), dict) else {}
|
|
audit = payload.get("audit") if isinstance(payload.get("audit"), dict) else {}
|
|
job_id = job.get("id")
|
|
audit_id = audit.get("id")
|
|
audit_run_id = job.get("audit_run_id")
|
|
if (measurements["provider_status"] not in {"succeeded", "failed", "cancelled"}
|
|
or not isinstance(job_id, str) or not job_id
|
|
or not isinstance(audit_id, str) or not audit_id
|
|
or not isinstance(audit_run_id, str) or not audit_run_id
|
|
or audit_id != audit_run_id
|
|
or audit.get("provider") != REAL_PROVIDER_NAME
|
|
or audit.get("model") != REAL_PROVIDER_MODEL
|
|
or audit.get("provider_status") != measurements["provider_status"]):
|
|
return None, "after_evidence_real_provider_mismatch"
|
|
return measurements, None
|
|
|
|
|
|
def _business_outcome(status: str, *, allow_resolved: bool = False) -> str:
|
|
if status == "succeeded" or (allow_resolved and status == "resolved"):
|
|
return "succeeded"
|
|
if status in {"failed", "cancelled", "awaiting_human"}:
|
|
return status
|
|
return "other"
|
|
|
|
|
|
def _historical_by_operation(
|
|
ledger: CallLedger, operation_names,
|
|
) -> tuple[dict[str, dict], dict[str, list[float]]]:
|
|
result = {
|
|
name: {
|
|
"attempted": 0,
|
|
"terminal": 0,
|
|
"unknown": 0,
|
|
"provider_outcomes": {
|
|
"succeeded": 0, "failed": 0, "cancelled": 0, "timeout": 0,
|
|
},
|
|
"business_outcomes": {
|
|
"succeeded": 0, "failed": 0, "cancelled": 0,
|
|
"awaiting_human": 0, "other": 0,
|
|
},
|
|
}
|
|
for name in operation_names
|
|
}
|
|
elapsed = {name: [] for name in operation_names}
|
|
for attempt in ledger.historical_usage["attempts"]:
|
|
operation = attempt["operation"]
|
|
current = result[operation]
|
|
current["attempted"] += 1
|
|
current["terminal"] += 1
|
|
current["provider_outcomes"][attempt["provider_status"]] += 1
|
|
if attempt["timed_out"]:
|
|
current["provider_outcomes"]["timeout"] += 1
|
|
current["business_outcomes"][_business_outcome(
|
|
attempt["business_status"], allow_resolved=True)] += 1
|
|
elapsed[operation].append(float(attempt["elapsed_seconds"]))
|
|
for operation, current in result.items():
|
|
values = elapsed[operation]
|
|
current["elapsed_seconds"] = {
|
|
"observed": len(values),
|
|
"p50": _percentile(values, 0.50),
|
|
"p95": _percentile(values, 0.95),
|
|
}
|
|
return result, elapsed
|
|
|
|
|
|
def _operation_statistics(out: Path, ledger: CallLedger) -> tuple[dict[str, dict], list[dict]]:
|
|
snapshot = ledger.snapshot()
|
|
records = ledger.attempt_records()
|
|
historical, historical_elapsed = _historical_by_operation(
|
|
ledger, snapshot["operations"])
|
|
elapsed: dict[str, list[float]] = {name: [] for name in snapshot["operations"]}
|
|
business_outcomes = {
|
|
name: {"succeeded": 0, "failed": 0, "cancelled": 0,
|
|
"awaiting_human": 0, "other": 0}
|
|
for name in snapshot["operations"]
|
|
}
|
|
provider_outcomes = {
|
|
name: {"succeeded": 0, "failed": 0, "cancelled": 0, "timeout": 0}
|
|
for name in snapshot["operations"]
|
|
}
|
|
evidence_errors: list[dict] = []
|
|
for record in records:
|
|
status = record["status"]
|
|
operation = str(record["operation"])
|
|
if status is None:
|
|
continue
|
|
evidence, error = _operation_after_evidence(
|
|
out, record, real_provider=ledger.real_provider)
|
|
if error is not None:
|
|
if status in ("succeeded", "failed", "cancelled"):
|
|
provider_outcomes[operation][status] += 1
|
|
evidence_errors.append({
|
|
"case_id": record["case_id"], "operation": operation, "error": error,
|
|
})
|
|
continue
|
|
elapsed[operation].append(float(evidence["elapsed_seconds"]))
|
|
provider_status = evidence["provider_status"] or status
|
|
if provider_status != status:
|
|
evidence_errors.append({
|
|
"case_id": record["case_id"], "operation": operation,
|
|
"error": "provider_status_ledger_mismatch",
|
|
})
|
|
if provider_status in ("succeeded", "failed", "cancelled"):
|
|
provider_outcomes[operation][provider_status] += 1
|
|
business = _business_outcome(str(evidence["business_status"]))
|
|
business_outcomes[operation][business] += 1
|
|
if provider_status == "failed" and evidence["timed_out"]:
|
|
provider_outcomes[operation]["timeout"] += 1
|
|
result = {}
|
|
for operation, counts in snapshot["operations"].items():
|
|
values = elapsed[operation]
|
|
prior = historical[operation]
|
|
combined_provider = {
|
|
status: provider_outcomes[operation][status]
|
|
+ prior["provider_outcomes"][status]
|
|
for status in ("succeeded", "failed", "cancelled", "timeout")
|
|
}
|
|
combined_business = {
|
|
status: business_outcomes[operation][status]
|
|
+ prior["business_outcomes"][status]
|
|
for status in business_outcomes[operation]
|
|
}
|
|
combined_elapsed = [*historical_elapsed[operation], *values]
|
|
result[operation] = {
|
|
"limit": counts["limit"],
|
|
"attempted": counts["attempted"],
|
|
"terminal": counts["terminal"],
|
|
"unknown": counts["unknown"],
|
|
"succeeded": business_outcomes[operation]["succeeded"],
|
|
"failed": business_outcomes[operation]["failed"],
|
|
"cancelled": business_outcomes[operation]["cancelled"],
|
|
"awaiting_human": business_outcomes[operation]["awaiting_human"],
|
|
"other": business_outcomes[operation]["other"],
|
|
"timeout": provider_outcomes[operation]["timeout"],
|
|
"business_outcomes": business_outcomes[operation],
|
|
"provider_outcomes": provider_outcomes[operation],
|
|
"elapsed_seconds": {
|
|
"observed": len(values),
|
|
"p50": _percentile(values, 0.50),
|
|
"p95": _percentile(values, 0.95),
|
|
},
|
|
"historical": prior,
|
|
"combined": {
|
|
"attempted": counts["attempted"] + prior["attempted"],
|
|
"terminal": counts["terminal"] + prior["terminal"],
|
|
"unknown": counts["unknown"] + prior["unknown"],
|
|
"provider_outcomes": combined_provider,
|
|
"business_outcomes": combined_business,
|
|
"elapsed_seconds": {
|
|
"observed": len(combined_elapsed),
|
|
"p50": _percentile(combined_elapsed, 0.50),
|
|
"p95": _percentile(combined_elapsed, 0.95),
|
|
},
|
|
},
|
|
}
|
|
return result, evidence_errors
|
|
|
|
|
|
def _case_operation_coverage(cases: list[dict], rows: list[dict], ledger: CallLedger) -> list[dict]:
|
|
"""Compare each result with the operations frozen for that exact case.
|
|
|
|
Repair operations are conditional, but must occur as a complete draft/review
|
|
pair. Every other frozen operation is mandatory and no undeclared operation
|
|
can be credited to the case.
|
|
"""
|
|
records_by_case: dict[str, list[dict[str, object]]] = {}
|
|
for record in ledger.attempt_records():
|
|
records_by_case.setdefault(str(record["case_id"]), []).append(record)
|
|
rows_by_case = {row.get("case_id"): row for row in rows}
|
|
results: list[dict] = []
|
|
repair_pair = {"repair_draft", "repair_review"}
|
|
for case in cases:
|
|
case_id = case["case_id"]
|
|
declared = list(case.get("operations") or [])
|
|
declared_set = set(declared)
|
|
actual_records = records_by_case.get(case_id, [])
|
|
actual = [str(record["operation"]) for record in actual_records]
|
|
actual_set = set(actual)
|
|
mandatory = declared_set - repair_pair
|
|
repair_actual = actual_set & repair_pair
|
|
row = rows_by_case.get(case_id) or {}
|
|
calls = row.get("provider_call_count")
|
|
passed = (
|
|
bool(declared)
|
|
and len(declared) == len(declared_set)
|
|
and declared_set <= set(OPERATIONS)
|
|
and actual_set <= declared_set
|
|
and mandatory <= actual_set
|
|
and repair_actual in (set(), repair_pair)
|
|
and len(actual) == len(actual_set)
|
|
and isinstance(calls, int) and not isinstance(calls, bool)
|
|
and calls == len(actual_records)
|
|
and calls > 0
|
|
)
|
|
results.append({
|
|
"case_id": case_id,
|
|
"declared_operations": declared,
|
|
"mandatory_operations": sorted(mandatory),
|
|
"actual_operations": sorted(actual),
|
|
"provider_call_count": calls,
|
|
"passed": passed,
|
|
})
|
|
return results
|
|
|
|
|
|
def _case_elapsed_statistics(rows: list[dict]) -> dict[str, float | int | None]:
|
|
values = [float(row["elapsed_seconds"]) for row in rows
|
|
if isinstance(row.get("elapsed_seconds"), (int, float))
|
|
and not isinstance(row.get("elapsed_seconds"), bool)
|
|
and row["elapsed_seconds"] >= 0]
|
|
return {
|
|
"observed": len(values),
|
|
"p50": _percentile(values, 0.50),
|
|
"p95": _percentile(values, 0.95),
|
|
}
|
|
|
|
|
|
def _current_operation_outcomes_passed(statistics: dict[str, dict]) -> bool:
|
|
for current in statistics.values():
|
|
attempted = current.get("attempted")
|
|
provider = current.get("provider_outcomes")
|
|
business = current.get("business_outcomes")
|
|
if (not isinstance(attempted, int) or isinstance(attempted, bool)
|
|
or current.get("terminal") != attempted
|
|
or current.get("unknown") != 0
|
|
or not isinstance(provider, dict)
|
|
or provider.get("succeeded") != attempted
|
|
or any(provider.get(status) != 0 for status in (
|
|
"failed", "cancelled", "timeout"
|
|
))
|
|
or not isinstance(business, dict)
|
|
or business.get("succeeded") != attempted
|
|
or any(business.get(status) != 0 for status in (
|
|
"failed", "cancelled", "awaiting_human", "other"
|
|
))):
|
|
return False
|
|
return True
|
|
|
|
|
|
def _historical_remediation(out: Path, ledger: CallLedger, rows: list[dict]):
|
|
records = {(record["case_id"], record["operation"]): record
|
|
for record in ledger.attempt_records()}
|
|
cases = {row.get("case_id"): row for row in rows}
|
|
results = []
|
|
for attempt in ledger.historical_usage["attempts"]:
|
|
disposition = attempt["disposition"]
|
|
historically_green = (
|
|
attempt["provider_status"] == "succeeded"
|
|
and attempt["business_status"] in {"succeeded", "resolved"}
|
|
and attempt["timed_out"] is False
|
|
)
|
|
if disposition["release_eligible"] and historically_green:
|
|
results.append({"attempt_id": attempt["attempt_id"],
|
|
"status": "historically_resolved", "passed": True,
|
|
"replacement_case_id": None, "replacement_operation": None})
|
|
continue
|
|
if disposition["release_eligible"]:
|
|
results.append({"attempt_id": attempt["attempt_id"],
|
|
"status": "replacement_required", "passed": False,
|
|
"replacement_case_id": None, "replacement_operation": None,
|
|
"provider_passed": False, "business_passed": False,
|
|
"quality_passed": False,
|
|
"evidence_error": "historical_release_eligibility_invalid"})
|
|
continue
|
|
case_id = disposition["replacement_case_id"]
|
|
operation = disposition["replacement_operation"]
|
|
record = records.get((case_id, operation))
|
|
case = cases.get(case_id) or {}
|
|
evidence = None
|
|
evidence_error = "replacement_not_attempted"
|
|
if record is not None and record["status"] is not None:
|
|
evidence, evidence_error = _operation_after_evidence(
|
|
out, record, real_provider=ledger.real_provider)
|
|
provider_passed = bool(record and record["status"] == "succeeded"
|
|
and evidence and evidence["provider_status"] == "succeeded")
|
|
business_passed = bool(evidence and evidence["business_status"] in {
|
|
"succeeded", "resolved",
|
|
})
|
|
quality_passed = bool(
|
|
case.get("status") == "completed" and case.get("passed") is True
|
|
and case.get("automated_model_quality_release_passed") is True
|
|
and case.get("content_quality_release_passed") is True
|
|
)
|
|
passed = provider_passed and business_passed and quality_passed and evidence_error is None
|
|
results.append({"attempt_id": attempt["attempt_id"],
|
|
"status": "replacement_verified" if passed else "replacement_required",
|
|
"passed": passed, "replacement_case_id": case_id,
|
|
"replacement_operation": operation,
|
|
"provider_passed": provider_passed,
|
|
"business_passed": business_passed,
|
|
"quality_passed": quality_passed,
|
|
"evidence_error": evidence_error})
|
|
return all(item["passed"] for item in results), results
|
|
|
|
|
|
def _source_file(runner, relative: object) -> Path:
|
|
if not isinstance(relative, str) or not relative or "\x00" in relative:
|
|
raise ValueError("source pin path is invalid")
|
|
supplied = Path(relative)
|
|
if supplied.is_absolute() or any(part in {"", ".", ".."} for part in supplied.parts):
|
|
raise ValueError(f"source pin path is not normalized: {relative}")
|
|
root = Path(runner.ROOT).resolve()
|
|
source = (root / supplied).absolute()
|
|
if not source.resolve().is_relative_to(root):
|
|
raise ValueError(f"source pin escapes repository: {relative}")
|
|
current = source
|
|
while True:
|
|
is_junction = getattr(current, "is_junction", lambda: False)
|
|
if current.is_symlink() or is_junction():
|
|
raise ValueError(f"source pin traverses a link or junction: {relative}")
|
|
if current == root:
|
|
break
|
|
if current.parent == current:
|
|
raise ValueError(f"source pin escapes repository: {relative}")
|
|
current = current.parent
|
|
if not source.is_file():
|
|
raise ValueError(f"source pin is not a regular file: {relative}")
|
|
return source
|
|
|
|
|
|
def _file_identity(metadata) -> tuple[int, int, int, int, int]:
|
|
return (
|
|
int(metadata.st_dev), int(metadata.st_ino), int(metadata.st_size),
|
|
int(getattr(metadata, "st_mtime_ns", round(metadata.st_mtime * 1_000_000_000))),
|
|
int(getattr(metadata, "st_ctime_ns", round(metadata.st_ctime * 1_000_000_000))),
|
|
)
|
|
|
|
|
|
def _same_open_path_file(
|
|
descriptor_identity: tuple[int, int, int, int, int],
|
|
path_identity: tuple[int, int, int, int, int],
|
|
) -> bool:
|
|
# Windows can report slightly different ctime values for an open handle and
|
|
# a path lookup. Device/inode plus size/mtime still bind the same object;
|
|
# each view's full fingerprint is compared with itself after execution.
|
|
return descriptor_identity[:4] == path_identity[:4]
|
|
|
|
|
|
@contextmanager
|
|
def _source_pin_execution_guard(runner, source_pins, stage: str):
|
|
"""Hold pinned sources open and require stable bytes and file identity.
|
|
|
|
The before/final summary checks catch persistent drift. This guard also
|
|
detects a file changed and restored while a long provider operation is in
|
|
flight, which would otherwise fit entirely inside that TOCTOU window.
|
|
"""
|
|
opened = []
|
|
try:
|
|
for relative, expected_sha in sorted(source_pins.items()):
|
|
source = _source_file(runner, relative)
|
|
handle = source.open("rb")
|
|
raw = handle.read()
|
|
descriptor_identity = _file_identity(os.fstat(handle.fileno()))
|
|
path_identity = _file_identity(source.lstat())
|
|
if not _same_open_path_file(descriptor_identity, path_identity):
|
|
handle.close()
|
|
raise ValueError(f"source identity drift {stage}: {relative}")
|
|
if runner.sha256(raw) != expected_sha:
|
|
handle.close()
|
|
raise ValueError(f"source drift {stage}: {relative}")
|
|
opened.append((relative, source, expected_sha, handle,
|
|
descriptor_identity, path_identity))
|
|
yield
|
|
for relative, source, expected_sha, handle, descriptor_before, path_before in opened:
|
|
handle.seek(0)
|
|
raw = handle.read()
|
|
descriptor_after = _file_identity(os.fstat(handle.fileno()))
|
|
current_source = _source_file(runner, relative)
|
|
path_after = _file_identity(current_source.lstat())
|
|
if (current_source != source
|
|
or descriptor_after != descriptor_before
|
|
or path_after != path_before
|
|
or not _same_open_path_file(descriptor_after, path_after)
|
|
or runner.sha256(raw) != expected_sha
|
|
or runner.sha256(current_source.read_bytes()) != expected_sha):
|
|
raise ValueError(f"source drift during {stage}: {relative}")
|
|
finally:
|
|
for _relative, _source, _expected, handle, _descriptor, _path in opened:
|
|
handle.close()
|
|
|
|
|
|
def _verify_source_pins(runner, source_pins, stage: str) -> None:
|
|
for relative, source_hash in source_pins.items():
|
|
source = _source_file(runner, relative)
|
|
if runner.sha256(source.read_bytes()) != source_hash:
|
|
raise ValueError(f"source drift {stage}: {relative}")
|
|
|
|
|
|
def _authoritative_registry(runner, supplied) -> Path:
|
|
configured = os.environ.get(REAL_RUN_REGISTRY_ENV, "").strip()
|
|
if not configured:
|
|
raise ValueError(
|
|
f"real RC-05 execution requires {REAL_RUN_REGISTRY_ENV}"
|
|
)
|
|
configured_path = Path(configured)
|
|
if not configured_path.is_absolute():
|
|
raise ValueError(
|
|
f"{REAL_RUN_REGISTRY_ENV} must be an absolute path outside the source worktree"
|
|
)
|
|
expected = configured_path.resolve()
|
|
source_root = Path(runner.ROOT).resolve()
|
|
if expected == source_root or source_root in expected.parents:
|
|
raise ValueError(
|
|
f"{REAL_RUN_REGISTRY_ENV} must resolve outside the source worktree"
|
|
)
|
|
supplied_path = Path(supplied) if supplied is not None else None
|
|
if (supplied_path is None or not supplied_path.is_absolute()
|
|
or supplied_path.resolve() != expected):
|
|
raise ValueError(
|
|
"real RC-05 execution must use the single configured campaign registry "
|
|
f"{expected}"
|
|
)
|
|
if not expected.is_dir():
|
|
raise ValueError("authoritative RC-05 campaign registry must be pre-created")
|
|
return expected
|
|
|
|
|
|
def execute(runner, plan_path, expected_sha, output_dir, factory, *, resume=False,
|
|
case_ids=None, historical_usage=None, execution_mode=None,
|
|
run_registry_dir=None, candidate_inputs=None):
|
|
from multiscene_eval_plan import (
|
|
load_candidate_binding, validate_plan, verify_source_pins_at_candidate,
|
|
)
|
|
from multiscene_supplemental_cases import run_supplemental_case
|
|
|
|
if execution_mode not in {"real", "synthetic"}:
|
|
raise ValueError("execution_mode must explicitly be real or synthetic")
|
|
real = execution_mode == "real"
|
|
if real and factory is not runner.CodexCliDevelopmentProviderAdapter:
|
|
raise ValueError("real execution requires the unwrapped frozen real provider factory")
|
|
if not real and getattr(factory, "_rc05_execution_mode", None) != "synthetic":
|
|
raise ValueError("synthetic execution requires an attested synthetic provider factory")
|
|
if real:
|
|
if not isinstance(candidate_inputs, dict):
|
|
raise ValueError("real RC-05 execution requires complete immutable candidate inputs")
|
|
candidate_binding, source_manifest_index = load_candidate_binding(
|
|
runner, **candidate_inputs, stage="before plan acceptance")
|
|
plan, digest = validate_plan(
|
|
runner, plan_path, expected_sha, candidate_binding,
|
|
source_manifest_index)
|
|
else:
|
|
candidate_binding = source_manifest_index = None
|
|
plan, digest = validate_plan(runner, plan_path, expected_sha)
|
|
inventory = {case["case_id"]: case for case in plan["cases"]}
|
|
selected = list(inventory) if case_ids is None else case_ids
|
|
if not selected or len(selected) != len(set(selected)) or set(selected) - set(inventory):
|
|
raise ValueError("selection must contain unique case IDs from the frozen plan")
|
|
out = Path(output_dir)
|
|
if out.exists() and not resume:
|
|
raise ValueError("fresh output directory required unless --resume is explicit")
|
|
if resume and not out.exists():
|
|
raise ValueError("resume requires the original evidence output directory")
|
|
if resume and not real and not (out / ".call-ledger/identity.json").is_file():
|
|
raise ValueError("resume requires an existing matching call ledger")
|
|
if real:
|
|
authoritative_registry = _authoritative_registry(runner, run_registry_dir)
|
|
if historical_usage is None:
|
|
raise ValueError("real RC-05 execution requires a historical usage receipt")
|
|
if (not isinstance(historical_usage.get("provider_call_count"), int)
|
|
or historical_usage["provider_call_count"]
|
|
< MINIMUM_KNOWN_HISTORICAL_PROVIDER_CALLS
|
|
or historical_usage.get("repair_budget_charged")
|
|
!= min(historical_usage["provider_call_count"], 16)):
|
|
raise ValueError("real RC-05 historical usage declaration is invalid")
|
|
elif historical_usage is not None:
|
|
raise ValueError("historical real-provider usage is not valid for synthetic execution")
|
|
elif run_registry_dir is not None:
|
|
raise ValueError("global run registry is only valid for real execution")
|
|
ledger = CallLedger(out, digest, plan["operation_limits"],
|
|
{key: case["provider_call_budget"] for key, case in inventory.items()},
|
|
plan["source_pins"], real, historical_usage=historical_usage,
|
|
registry_root=authoritative_registry if real else None,
|
|
candidate_binding=plan.get("candidate") if real else None,
|
|
release_controls=plan.get("release_controls") if real else None)
|
|
control_before = None
|
|
if real:
|
|
control_before = ledger.authorize_selection(
|
|
selected, resume=resume, explicit_selection=case_ids is not None)
|
|
runner.write_x(out / f"selection-{uuid4().hex}.json", {
|
|
"plan_raw_sha256": digest, "selected": selected, "resume": resume,
|
|
"historical_usage": historical_usage,
|
|
"started_at": datetime.now(UTC).isoformat()})
|
|
for key in selected:
|
|
snapshot = ledger.snapshot()
|
|
prior = _existing_case(runner, out, inventory[key], snapshot, ledger)
|
|
if prior["status"] != "not_attempted":
|
|
continue
|
|
if real:
|
|
current_binding, current_source_index = load_candidate_binding(
|
|
runner, **candidate_inputs, stage=f"before case {key}")
|
|
if current_binding != candidate_binding:
|
|
raise ValueError("candidate binding drift before next case")
|
|
verify_source_pins_at_candidate(
|
|
runner, plan["source_pins"], current_source_index,
|
|
candidate_binding["commit"])
|
|
# Source drift cannot consume the next case; never update a frozen plan.
|
|
_verify_source_pins(runner, plan["source_pins"], "before next case")
|
|
case, state, started = inventory[key], {"provider_call_count": 0}, perf_counter()
|
|
directory = out / key
|
|
try:
|
|
with _source_pin_execution_guard(
|
|
runner, plan["source_pins"], f"case {key}"
|
|
):
|
|
if case["kind"] == "drafting":
|
|
if case["mode"] == "scene_evaluation":
|
|
result = run_scene_case(
|
|
runner, case, directory, factory, state, ledger=ledger)
|
|
else:
|
|
result = runner.run_case(
|
|
case, directory, factory, state, ledger=ledger)
|
|
else:
|
|
result = run_supplemental_case(case, directory, factory, state,
|
|
ledger=ledger, runner=runner)
|
|
runner.write_x(directory / "result.json", result)
|
|
_bind_case_outcome(runner, out, ledger, case, "result.json")
|
|
if result.get("passed") is not True:
|
|
if real:
|
|
ledger.trip_failure_class(key, "failed", "case result did not pass")
|
|
break
|
|
except Exception as error:
|
|
failure = {"case_id": key, "status": "stopped", "passed": False,
|
|
"error": f"{type(error).__name__}:{error}",
|
|
"provider_call_count": state["provider_call_count"],
|
|
"elapsed_seconds": round(perf_counter() - started, 3),
|
|
"automatic_retry": False, "not_legal_approval": True}
|
|
runner.write_x(directory / "failure.json", failure)
|
|
if real:
|
|
ledger.trip_failure_class(key, "failed", failure["error"][:1000])
|
|
_bind_case_outcome(runner, out, ledger, case, "failure.json")
|
|
break # Do not consume the remainder after a newly observed failure.
|
|
# A drift during the final case must not be hidden by a green summary.
|
|
if real:
|
|
final_binding, final_source_index = load_candidate_binding(
|
|
runner, **candidate_inputs, stage="before final summary")
|
|
if final_binding != candidate_binding:
|
|
raise ValueError("candidate binding drift before final summary")
|
|
verify_source_pins_at_candidate(
|
|
runner, plan["source_pins"], final_source_index,
|
|
candidate_binding["commit"])
|
|
_verify_source_pins(runner, plan["source_pins"], "before final summary")
|
|
snapshot = ledger.snapshot()
|
|
rows = [_existing_case(runner, out, inventory[key], snapshot, ledger) for key in inventory]
|
|
compact = [{key: row[key] for key in (
|
|
"case_id", "status", "passed", "provider_call_count", "elapsed_seconds",
|
|
"automated_quality_passed", "human_calibration_completed",
|
|
"automated_model_quality_release_passed",
|
|
"content_quality_release_passed", "legal_approval_completed",
|
|
)
|
|
if key in row}
|
|
for row in rows]
|
|
complete = all(
|
|
row.get("status") == "completed" and row.get("passed") is True
|
|
for row in compact
|
|
)
|
|
operation_statistics, operation_evidence_errors = _operation_statistics(out, ledger)
|
|
operation_statistics_complete = not operation_evidence_errors
|
|
current_operation_outcomes_passed = _current_operation_outcomes_passed(
|
|
operation_statistics)
|
|
case_operation_coverage = _case_operation_coverage(
|
|
list(inventory.values()), rows, ledger)
|
|
case_operation_coverage_passed = all(
|
|
item["passed"] for item in case_operation_coverage)
|
|
selected_operation_coverage_passed = all(
|
|
item["passed"] for item in case_operation_coverage
|
|
if item["case_id"] in selected)
|
|
campaign = snapshot["campaign"]
|
|
campaign_outcomes = campaign.get("provider_outcomes") or {}
|
|
campaign_release_passed = (
|
|
campaign.get("foreign_plan_attempted") == 0
|
|
and campaign.get("current_plan_attempted") == snapshot["attempted"]
|
|
and campaign.get("attempted") == snapshot["attempted"]
|
|
and campaign.get("terminal") == campaign.get("attempted")
|
|
and campaign.get("unknown") == 0
|
|
and campaign_outcomes.get("succeeded") == campaign.get("attempted")
|
|
and campaign_outcomes.get("failed") == 0
|
|
and campaign_outcomes.get("cancelled") == 0
|
|
)
|
|
matrix_execution_passed = (
|
|
complete and operation_statistics_complete
|
|
and current_operation_outcomes_passed and campaign_release_passed
|
|
and case_operation_coverage_passed
|
|
)
|
|
automated_model_quality_release_passed = all(
|
|
row.get("automated_quality_passed") is True
|
|
and row.get("automated_model_quality_release_passed") is True
|
|
and row.get("content_quality_release_passed") is True
|
|
for row in compact
|
|
)
|
|
selected_release_quality_passed = all(
|
|
row.get("automated_quality_passed") is True
|
|
and row.get("automated_model_quality_release_passed") is True
|
|
and row.get("content_quality_release_passed") is True
|
|
for row in compact if row["case_id"] in selected
|
|
)
|
|
current_run_passed = matrix_execution_passed
|
|
current_selection_passed = (
|
|
operation_statistics_complete and current_operation_outcomes_passed
|
|
and campaign_release_passed and selected_operation_coverage_passed
|
|
and all(row.get("status") == "completed" and row.get("passed") is True
|
|
for row in compact if row["case_id"] in selected)
|
|
)
|
|
historical_release_eligible, historical_remediation = _historical_remediation(
|
|
out, ledger, rows)
|
|
canary_gate_passed = None
|
|
if real and control_before["phase"] == "canary_required":
|
|
canaries = plan["release_controls"]["canary_case_ids"]
|
|
canary_gate_passed = bool(
|
|
selected == canaries
|
|
and current_selection_passed
|
|
and selected_release_quality_passed
|
|
)
|
|
# Persist authorization only after provider/business outcomes, immutable
|
|
# result/evidence bindings, mandatory operation coverage, and every
|
|
# quality gate above have all been evaluated. A failing summary must
|
|
# never leave behind a green transition for the next resume.
|
|
if canary_gate_passed:
|
|
ledger.complete_canary({
|
|
key: runner.sha256((out / key / "result.json").read_bytes())
|
|
for key in canaries
|
|
})
|
|
snapshot = ledger.snapshot()
|
|
overall_release_passed = (matrix_execution_passed
|
|
and automated_model_quality_release_passed
|
|
and historical_release_eligible)
|
|
summary = {"schema_version": "multiscene-evaluation-run.v1",
|
|
"plan_raw_sha256": digest, "real_provider": real,
|
|
"execution_mode": execution_mode,
|
|
"case_count": len(inventory), "drafting_case_count": 120,
|
|
"asset_count": 60, "supplemental_case_count": 24,
|
|
"cases": compact, "complete": complete,
|
|
"passed": overall_release_passed,
|
|
"matrix_execution_passed": matrix_execution_passed,
|
|
"release_quality_gate_passed": automated_model_quality_release_passed,
|
|
"automated_model_quality_release_passed": automated_model_quality_release_passed,
|
|
"human_legal_approval_in_scope": False,
|
|
"current_run_passed": current_run_passed,
|
|
"current_selection_passed": current_selection_passed,
|
|
"historical_release_eligible": historical_release_eligible,
|
|
"historical_remediation": historical_remediation,
|
|
"overall_release_passed": overall_release_passed,
|
|
"selected_release_quality_passed": selected_release_quality_passed,
|
|
"selected_passed": (current_selection_passed
|
|
and selected_release_quality_passed
|
|
and historical_release_eligible),
|
|
# Backward-compatible provider_call_count is current-plan only; the
|
|
# explicit campaign fields are authoritative for the global ceiling.
|
|
"provider_call_count": snapshot["attempted"],
|
|
"provider_call_count_scope": "current_plan",
|
|
"current_plan_provider_call_count": snapshot["attempted"],
|
|
"campaign_new_provider_call_count": snapshot["campaign"]["attempted"],
|
|
"ledger": snapshot,
|
|
"provider_call_count_including_historical": (
|
|
snapshot["campaign"]["attempted"]
|
|
+ snapshot["budget"]["historical_provider_call_count"]
|
|
),
|
|
"historical_usage": historical_usage,
|
|
"operation_statistics": operation_statistics,
|
|
"case_operation_coverage": case_operation_coverage,
|
|
"case_operation_coverage_passed": case_operation_coverage_passed,
|
|
"selected_operation_coverage_passed": selected_operation_coverage_passed,
|
|
"historical_by_operation": {
|
|
operation: statistics["historical"]
|
|
for operation, statistics in operation_statistics.items()
|
|
},
|
|
"combined_call_counts": {
|
|
"attempted": (
|
|
snapshot["campaign"]["attempted"]
|
|
+ snapshot["budget"]["historical_provider_call_count"]
|
|
),
|
|
"terminal": (
|
|
snapshot["campaign"]["terminal"]
|
|
+ snapshot["budget"]["historical_provider_call_count"]
|
|
),
|
|
"unknown": snapshot["campaign"]["unknown"],
|
|
},
|
|
"operation_statistics_complete": operation_statistics_complete,
|
|
"operation_evidence_errors": operation_evidence_errors,
|
|
"case_elapsed_seconds": _case_elapsed_statistics(compact),
|
|
"live_database_writes": 0, "live_service_calls": 0,
|
|
"member_release_acceptance_completed": False, "not_legal_approval": True}
|
|
runner.write_x(out / f"summary-{uuid4().hex}.json", summary)
|
|
return summary
|
|
|
|
|
|
def dispatch(runner, args):
|
|
from multiscene_eval_plan import (
|
|
build_plan, load_candidate_binding, validate_plan,
|
|
verify_source_pins_at_candidate,
|
|
)
|
|
|
|
historical_usage_path = getattr(args, "historical_usage", None)
|
|
run_registry_dir = getattr(args, "run_registry_dir", None)
|
|
if args.inputs is not None:
|
|
raise SystemExit("multiscene mode derives its complete inventory; no legacy input override")
|
|
candidate_inputs = {
|
|
"commit": getattr(args, "candidate_commit", None),
|
|
"tag": getattr(args, "candidate_tag", None),
|
|
"source_manifest_path": getattr(args, "source_manifest", None),
|
|
"source_manifest_sha256": getattr(args, "source_manifest_sha256", None),
|
|
"source_manifest_payload_sha256": getattr(
|
|
args, "source_manifest_payload_sha256", None),
|
|
"image_identities_path": getattr(args, "image_identities", None),
|
|
"image_identities_sha256": getattr(args, "image_identities_sha256", None),
|
|
"image_identity_set_sha256": getattr(
|
|
args, "image_identity_set_sha256", None),
|
|
}
|
|
candidate_supplied = any(value is not None for value in candidate_inputs.values())
|
|
candidate_complete = all(value is not None for value in candidate_inputs.values())
|
|
if candidate_supplied and not candidate_complete:
|
|
raise SystemExit("immutable candidate arguments must be supplied as one complete set")
|
|
if historical_usage_path is not None and not args.execute:
|
|
raise SystemExit("--historical-usage is only valid with --execute")
|
|
if run_registry_dir is not None and not args.execute:
|
|
raise SystemExit("--run-registry-dir is only valid with --execute")
|
|
if args.prepare:
|
|
if args.resume or args.case_id:
|
|
raise SystemExit("prepare freezes the complete matrix, not a selection")
|
|
if not candidate_complete:
|
|
raise SystemExit("RC-05 plan preparation requires a complete built candidate identity")
|
|
binding, source_index = load_candidate_binding(
|
|
runner, **candidate_inputs, stage="before plan preparation")
|
|
plan = build_plan(runner, binding)
|
|
verify_source_pins_at_candidate(
|
|
runner, plan["source_pins"], source_index, binding["commit"])
|
|
runner.write_x(args.plan, plan)
|
|
return {"prepared": True, "case_count": len(plan["cases"]),
|
|
"total_provider_call_budget": plan["total_provider_call_budget"],
|
|
"model_calls": 0, "plan_raw_sha256": runner.sha256(args.plan.read_bytes())}
|
|
if args.synthetic or args.execute:
|
|
if not args.plan_sha256 or args.output_dir is None:
|
|
raise SystemExit("execution requires --plan-sha256 and --output-dir")
|
|
if args.execute:
|
|
factory = runner.CodexCliDevelopmentProviderAdapter
|
|
if historical_usage_path is None:
|
|
raise SystemExit("real RC-05 execution requires --historical-usage")
|
|
if run_registry_dir is None:
|
|
raise SystemExit("real RC-05 execution requires --run-registry-dir")
|
|
historical_usage = load_historical_usage(runner, historical_usage_path)
|
|
execution_mode = "real"
|
|
if not candidate_complete:
|
|
raise SystemExit("real RC-05 execution requires a complete immutable candidate identity")
|
|
else:
|
|
from multiscene_supplemental_cases import synthetic_adapter_factory
|
|
factory = synthetic_adapter_factory(runner)
|
|
historical_usage = None
|
|
execution_mode = "synthetic"
|
|
return execute(runner, args.plan, args.plan_sha256, args.output_dir, factory,
|
|
resume=args.resume, case_ids=args.case_id, historical_usage=historical_usage,
|
|
execution_mode=execution_mode, run_registry_dir=run_registry_dir,
|
|
candidate_inputs=candidate_inputs if args.execute else None)
|
|
if args.resume or args.case_id:
|
|
raise SystemExit("--resume/--case-id require execution mode")
|
|
if candidate_complete:
|
|
binding, source_index = load_candidate_binding(
|
|
runner, **candidate_inputs, stage="during plan validation")
|
|
plan, digest = validate_plan(
|
|
runner, args.plan, args.plan_sha256, binding, source_index)
|
|
else:
|
|
plan, digest = validate_plan(runner, args.plan, args.plan_sha256)
|
|
return {"validated": True, "case_count": len(plan["cases"]),
|
|
"model_calls": 0, "plan_raw_sha256": digest}
|