1421 lines
75 KiB
Python
1421 lines
75 KiB
Python
"""Small local recovery ledger for bounded multi-scene provider runs.
|
|
|
|
SHA-256 values bind frozen inputs for reproducibility; they are not signatures
|
|
and provide no caller authentication or tamper-proof evidence. The ledger
|
|
prevents accidental duplicate calls and coordinates cooperating local runners.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import stat
|
|
from contextlib import contextmanager
|
|
from pathlib import Path
|
|
from typing import Iterator, Mapping
|
|
from uuid import uuid4
|
|
|
|
|
|
OPERATIONS = (
|
|
"draft", "semantic_review", "repair_draft", "repair_review",
|
|
"specialist_review", "intake",
|
|
)
|
|
TERMINAL_STATUSES = frozenset({"succeeded", "failed", "cancelled"})
|
|
MAX_PREALLOCATED_ATTEMPTS = 280
|
|
MAX_REPAIR_ATTEMPTS = 16
|
|
REPAIR_OPERATIONS = frozenset({"repair_draft", "repair_review"})
|
|
REAL_PROVIDER_NAME = "codex-cli-development"
|
|
REAL_PROVIDER_MODEL = "gpt-5.6-sol"
|
|
AFTER_FILES = {
|
|
"draft": frozenset({"01-draft-after.json"}),
|
|
"semantic_review": frozenset({"02-semantic-after.json"}),
|
|
"repair_draft": frozenset({"03-repair-after.json"}),
|
|
"repair_review": frozenset({"04-post-review-after.json"}),
|
|
"specialist_review": frozenset({"02-specialist-after.json", "02-after.json"}),
|
|
"intake": frozenset({"02-after.json"}),
|
|
}
|
|
JOB_OPERATIONS = {
|
|
"contract_draft": frozenset({"draft", "repair_draft"}),
|
|
"contract_draft_semantic_review": frozenset({"semantic_review", "repair_review"}),
|
|
"contract_review": frozenset({"specialist_review"}),
|
|
"contract_intake": frozenset({"intake"}),
|
|
}
|
|
LEDGER_SCHEMA = "multiscene-call-ledger.v2"
|
|
CAMPAIGN_SCHEMA = "multiscene-call-campaign.v2"
|
|
CONTROL_SCHEMA = "multiscene-release-control-state.v1"
|
|
_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z")
|
|
_SHA_RE = re.compile(r"[0-9a-f]{64}\Z")
|
|
|
|
|
|
class CallLedgerError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class LedgerBusyError(CallLedgerError):
|
|
pass
|
|
|
|
|
|
class LedgerIdentityError(CallLedgerError):
|
|
pass
|
|
|
|
|
|
class CallLedger:
|
|
def __init__(self, output_dir: str | Path, plan_sha256: str,
|
|
operation_limits: Mapping[str, int], case_limits: Mapping[str, int],
|
|
source_pins: Mapping[str, str], real_provider: bool,
|
|
historical_usage: Mapping[str, object] | None = None,
|
|
registry_root: str | Path | None = None,
|
|
candidate_binding: Mapping[str, object] | None = None,
|
|
release_controls: Mapping[str, object] | None = None) -> None:
|
|
requested_output = Path(output_dir).absolute()
|
|
_reject_path_chain(requested_output, "output directory")
|
|
self.output_dir = requested_output.resolve()
|
|
self.plan_sha256 = _sha(plan_sha256, "plan_sha256")
|
|
if not isinstance(real_provider, bool):
|
|
raise TypeError("real_provider must be bool")
|
|
self.operation_limits = _limits(operation_limits, operations=True)
|
|
self.case_limits = _limits(case_limits, operations=False)
|
|
if not self.operation_limits or not self.case_limits:
|
|
raise ValueError("operation_limits and case_limits must be non-empty")
|
|
if (sum(self.operation_limits.values()) > MAX_PREALLOCATED_ATTEMPTS
|
|
or sum(self.case_limits.values()) > MAX_PREALLOCATED_ATTEMPTS):
|
|
raise ValueError("preallocated attempt budgets exceed the 280-call safety ceiling")
|
|
if sum(self.operation_limits.get(name, 0) for name in REPAIR_OPERATIONS) > MAX_REPAIR_ATTEMPTS:
|
|
raise ValueError("preallocated repair budgets exceed the 16-call safety ceiling")
|
|
if not isinstance(source_pins, Mapping) or not source_pins:
|
|
raise ValueError("source_pins must be a non-empty mapping")
|
|
self.source_pins: dict[str, str] = {}
|
|
for name, digest in source_pins.items():
|
|
if not isinstance(name, str) or not name or len(name) > 512 or "\x00" in name:
|
|
raise ValueError("source pin names must be bounded strings")
|
|
self.source_pins[name] = _sha(digest, "source pin")
|
|
self.real_provider = real_provider
|
|
self.historical_usage = _historical_usage(historical_usage)
|
|
self.candidate_binding = _json_mapping(candidate_binding, "candidate binding")
|
|
self.release_controls = _release_controls(release_controls, self.case_limits)
|
|
self.controls_enabled = self.release_controls is not None
|
|
# This is deliberately process-local. The same runner instance may
|
|
# advance a declared multi-operation case between immutable provider
|
|
# finishes. A restarted or concurrent runner has no recoverable
|
|
# in-memory workflow state and must therefore treat that gap as unknown.
|
|
self._active_case_ids: set[str] = set()
|
|
if registry_root is not None:
|
|
if not real_provider or historical_usage is None:
|
|
raise ValueError("global registry is only valid for a receipt-bound real run")
|
|
requested_registry = Path(registry_root).absolute()
|
|
_reject_path_chain(requested_registry, "global registry")
|
|
if not requested_registry.is_dir():
|
|
raise ValueError("global registry root must be a pre-created directory")
|
|
self.registry_root = requested_registry.resolve()
|
|
run_key = _digest({
|
|
"plan_sha256": self.plan_sha256,
|
|
"historical_receipt_sha256": self.historical_usage["receipt_sha256"],
|
|
})
|
|
self.root = self.registry_root / run_key
|
|
self.campaign_identity = {
|
|
"schema_version": CAMPAIGN_SCHEMA,
|
|
"campaign_id": "rc05-real-usage-v2",
|
|
"historical_receipt_sha256": self.historical_usage["receipt_sha256"],
|
|
"historical_attempts_sha256": self.historical_usage["attempts_sha256"],
|
|
"historical_provider_call_count": self.historical_usage["provider_call_count"],
|
|
"historical_repair_budget_charged": self.historical_usage["repair_budget_charged"],
|
|
"provider_call_limit": MAX_PREALLOCATED_ATTEMPTS,
|
|
"repair_call_limit": MAX_REPAIR_ATTEMPTS,
|
|
}
|
|
self.campaign_identity_sha256 = _digest(self.campaign_identity)
|
|
self.campaign_attempts = self.registry_root / "campaign-attempts"
|
|
authority = {
|
|
"mode": "global_registry",
|
|
"registry_root": str(self.registry_root),
|
|
"run_key": run_key,
|
|
# A plan/receipt pair has one evidence output. Binding the path
|
|
# in the immutable identity makes a fresh output-dir fork fail.
|
|
"bound_output_dir": str(self.output_dir),
|
|
}
|
|
else:
|
|
self.registry_root = None
|
|
self.campaign_identity = None
|
|
self.campaign_identity_sha256 = None
|
|
self.campaign_attempts = None
|
|
self.root = self.output_dir / ".call-ledger"
|
|
authority = {
|
|
"mode": "output_local",
|
|
"bound_output_dir": str(self.output_dir),
|
|
}
|
|
self.attempts = self.root / "attempts"
|
|
self.case_outcomes = self.root / "case-outcomes"
|
|
self.control_events = self.root / "control-events"
|
|
self.lock_path = ((self.registry_root / ".lock")
|
|
if self.registry_root is not None else self.root / ".lock")
|
|
self.identity = {
|
|
"schema_version": LEDGER_SCHEMA,
|
|
"plan_sha256": self.plan_sha256,
|
|
"operation_limits": {name: self.operation_limits.get(name, 0) for name in OPERATIONS},
|
|
"case_limits": dict(sorted(self.case_limits.items())),
|
|
"source_pins": dict(sorted(self.source_pins.items())),
|
|
"real_provider": real_provider,
|
|
"candidate_binding": self.candidate_binding,
|
|
"release_controls": self.release_controls,
|
|
"authority": authority,
|
|
"integrity_not_authentication": True,
|
|
}
|
|
if historical_usage is not None:
|
|
self.identity["historical_usage"] = self.historical_usage
|
|
self.identity_sha256 = _digest(self.identity)
|
|
if self.registry_root is not None:
|
|
_reject_path_chain(self.registry_root, "global registry")
|
|
_reject_link_or_reparse(self.root, "global run ledger")
|
|
else:
|
|
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
_reject_link_or_reparse(self.output_dir, "output directory")
|
|
_reject_link_or_reparse(self.root, "ledger directory")
|
|
self.root.mkdir(exist_ok=True)
|
|
_reject_link_or_reparse(self.root, "ledger directory")
|
|
with self._locked():
|
|
if self.registry_root is not None:
|
|
campaign_identity_path = self.registry_root / "campaign-identity.json"
|
|
_reject_link_or_reparse(campaign_identity_path, "campaign identity")
|
|
if campaign_identity_path.exists():
|
|
stored_campaign = _read_json(campaign_identity_path)
|
|
if stored_campaign.get("schema_version") != CAMPAIGN_SCHEMA:
|
|
raise LedgerIdentityError(
|
|
"legacy campaign schema is read-only diagnostic material; execution refused"
|
|
)
|
|
if stored_campaign != self.campaign_identity:
|
|
raise LedgerIdentityError(
|
|
"global campaign is already bound to a different historical receipt"
|
|
)
|
|
if (not self.campaign_attempts.is_dir()
|
|
or self.campaign_attempts.is_symlink()):
|
|
raise LedgerIdentityError("global campaign attempt journal is missing")
|
|
else:
|
|
unexpected = [entry.name for entry in self.registry_root.iterdir()
|
|
if entry.name != ".lock"]
|
|
if unexpected:
|
|
raise LedgerIdentityError(
|
|
"uninitialized global campaign registry must be empty"
|
|
)
|
|
_write_x(campaign_identity_path, self.campaign_identity)
|
|
self.campaign_attempts.mkdir()
|
|
_reject_link_or_reparse(self.campaign_attempts, "campaign attempt directory")
|
|
_reject_link_or_reparse(self.root, "global run ledger")
|
|
self.root.mkdir(exist_ok=True)
|
|
_reject_link_or_reparse(self.root, "global run ledger")
|
|
identity_path = self.root / "identity.json"
|
|
_reject_link_or_reparse(identity_path, "ledger identity")
|
|
if identity_path.exists():
|
|
stored_identity = _read_json(identity_path)
|
|
if stored_identity.get("schema_version") != LEDGER_SCHEMA:
|
|
raise LedgerIdentityError(
|
|
"legacy ledger schema is read-only diagnostic material; execution refused"
|
|
)
|
|
if stored_identity != self.identity:
|
|
raise LedgerIdentityError("ledger identity does not match the frozen run")
|
|
else:
|
|
_write_x(identity_path, self.identity)
|
|
_reject_link_or_reparse(self.attempts, "attempt directory")
|
|
self.attempts.mkdir(exist_ok=True)
|
|
_reject_link_or_reparse(self.attempts, "attempt directory")
|
|
_reject_link_or_reparse(self.case_outcomes, "case outcome directory")
|
|
self.case_outcomes.mkdir(exist_ok=True)
|
|
_reject_link_or_reparse(self.case_outcomes, "case outcome directory")
|
|
_reject_link_or_reparse(self.control_events, "control event directory")
|
|
self.control_events.mkdir(exist_ok=True)
|
|
_reject_link_or_reparse(self.control_events, "control event directory")
|
|
# Create the output only after the global identity accepted this exact
|
|
# path. A competing fresh output therefore cannot leave a fork behind.
|
|
self.output_dir.mkdir(parents=True, exist_ok=True)
|
|
_reject_link_or_reparse(self.output_dir, "output directory")
|
|
|
|
def control_snapshot(self) -> dict[str, object]:
|
|
with self._locked():
|
|
self._materialize_attempt_breakers_unlocked()
|
|
return self._control_snapshot_unlocked()
|
|
|
|
def authorize_selection(self, selected: list[str], *, resume: bool,
|
|
explicit_selection: bool) -> dict[str, object]:
|
|
"""Fail closed before dispatch unless persisted release controls permit it."""
|
|
if not self.controls_enabled:
|
|
raise CallLedgerError("release controls are required for controlled execution")
|
|
if (not isinstance(selected, list) or not selected
|
|
or len(selected) != len(set(selected))
|
|
or any(case_id not in self.case_limits for case_id in selected)):
|
|
raise CallLedgerError("selection is outside frozen release controls")
|
|
with self._locked():
|
|
self._materialize_attempt_breakers_unlocked()
|
|
state = self._control_snapshot_unlocked()
|
|
canaries = self.release_controls["canary_case_ids"]
|
|
if state["phase"] == "canary_required":
|
|
if not explicit_selection or selected != canaries:
|
|
raise CallLedgerError(
|
|
"canary phase requires the explicit predefined canary case sequence"
|
|
)
|
|
if state["blocked_failure_classes"]:
|
|
raise CallLedgerError("canary phase is blocked by a persisted failure class")
|
|
records = self._records()
|
|
blocked = set(state["blocked_failure_classes"])
|
|
future = [case_id for case_id in selected
|
|
if not any(key[0] == case_id for key in records)]
|
|
forbidden = sorted({self.release_controls["failure_classes"][case_id]
|
|
for case_id in future} & blocked)
|
|
if forbidden:
|
|
raise CallLedgerError(
|
|
f"selection contains persisted blocked failure classes: {forbidden}"
|
|
)
|
|
return state
|
|
|
|
def complete_canary(self, outcome_sha256s: Mapping[str, str]) -> None:
|
|
"""Persist the single immutable transition that authorizes full dispatch."""
|
|
if not self.controls_enabled:
|
|
raise CallLedgerError("release controls are not configured")
|
|
canaries = self.release_controls["canary_case_ids"]
|
|
if not isinstance(outcome_sha256s, Mapping) or set(outcome_sha256s) != set(canaries):
|
|
raise CallLedgerError("canary outcome set is incomplete")
|
|
normalized = {case_id: _sha(outcome_sha256s[case_id], "canary outcome SHA-256")
|
|
for case_id in canaries}
|
|
with self._locked():
|
|
self._materialize_attempt_breakers_unlocked()
|
|
state = self._control_snapshot_unlocked()
|
|
if state["blocked_failure_classes"]:
|
|
raise CallLedgerError("failed or unknown canary cannot authorize full dispatch")
|
|
records = self._records()
|
|
binding_sha256s: dict[str, str] = {}
|
|
for case_id in canaries:
|
|
relevant = [record for (current, _operation), record in records.items()
|
|
if current == case_id]
|
|
if not relevant or any(record["finish"] is None
|
|
or record["finish"]["status"] != "succeeded"
|
|
for record in relevant):
|
|
raise CallLedgerError("canary provider attempts are incomplete or unsuccessful")
|
|
binding = self._validated_persisted_outcome_unlocked(
|
|
case_id, records, require_success=True)
|
|
if binding.get("outcome_sha256") != normalized[case_id]:
|
|
raise CallLedgerError("canary outcome does not match immutable binding")
|
|
binding_sha256s[case_id] = _digest(binding)
|
|
payload = {"schema_version": CONTROL_SCHEMA,
|
|
"event": "canary_to_full", "ledger_identity_sha256": self.identity_sha256,
|
|
"canary_case_ids": canaries, "outcome_sha256s": normalized,
|
|
"case_outcome_binding_sha256s": binding_sha256s}
|
|
transition = self.control_events / "canary-to-full.json"
|
|
if transition.exists():
|
|
if _read_json(transition) != payload:
|
|
raise CallLedgerError("canary transition is already bound differently")
|
|
else:
|
|
_write_x(transition, payload)
|
|
|
|
def trip_failure_class(self, case_id: str, status: str, reason: str) -> None:
|
|
if not self.controls_enabled:
|
|
return
|
|
case_id = _identifier(case_id, "failure case_id")
|
|
if case_id not in self.case_limits or status not in {"failed", "cancelled", "unknown"}:
|
|
raise CallLedgerError("failure circuit event is invalid")
|
|
failure_class = self.release_controls["failure_classes"][case_id]
|
|
if not isinstance(reason, str) or not reason or len(reason) > 1000:
|
|
raise CallLedgerError("failure circuit reason is invalid")
|
|
payload = {"schema_version": CONTROL_SCHEMA, "event": "failure_class_blocked",
|
|
"ledger_identity_sha256": self.identity_sha256, "failure_class": failure_class,
|
|
"case_id": case_id, "status": status, "reason": reason}
|
|
path = self.control_events / f"failure-class--{failure_class}.json"
|
|
with self._locked():
|
|
if path.exists():
|
|
existing = _read_json(path)
|
|
if existing.get("failure_class") != failure_class:
|
|
raise CallLedgerError("failure circuit event is inconsistent")
|
|
else:
|
|
_write_x(path, payload)
|
|
|
|
def _materialize_attempt_breakers_unlocked(
|
|
self,
|
|
*,
|
|
progress_case_id: str | None = None,
|
|
progress_operation: str | None = None,
|
|
) -> None:
|
|
if not self.controls_enabled:
|
|
return
|
|
records = self._records()
|
|
cases = sorted({case_id for case_id, _operation in records})
|
|
for case_id in cases:
|
|
relevant = [record for (current, _operation), record in records.items()
|
|
if current == case_id]
|
|
expected_operations = self.release_controls["expected_operations"][case_id]
|
|
actual_operations = {
|
|
operation for current, operation in records if current == case_id
|
|
}
|
|
unfinished = any(record["finish"] is None for record in relevant)
|
|
unsuccessful = next((record["finish"]["status"] for record in relevant
|
|
if record["finish"] is not None
|
|
and record["finish"]["status"] != "succeeded"), None)
|
|
if unfinished:
|
|
status, reason = "unknown", "ledger has an unfinished provider attempt"
|
|
elif unsuccessful is not None:
|
|
status, reason = unsuccessful, f"provider attempt finished as {unsuccessful}"
|
|
elif actual_operations - set(expected_operations):
|
|
status, reason = "unknown", "case contains an operation outside frozen controls"
|
|
elif (case_id == progress_case_id
|
|
and progress_operation in expected_operations
|
|
and progress_operation not in actual_operations
|
|
and not (self.case_outcomes / f"{case_id}.json").exists()):
|
|
# Only the same live runner instance may cross the small
|
|
# finish->next-claim gap. A restart cannot reconstruct the
|
|
# isolated case workflow, so it must materialize an unknown
|
|
# breaker rather than silently continue or repeat a call.
|
|
continue
|
|
else:
|
|
try:
|
|
self._validated_persisted_outcome_unlocked(
|
|
case_id, records, require_success=True)
|
|
except (OSError, UnicodeError, TypeError, ValueError,
|
|
RuntimeError, CallLedgerError) as error:
|
|
status = "unknown"
|
|
reason = f"successful provider attempt has invalid case outcome: {error}"
|
|
else:
|
|
continue
|
|
failure_class = self.release_controls["failure_classes"][case_id]
|
|
path = self.control_events / f"failure-class--{failure_class}.json"
|
|
if not path.exists():
|
|
_write_x(path, {"schema_version": CONTROL_SCHEMA,
|
|
"event": "failure_class_blocked",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"failure_class": failure_class, "case_id": case_id,
|
|
"status": status, "reason": reason[:1000]})
|
|
|
|
def _control_snapshot_unlocked(self) -> dict[str, object]:
|
|
if not self.controls_enabled:
|
|
return {"schema_version": CONTROL_SCHEMA, "phase": "not_configured",
|
|
"canary_case_ids": [], "blocked_failure_classes": [],
|
|
"canary_transition_sha256": None}
|
|
blocked: list[str] = []
|
|
transition_sha = None
|
|
records = self._records()
|
|
for path in self.control_events.iterdir():
|
|
_reject_link_or_reparse(path, "control event")
|
|
if not path.is_file():
|
|
raise CallLedgerError("control event journal contains a non-file entry")
|
|
payload = _read_json(path)
|
|
if (not isinstance(payload, dict)
|
|
or payload.get("schema_version") != CONTROL_SCHEMA
|
|
or payload.get("ledger_identity_sha256") != self.identity_sha256):
|
|
raise CallLedgerError("control event is invalid")
|
|
if path.name == "canary-to-full.json" and payload.get("event") == "canary_to_full":
|
|
expected_keys = {
|
|
"schema_version", "event", "ledger_identity_sha256",
|
|
"canary_case_ids", "outcome_sha256s",
|
|
"case_outcome_binding_sha256s",
|
|
}
|
|
canaries = self.release_controls["canary_case_ids"]
|
|
outcomes = payload.get("outcome_sha256s")
|
|
bindings = payload.get("case_outcome_binding_sha256s")
|
|
if (set(payload) != expected_keys
|
|
or payload.get("canary_case_ids") != canaries
|
|
or not isinstance(outcomes, dict) or set(outcomes) != set(canaries)
|
|
or not isinstance(bindings, dict) or set(bindings) != set(canaries)):
|
|
raise CallLedgerError("canary transition shape or inventory is invalid")
|
|
for case_id in canaries:
|
|
outcome_sha = _sha(outcomes[case_id], "canary transition outcome SHA-256")
|
|
binding_sha = _sha(
|
|
bindings[case_id], "canary transition binding SHA-256")
|
|
binding = self._validated_persisted_outcome_unlocked(
|
|
case_id, records, require_success=True)
|
|
if (binding["outcome_sha256"] != outcome_sha
|
|
or _digest(binding) != binding_sha):
|
|
raise CallLedgerError(
|
|
"canary transition differs from immutable outcome binding")
|
|
transition_sha = _digest(payload)
|
|
elif (re.fullmatch(r"failure-class--[A-Za-z0-9._-]+\.json", path.name)
|
|
and payload.get("event") == "failure_class_blocked"):
|
|
expected_keys = {
|
|
"schema_version", "event", "ledger_identity_sha256",
|
|
"failure_class", "case_id", "status", "reason",
|
|
}
|
|
if set(payload) != expected_keys:
|
|
raise CallLedgerError("failure-class control event shape is invalid")
|
|
failure_class = _identifier(payload.get("failure_class"), "failure class")
|
|
case_id = _identifier(payload.get("case_id"), "failure event case ID")
|
|
expected_name = f"failure-class--{failure_class}.json"
|
|
if (path.name != expected_name or case_id not in self.case_limits
|
|
or self.release_controls["failure_classes"][case_id] != failure_class
|
|
or payload.get("status") not in {"failed", "cancelled", "unknown"}
|
|
or not isinstance(payload.get("reason"), str)
|
|
or not payload["reason"] or len(payload["reason"]) > 1000
|
|
or failure_class not in set(
|
|
self.release_controls["failure_classes"].values())):
|
|
raise CallLedgerError("control event failure class is outside frozen controls")
|
|
blocked.append(failure_class)
|
|
else:
|
|
raise CallLedgerError("control event journal contains an unknown entry")
|
|
return {"schema_version": CONTROL_SCHEMA,
|
|
"phase": "full_authorized" if transition_sha else "canary_required",
|
|
"canary_case_ids": self.release_controls["canary_case_ids"],
|
|
"blocked_failure_classes": sorted(set(blocked)),
|
|
"canary_transition_sha256": transition_sha}
|
|
|
|
def claim(self, case_id: str, operation: str, request_sha256: str) -> str:
|
|
case_id = _identifier(case_id, "case_id")
|
|
if case_id not in self.case_limits:
|
|
raise CallLedgerError("case is not in the frozen ledger limits")
|
|
if operation not in OPERATIONS or operation not in self.operation_limits:
|
|
raise CallLedgerError("operation is not enabled for this ledger")
|
|
if (self.controls_enabled
|
|
and operation not in self.release_controls["expected_operations"][case_id]):
|
|
raise CallLedgerError("operation is outside frozen controls for this case")
|
|
request_sha256 = _sha(request_sha256, "request_sha256")
|
|
stem = f"{case_id}--{operation}"
|
|
with self._locked():
|
|
if self.controls_enabled:
|
|
self._materialize_attempt_breakers_unlocked(
|
|
progress_case_id=(case_id if case_id in self._active_case_ids else None),
|
|
progress_operation=operation,
|
|
)
|
|
state = self._control_snapshot_unlocked()
|
|
failure_class = self.release_controls["failure_classes"][case_id]
|
|
if failure_class in state["blocked_failure_classes"]:
|
|
raise CallLedgerError("failure-class circuit breaker is open")
|
|
if (state["phase"] == "canary_required"
|
|
and case_id not in self.release_controls["canary_case_ids"]):
|
|
raise CallLedgerError("full dispatch is forbidden before canary transition")
|
|
records = self._records()
|
|
campaign_records = self._campaign_records() if self.registry_root is not None else records
|
|
if (case_id, operation) in records:
|
|
raise CallLedgerError("case operation was already attempted")
|
|
operation_count = sum(1 for case, op in records if op == operation)
|
|
case_count = sum(1 for case, _op in records if case == case_id)
|
|
if len(campaign_records) + self.historical_usage["provider_call_count"] >= MAX_PREALLOCATED_ATTEMPTS:
|
|
raise CallLedgerError("global attempt budget including historical usage is exhausted")
|
|
if operation in REPAIR_OPERATIONS:
|
|
repair_count = sum(
|
|
1 for record in campaign_records.values()
|
|
if record["claim"]["operation"] in REPAIR_OPERATIONS
|
|
)
|
|
if repair_count + self.historical_usage["repair_budget_charged"] >= MAX_REPAIR_ATTEMPTS:
|
|
raise CallLedgerError("repair attempt budget including historical usage is exhausted")
|
|
if operation_count >= self.operation_limits[operation]:
|
|
raise CallLedgerError("operation attempt budget is exhausted")
|
|
if case_count >= self.case_limits[case_id]:
|
|
raise CallLedgerError("case attempt budget is exhausted")
|
|
claim_id = uuid4().hex
|
|
claim_payload = {
|
|
"schema_version": "multiscene-call-claim.v1",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"claim_id": claim_id,
|
|
"case_id": case_id,
|
|
"operation": operation,
|
|
"request_sha256": request_sha256,
|
|
"attempt_number": 1,
|
|
}
|
|
if self.registry_root is not None:
|
|
campaign_key = _digest({"plan_sha256": self.plan_sha256,
|
|
"case_id": case_id, "operation": operation})
|
|
_write_x(self.campaign_attempts / f"{campaign_key}.claim.json", {
|
|
"schema_version": "multiscene-campaign-claim.v1",
|
|
"campaign_identity_sha256": self.campaign_identity_sha256,
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"claim_id": claim_id, "plan_sha256": self.plan_sha256,
|
|
"case_id": case_id, "operation": operation,
|
|
"request_sha256": request_sha256,
|
|
})
|
|
_write_x(self.attempts / f"{stem}.claim.json", claim_payload)
|
|
self._active_case_ids.add(case_id)
|
|
return claim_id
|
|
|
|
def finish(self, token: str, status: str, response_sha256: str | None = None) -> None:
|
|
if not isinstance(token, str) or not re.fullmatch(r"[0-9a-f]{32}", token):
|
|
raise CallLedgerError("invalid claim token")
|
|
if status not in TERMINAL_STATUSES:
|
|
raise CallLedgerError("invalid terminal status")
|
|
if response_sha256 is not None:
|
|
response_sha256 = _sha(response_sha256, "response_sha256")
|
|
with self._locked():
|
|
records = self._records()
|
|
matches = [(key, value) for key, value in records.items()
|
|
if value["claim"]["claim_id"] == token]
|
|
if len(matches) != 1:
|
|
raise CallLedgerError("claim token is unknown")
|
|
(case_id, operation), record = matches[0]
|
|
if record["finish"] is not None:
|
|
raise CallLedgerError("claim already has an immutable terminal record")
|
|
finish_payload = {
|
|
"schema_version": "multiscene-call-finish.v1",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"claim_id": token,
|
|
"case_id": case_id,
|
|
"operation": operation,
|
|
"status": status,
|
|
"response_sha256": response_sha256,
|
|
}
|
|
if self.registry_root is not None:
|
|
campaign_records = self._campaign_records()
|
|
campaign_match = [item for item in campaign_records.values()
|
|
if item["claim"]["claim_id"] == token]
|
|
if len(campaign_match) != 1 or campaign_match[0]["finish"] is not None:
|
|
raise CallLedgerError("global campaign claim is missing or already finished")
|
|
campaign_key = _digest({"plan_sha256": self.plan_sha256,
|
|
"case_id": case_id, "operation": operation})
|
|
_write_x(self.campaign_attempts / f"{campaign_key}.finish.json", {
|
|
"schema_version": "multiscene-campaign-finish.v1",
|
|
"campaign_identity_sha256": self.campaign_identity_sha256,
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"claim_id": token, "plan_sha256": self.plan_sha256,
|
|
"case_id": case_id, "operation": operation,
|
|
"status": status, "response_sha256": response_sha256,
|
|
})
|
|
_write_x(self.attempts / f"{case_id}--{operation}.finish.json", finish_payload)
|
|
|
|
def snapshot(self) -> dict:
|
|
with self._locked():
|
|
self._materialize_attempt_breakers_unlocked()
|
|
records = self._records()
|
|
successful_case_evidence: dict[str, bool] = {}
|
|
for case_id in self.case_limits if self.controls_enabled else ():
|
|
relevant = [record for (current, _operation), record in records.items()
|
|
if current == case_id]
|
|
if not relevant or any(record["finish"] is None
|
|
or record["finish"]["status"] != "succeeded"
|
|
for record in relevant):
|
|
continue
|
|
try:
|
|
self._validated_persisted_outcome_unlocked(
|
|
case_id, records, require_success=True)
|
|
except (OSError, UnicodeError, TypeError, ValueError,
|
|
RuntimeError, CallLedgerError):
|
|
successful_case_evidence[case_id] = False
|
|
else:
|
|
successful_case_evidence[case_id] = True
|
|
|
|
def evidence_terminal(case_id: str, record: dict) -> bool:
|
|
finish = record["finish"]
|
|
if finish is None:
|
|
return False
|
|
if not self.controls_enabled:
|
|
return True
|
|
if finish["status"] != "succeeded":
|
|
return True
|
|
return successful_case_evidence.get(case_id, False)
|
|
|
|
operations = {}
|
|
for operation in OPERATIONS:
|
|
relevant = [(case, record) for (case, op), record in records.items()
|
|
if op == operation]
|
|
terminal = sum(evidence_terminal(case, record)
|
|
for case, record in relevant)
|
|
operations[operation] = {
|
|
"limit": self.operation_limits.get(operation, 0),
|
|
"attempted": len(relevant), "terminal": terminal,
|
|
"unknown": len(relevant) - terminal,
|
|
}
|
|
cases = {}
|
|
for case_id, limit in sorted(self.case_limits.items()):
|
|
relevant = [record for (case, _op), record in records.items() if case == case_id]
|
|
terminal = sum(evidence_terminal(case_id, record) for record in relevant)
|
|
cases[case_id] = {"limit": limit, "attempted": len(relevant),
|
|
"terminal": terminal, "unknown": len(relevant) - terminal}
|
|
attempted = len(records)
|
|
provider_terminal = sum(
|
|
record["finish"] is not None for record in records.values())
|
|
terminal = sum(evidence_terminal(case_id, record)
|
|
for (case_id, _operation), record in records.items())
|
|
campaign_records = self._campaign_records() if self.registry_root is not None else records
|
|
campaign_attempted = len(campaign_records)
|
|
campaign_provider_terminal = sum(
|
|
record["finish"] is not None for record in campaign_records.values())
|
|
|
|
def campaign_evidence_terminal(record: dict) -> bool:
|
|
finish = record["finish"]
|
|
if finish is None:
|
|
return False
|
|
if finish["status"] != "succeeded":
|
|
return True
|
|
claim = record["claim"]
|
|
if self.registry_root is None:
|
|
return evidence_terminal(claim["case_id"], record)
|
|
if claim["plan_sha256"] != self.plan_sha256:
|
|
# Foreign-plan attempts already fail the current candidate
|
|
# gate and their output directory is deliberately not read.
|
|
return True
|
|
local = records.get((claim["case_id"], claim["operation"]))
|
|
return bool(local is not None
|
|
and evidence_terminal(claim["case_id"], local))
|
|
|
|
campaign_terminal = sum(
|
|
campaign_evidence_terminal(record)
|
|
for record in campaign_records.values())
|
|
if self.registry_root is not None:
|
|
current_plan_attempted = sum(
|
|
record["claim"]["plan_sha256"] == self.plan_sha256
|
|
for record in campaign_records.values()
|
|
)
|
|
campaign_provider_outcomes = {
|
|
status: sum(
|
|
record["finish"] is not None
|
|
and record["finish"]["status"] == status
|
|
for record in campaign_records.values()
|
|
)
|
|
for status in ("succeeded", "failed", "cancelled")
|
|
}
|
|
else:
|
|
current_plan_attempted = campaign_attempted
|
|
campaign_provider_outcomes = {
|
|
status: sum(
|
|
record["finish"] is not None
|
|
and record["finish"]["status"] == status
|
|
for record in campaign_records.values()
|
|
)
|
|
for status in ("succeeded", "failed", "cancelled")
|
|
}
|
|
repair_attempted = sum(
|
|
1 for record in campaign_records.values()
|
|
if record["claim"]["operation"] in REPAIR_OPERATIONS)
|
|
controls = self._control_snapshot_unlocked()
|
|
return {"schema_version": "multiscene-call-ledger-snapshot.v2",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"real_provider": self.real_provider,
|
|
"authority": self.identity["authority"],
|
|
"budget": {
|
|
"provider_call_limit": MAX_PREALLOCATED_ATTEMPTS,
|
|
"historical_provider_call_count": self.historical_usage["provider_call_count"],
|
|
"new_provider_call_limit": (
|
|
MAX_PREALLOCATED_ATTEMPTS
|
|
- self.historical_usage["provider_call_count"]
|
|
),
|
|
"new_provider_calls_remaining": max(
|
|
0,
|
|
MAX_PREALLOCATED_ATTEMPTS
|
|
- self.historical_usage["provider_call_count"]
|
|
- campaign_attempted,
|
|
),
|
|
"repair_call_limit": MAX_REPAIR_ATTEMPTS,
|
|
"historical_repair_budget_charged": self.historical_usage[
|
|
"repair_budget_charged"
|
|
],
|
|
"new_repair_call_limit": (
|
|
MAX_REPAIR_ATTEMPTS
|
|
- self.historical_usage["repair_budget_charged"]
|
|
),
|
|
"new_repair_calls_remaining": max(
|
|
0,
|
|
MAX_REPAIR_ATTEMPTS
|
|
- self.historical_usage["repair_budget_charged"]
|
|
- repair_attempted,
|
|
),
|
|
"historical_attempts_sha256": self.historical_usage[
|
|
"attempts_sha256"
|
|
],
|
|
"historical_receipt_sha256": self.historical_usage[
|
|
"receipt_sha256"
|
|
],
|
|
},
|
|
"operations": operations, "cases": cases,
|
|
"campaign": {"attempted": campaign_attempted,
|
|
"terminal": campaign_terminal,
|
|
"unknown": campaign_attempted - campaign_terminal,
|
|
"provider_terminal": campaign_provider_terminal,
|
|
"current_plan_attempted": current_plan_attempted,
|
|
"foreign_plan_attempted": (
|
|
campaign_attempted - current_plan_attempted
|
|
),
|
|
"provider_outcomes": campaign_provider_outcomes},
|
|
"attempted": attempted,
|
|
"terminal": terminal,
|
|
"unknown": attempted - terminal,
|
|
"provider_terminal": provider_terminal,
|
|
"release_controls": controls}
|
|
|
|
def attempt_records(self) -> list[dict[str, object]]:
|
|
"""Return immutable attempt outcomes for evidence aggregation."""
|
|
with self._locked():
|
|
records = self._records()
|
|
return [
|
|
{
|
|
"case_id": case_id,
|
|
"operation": operation,
|
|
"request_sha256": record["claim"]["request_sha256"],
|
|
"response_sha256": (
|
|
record["finish"]["response_sha256"]
|
|
if record["finish"] is not None else None
|
|
),
|
|
"status": (
|
|
record["finish"]["status"] if record["finish"] is not None else None
|
|
),
|
|
}
|
|
for (case_id, operation), record in sorted(records.items())
|
|
]
|
|
|
|
def bind_case_outcome(
|
|
self,
|
|
case_id: str,
|
|
outcome_file: str,
|
|
outcome_sha256: str,
|
|
evidence_sha256s: Mapping[str, str],
|
|
expected_operations: list[str] | tuple[str, ...],
|
|
) -> None:
|
|
"""Bind a terminal case result to its exact attempts and evidence files.
|
|
|
|
Provider finishes are written before the business/result artifact exists.
|
|
This separate immutable record prevents a later resume from trusting a
|
|
replaced ``result.json`` or its operation evidence without another call.
|
|
"""
|
|
case_id = _identifier(case_id, "case outcome case_id")
|
|
if case_id not in self.case_limits:
|
|
raise CallLedgerError("case outcome is outside frozen limits")
|
|
if outcome_file not in {"result.json", "failure.json"}:
|
|
raise CallLedgerError("case outcome file is invalid")
|
|
outcome_sha256 = _sha(outcome_sha256, "case outcome SHA-256")
|
|
evidence = _evidence_hashes(evidence_sha256s)
|
|
expected = _operation_projection(expected_operations)
|
|
if (self.controls_enabled
|
|
and expected != self.release_controls["expected_operations"][case_id]):
|
|
raise CallLedgerError("case outcome operations differ from frozen controls")
|
|
with self._locked():
|
|
records = self._records()
|
|
projection = _case_attempt_projection(records, case_id)
|
|
if any(item["status"] is None for item in projection):
|
|
raise CallLedgerError("case outcome cannot bind an unfinished provider attempt")
|
|
payload = {
|
|
"schema_version": "multiscene-case-outcome.v2",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"case_id": case_id,
|
|
"outcome_file": outcome_file,
|
|
"outcome_sha256": outcome_sha256,
|
|
"expected_operations": expected,
|
|
"case_operation_projection": projection,
|
|
"case_attempts_sha256": _digest(projection),
|
|
"evidence_sha256s": evidence,
|
|
}
|
|
_write_x(self.case_outcomes / f"{case_id}.json", payload)
|
|
self._active_case_ids.discard(case_id)
|
|
|
|
def verify_case_outcome(
|
|
self,
|
|
case_id: str,
|
|
outcome_file: str,
|
|
outcome_sha256: str,
|
|
evidence_sha256s: Mapping[str, str],
|
|
expected_operations: list[str] | tuple[str, ...],
|
|
) -> None:
|
|
"""Fail closed unless a case artifact still matches its immutable binding."""
|
|
case_id = _identifier(case_id, "case outcome case_id")
|
|
if outcome_file not in {"result.json", "failure.json"}:
|
|
raise CallLedgerError("case outcome file is invalid")
|
|
outcome_sha256 = _sha(outcome_sha256, "case outcome SHA-256")
|
|
evidence = _evidence_hashes(evidence_sha256s)
|
|
declared_operations = _operation_projection(expected_operations)
|
|
if (self.controls_enabled
|
|
and declared_operations
|
|
!= self.release_controls["expected_operations"][case_id]):
|
|
raise CallLedgerError("case outcome operations differ from frozen controls")
|
|
with self._locked():
|
|
path = self.case_outcomes / f"{case_id}.json"
|
|
_reject_link_or_reparse(path, "case outcome binding")
|
|
if not path.is_file():
|
|
raise CallLedgerError("case outcome has no immutable binding")
|
|
payload = _read_json(path)
|
|
expected_keys = {
|
|
"schema_version", "ledger_identity_sha256", "case_id",
|
|
"outcome_file", "outcome_sha256", "case_attempts_sha256",
|
|
"expected_operations", "case_operation_projection",
|
|
"evidence_sha256s",
|
|
}
|
|
if not isinstance(payload, dict) or set(payload) != expected_keys:
|
|
raise CallLedgerError("case outcome binding shape is invalid")
|
|
records = self._records()
|
|
projection = _case_attempt_projection(records, case_id)
|
|
expected = {
|
|
"schema_version": "multiscene-case-outcome.v2",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"case_id": case_id,
|
|
"outcome_file": outcome_file,
|
|
"outcome_sha256": outcome_sha256,
|
|
"expected_operations": declared_operations,
|
|
"case_operation_projection": projection,
|
|
"case_attempts_sha256": _digest(projection),
|
|
"evidence_sha256s": evidence,
|
|
}
|
|
if payload != expected:
|
|
raise CallLedgerError("case outcome or evidence changed after binding")
|
|
|
|
def _validated_persisted_outcome_unlocked(
|
|
self,
|
|
case_id: str,
|
|
records: Mapping[tuple[str, str], dict],
|
|
*,
|
|
require_success: bool,
|
|
) -> dict[str, object]:
|
|
"""Validate a persisted case outcome and every file it binds.
|
|
|
|
This is intentionally independent of the caller-provided hashes used by
|
|
``verify_case_outcome``. Resume authorization must rediscover the files
|
|
from the ledger and fail closed after a crash or later mutation.
|
|
"""
|
|
case_id = _identifier(case_id, "persisted outcome case ID")
|
|
binding_path = self.case_outcomes / f"{case_id}.json"
|
|
_reject_link_or_reparse(binding_path, "case outcome binding")
|
|
if not binding_path.is_file():
|
|
raise CallLedgerError("case outcome binding is missing")
|
|
payload = _read_json(binding_path)
|
|
expected_keys = {
|
|
"schema_version", "ledger_identity_sha256", "case_id",
|
|
"outcome_file", "outcome_sha256", "case_attempts_sha256",
|
|
"expected_operations", "case_operation_projection",
|
|
"evidence_sha256s",
|
|
}
|
|
if not isinstance(payload, dict) or set(payload) != expected_keys:
|
|
raise CallLedgerError("case outcome binding shape is invalid")
|
|
outcome_file = payload.get("outcome_file")
|
|
if outcome_file not in {"result.json", "failure.json"}:
|
|
raise CallLedgerError("case outcome binding file is invalid")
|
|
outcome_sha = _sha(payload.get("outcome_sha256"), "case outcome SHA-256")
|
|
expected_operations = _operation_projection(payload.get("expected_operations"))
|
|
if (self.controls_enabled
|
|
and expected_operations
|
|
!= self.release_controls["expected_operations"][case_id]):
|
|
raise CallLedgerError("persisted case operations differ from frozen controls")
|
|
evidence = _evidence_hashes(payload.get("evidence_sha256s"))
|
|
projection = _case_attempt_projection(records, case_id)
|
|
expected = {
|
|
"schema_version": "multiscene-case-outcome.v2",
|
|
"ledger_identity_sha256": self.identity_sha256,
|
|
"case_id": case_id,
|
|
"outcome_file": outcome_file,
|
|
"outcome_sha256": outcome_sha,
|
|
"expected_operations": expected_operations,
|
|
"case_operation_projection": projection,
|
|
"case_attempts_sha256": _digest(projection),
|
|
"evidence_sha256s": evidence,
|
|
}
|
|
if payload != expected:
|
|
raise CallLedgerError("case outcome binding no longer matches its attempts")
|
|
|
|
case_dir = self.output_dir / case_id
|
|
_reject_link_or_reparse(case_dir, "case outcome directory")
|
|
if not case_dir.is_dir():
|
|
raise CallLedgerError("case outcome directory is missing")
|
|
for candidate_name in ("result.json", "failure.json"):
|
|
_reject_link_or_reparse(case_dir / candidate_name, "case outcome artifact")
|
|
candidates = [case_dir / name for name in ("result.json", "failure.json")
|
|
if (case_dir / name).is_file()]
|
|
if len(candidates) != 1 or candidates[0].name != outcome_file:
|
|
raise CallLedgerError("case outcome artifact is missing or ambiguous")
|
|
outcome_path = candidates[0]
|
|
if hashlib.sha256(outcome_path.read_bytes()).hexdigest() != outcome_sha:
|
|
raise CallLedgerError("case outcome artifact digest mismatch")
|
|
outcome = _read_json(outcome_path)
|
|
if not isinstance(outcome, dict):
|
|
raise CallLedgerError("case outcome artifact must be an object")
|
|
calls = outcome.get("provider_call_count")
|
|
if (outcome.get("case_id") != case_id
|
|
or not isinstance(calls, int) or isinstance(calls, bool)
|
|
or calls != len(projection)):
|
|
raise CallLedgerError("case outcome artifact identity or call count is invalid")
|
|
|
|
actual_after_names: set[str] = set()
|
|
for child in case_dir.iterdir():
|
|
_reject_link_or_reparse(child, "case evidence artifact")
|
|
if child.name.endswith("-after.json"):
|
|
if not child.is_file():
|
|
raise CallLedgerError("case evidence artifact is not a regular file")
|
|
actual_after_names.add(child.name)
|
|
if actual_after_names != set(evidence):
|
|
raise CallLedgerError("case evidence inventory differs from immutable binding")
|
|
evidence_by_operation: dict[str, dict] = {}
|
|
for name, digest in evidence.items():
|
|
evidence_path = case_dir / name
|
|
if (not evidence_path.is_file()
|
|
or hashlib.sha256(evidence_path.read_bytes()).hexdigest() != digest):
|
|
raise CallLedgerError("case evidence artifact digest mismatch")
|
|
evidence_payload = _read_json(evidence_path)
|
|
if not isinstance(evidence_payload, dict):
|
|
raise CallLedgerError("case evidence artifact must be an object")
|
|
operation = evidence_payload.get("operation")
|
|
if operation not in OPERATIONS or operation in evidence_by_operation:
|
|
raise CallLedgerError("case evidence operation is invalid or duplicate")
|
|
if name not in AFTER_FILES[operation]:
|
|
raise CallLedgerError("case evidence file name does not match its operation")
|
|
evidence_by_operation[operation] = evidence_payload
|
|
|
|
if require_success:
|
|
if (outcome_file != "result.json" or outcome.get("status") != "completed"
|
|
or outcome.get("passed") is not True
|
|
or outcome.get("automated_quality_passed") is not True
|
|
or outcome.get("automated_model_quality_release_passed") is not True
|
|
or outcome.get("content_quality_release_passed") is not True):
|
|
raise CallLedgerError("successful case outcome quality gate is incomplete")
|
|
if any(item["status"] != "succeeded" for item in projection):
|
|
raise CallLedgerError("successful case outcome has a non-successful provider attempt")
|
|
actual_operations = [item["operation"] for item in projection]
|
|
actual_set, expected_set = set(actual_operations), set(expected_operations)
|
|
repair_actual = actual_set & REPAIR_OPERATIONS
|
|
mandatory = expected_set - REPAIR_OPERATIONS
|
|
if (not projection or len(actual_operations) != len(actual_set)
|
|
or not actual_set <= expected_set or not mandatory <= actual_set
|
|
or repair_actual not in (set(), set(REPAIR_OPERATIONS))
|
|
or set(evidence_by_operation) != actual_set):
|
|
raise CallLedgerError("successful case outcome operation coverage is incomplete")
|
|
for item in projection:
|
|
after = evidence_by_operation[item["operation"]]
|
|
elapsed = after.get("execution_elapsed_seconds")
|
|
job = after.get("job")
|
|
audit = after.get("audit")
|
|
if (not isinstance(elapsed, (int, float)) or isinstance(elapsed, bool)
|
|
or elapsed < 0 or not isinstance(job, dict)
|
|
or not isinstance(audit, dict)):
|
|
raise CallLedgerError("case evidence measurements are invalid")
|
|
job_id = job.get("id")
|
|
audit_id = audit.get("id")
|
|
audit_run_id = job.get("audit_run_id")
|
|
job_operation = job.get("operation")
|
|
if (after.get("case_id") != case_id
|
|
or after.get("request_sha256") != item["request_sha256"]
|
|
or after.get("response_sha256") != item["response_sha256"]
|
|
or after.get("provider_invocation_count") != 1
|
|
or job.get("status") != "succeeded"
|
|
or job.get("provider_status") != "succeeded"
|
|
or job_operation not in JOB_OPERATIONS
|
|
or item["operation"] not in JOB_OPERATIONS[job_operation]
|
|
or not isinstance(job_id, str) or not job_id
|
|
or not isinstance(audit_id, str) or not audit_id
|
|
or not isinstance(audit_run_id, str) or not audit_run_id
|
|
or audit_id != audit_run_id
|
|
or audit.get("provider") != REAL_PROVIDER_NAME
|
|
or audit.get("model") != REAL_PROVIDER_MODEL
|
|
or audit.get("provider_status") != "succeeded"):
|
|
raise CallLedgerError("case evidence differs from immutable provider attempt")
|
|
return payload
|
|
|
|
def _records(self) -> dict[tuple[str, str], dict]:
|
|
_reject_link_or_reparse(self.root / "identity.json", "ledger identity")
|
|
expected_identity = _read_json(self.root / "identity.json")
|
|
if expected_identity.get("schema_version") != LEDGER_SCHEMA:
|
|
raise LedgerIdentityError(
|
|
"legacy ledger schema is read-only diagnostic material; execution refused"
|
|
)
|
|
if expected_identity != self.identity:
|
|
raise LedgerIdentityError("ledger identity changed")
|
|
records: dict[tuple[str, str], dict] = {}
|
|
for path in self.attempts.iterdir():
|
|
match = re.fullmatch(r"([A-Za-z0-9][A-Za-z0-9._-]{0,127})--([a-z_]+)\.(claim|finish)\.json", path.name)
|
|
_reject_link_or_reparse(path, "ledger attempt")
|
|
if not path.is_file() or match is None:
|
|
raise CallLedgerError("ledger contains an unknown attempt entry")
|
|
case_id, operation, kind = match.groups()
|
|
if case_id not in self.case_limits or operation not in self.operation_limits:
|
|
raise CallLedgerError("ledger attempt is outside frozen limits")
|
|
value = _read_json(path)
|
|
expected_keys = ({"schema_version", "ledger_identity_sha256", "claim_id", "case_id",
|
|
"operation", "request_sha256", "attempt_number"}
|
|
if kind == "claim" else
|
|
{"schema_version", "ledger_identity_sha256", "claim_id", "case_id",
|
|
"operation", "status", "response_sha256"})
|
|
if not isinstance(value, dict) or set(value) != expected_keys:
|
|
raise CallLedgerError("ledger attempt shape is invalid")
|
|
if (value["ledger_identity_sha256"] != self.identity_sha256
|
|
or value["case_id"] != case_id or value["operation"] != operation):
|
|
raise CallLedgerError("ledger attempt identity is invalid")
|
|
if not isinstance(value["claim_id"], str) or not re.fullmatch(r"[0-9a-f]{32}", value["claim_id"]):
|
|
raise CallLedgerError("ledger claim identity is invalid")
|
|
if kind == "claim":
|
|
_sha(value["request_sha256"], "stored request_sha256")
|
|
if value["schema_version"] != "multiscene-call-claim.v1" or value["attempt_number"] != 1:
|
|
raise CallLedgerError("ledger claim is invalid")
|
|
else:
|
|
if value["schema_version"] != "multiscene-call-finish.v1" or value["status"] not in TERMINAL_STATUSES:
|
|
raise CallLedgerError("ledger finish is invalid")
|
|
if value["response_sha256"] is not None:
|
|
_sha(value["response_sha256"], "stored response_sha256")
|
|
key = (case_id, operation)
|
|
record = records.setdefault(key, {"claim": None, "finish": None})
|
|
if record[kind] is not None:
|
|
raise CallLedgerError("duplicate immutable ledger entry")
|
|
record[kind] = value
|
|
for record in records.values():
|
|
if record["claim"] is None:
|
|
raise CallLedgerError("terminal record has no claim")
|
|
if record["finish"] is not None and record["finish"]["claim_id"] != record["claim"]["claim_id"]:
|
|
raise CallLedgerError("terminal record does not bind its claim")
|
|
return records
|
|
|
|
def _campaign_records(self) -> dict[str, dict]:
|
|
if self.registry_root is None or self.campaign_attempts is None:
|
|
raise CallLedgerError("campaign records require a global registry")
|
|
_reject_link_or_reparse(self.registry_root / "campaign-identity.json",
|
|
"campaign identity")
|
|
stored_campaign = _read_json(self.registry_root / "campaign-identity.json")
|
|
if stored_campaign.get("schema_version") != CAMPAIGN_SCHEMA:
|
|
raise LedgerIdentityError(
|
|
"legacy campaign schema is read-only diagnostic material; execution refused"
|
|
)
|
|
if stored_campaign != self.campaign_identity:
|
|
raise LedgerIdentityError("global campaign identity changed")
|
|
records: dict[str, dict] = {}
|
|
for path in self.campaign_attempts.iterdir():
|
|
match = re.fullmatch(r"([0-9a-f]{64})\.(claim|finish)\.json", path.name)
|
|
_reject_link_or_reparse(path, "campaign attempt")
|
|
if not path.is_file() or match is None:
|
|
raise CallLedgerError("campaign contains an unknown attempt entry")
|
|
key, kind = match.groups()
|
|
value = _read_json(path)
|
|
expected_keys = ({
|
|
"schema_version", "campaign_identity_sha256", "ledger_identity_sha256",
|
|
"claim_id", "plan_sha256", "case_id", "operation", "request_sha256",
|
|
} if kind == "claim" else {
|
|
"schema_version", "campaign_identity_sha256", "ledger_identity_sha256",
|
|
"claim_id", "plan_sha256", "case_id", "operation", "status",
|
|
"response_sha256",
|
|
})
|
|
if not isinstance(value, dict) or set(value) != expected_keys:
|
|
raise CallLedgerError("campaign attempt shape is invalid")
|
|
if (value["campaign_identity_sha256"] != self.campaign_identity_sha256
|
|
or _digest({"plan_sha256": value["plan_sha256"],
|
|
"case_id": value["case_id"],
|
|
"operation": value["operation"]}) != key):
|
|
raise CallLedgerError("campaign attempt identity is invalid")
|
|
_sha(value["plan_sha256"], "campaign plan_sha256")
|
|
_sha(value["ledger_identity_sha256"], "campaign ledger identity")
|
|
_identifier(value["case_id"], "campaign case_id")
|
|
if value["operation"] not in OPERATIONS:
|
|
raise CallLedgerError("campaign operation is invalid")
|
|
if (not isinstance(value["claim_id"], str)
|
|
or not re.fullmatch(r"[0-9a-f]{32}", value["claim_id"])):
|
|
raise CallLedgerError("campaign claim identity is invalid")
|
|
if kind == "claim":
|
|
if value["schema_version"] != "multiscene-campaign-claim.v1":
|
|
raise CallLedgerError("campaign claim schema is invalid")
|
|
_sha(value["request_sha256"], "campaign request_sha256")
|
|
else:
|
|
if (value["schema_version"] != "multiscene-campaign-finish.v1"
|
|
or value["status"] not in TERMINAL_STATUSES):
|
|
raise CallLedgerError("campaign finish is invalid")
|
|
if value["response_sha256"] is not None:
|
|
_sha(value["response_sha256"], "campaign response_sha256")
|
|
record = records.setdefault(key, {"claim": None, "finish": None})
|
|
if record[kind] is not None:
|
|
raise CallLedgerError("duplicate campaign attempt entry")
|
|
record[kind] = value
|
|
for record in records.values():
|
|
claim, finish = record["claim"], record["finish"]
|
|
if claim is None:
|
|
raise CallLedgerError("campaign finish has no claim")
|
|
if finish is not None and any(
|
|
finish[field] != claim[field]
|
|
for field in ("claim_id", "plan_sha256", "case_id", "operation",
|
|
"ledger_identity_sha256")
|
|
):
|
|
raise CallLedgerError("campaign finish does not bind its claim")
|
|
return records
|
|
|
|
@contextmanager
|
|
def _locked(self) -> Iterator[None]:
|
|
lock_id = uuid4().hex
|
|
try:
|
|
descriptor = os.open(self.lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
except FileExistsError as error:
|
|
raise LedgerBusyError("call ledger lock is already held") from error
|
|
try:
|
|
with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream:
|
|
json.dump({"lock_id": lock_id}, stream, separators=(",", ":"))
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
yield
|
|
finally:
|
|
try:
|
|
current = _read_json(self.lock_path)
|
|
except (OSError, ValueError, CallLedgerError):
|
|
current = None
|
|
if current == {"lock_id": lock_id}:
|
|
self.lock_path.unlink()
|
|
|
|
|
|
def _limits(value: Mapping[str, int], *, operations: bool) -> dict[str, int]:
|
|
if not isinstance(value, Mapping):
|
|
raise TypeError("limits must be a mapping")
|
|
result = {}
|
|
for name, limit in value.items():
|
|
if operations:
|
|
if name not in OPERATIONS:
|
|
raise ValueError("unknown operation limit")
|
|
else:
|
|
name = _identifier(name, "case limit")
|
|
if not isinstance(limit, int) or isinstance(limit, bool) or limit < 0:
|
|
raise TypeError("budgets must be non-negative integers, not bool")
|
|
result[name] = limit
|
|
return result
|
|
|
|
|
|
def _json_mapping(value: Mapping[str, object] | None, label: str) -> dict | None:
|
|
if value is None:
|
|
return None
|
|
if not isinstance(value, Mapping):
|
|
raise TypeError(f"{label} must be a mapping")
|
|
try:
|
|
return json.loads(json.dumps(dict(value), ensure_ascii=False, sort_keys=True,
|
|
separators=(",", ":"), allow_nan=False))
|
|
except (TypeError, ValueError) as error:
|
|
raise ValueError(f"{label} must be canonical JSON data") from error
|
|
|
|
|
|
def _release_controls(value: Mapping[str, object] | None,
|
|
case_limits: Mapping[str, int]) -> dict | None:
|
|
normalized = _json_mapping(value, "release controls")
|
|
if normalized is None:
|
|
return None
|
|
if set(normalized) != {
|
|
"schema_version", "canary_case_ids", "failure_classes", "expected_operations",
|
|
}:
|
|
raise ValueError("release controls shape is invalid")
|
|
if normalized["schema_version"] != "multiscene-release-controls.v2":
|
|
raise ValueError("release controls schema is unsupported")
|
|
canaries = normalized["canary_case_ids"]
|
|
classes = normalized["failure_classes"]
|
|
expected_operations = normalized["expected_operations"]
|
|
if (not isinstance(canaries, list) or not canaries
|
|
or len(canaries) != len(set(canaries))
|
|
or any(case_id not in case_limits for case_id in canaries)):
|
|
raise ValueError("release control canary inventory is invalid")
|
|
if not isinstance(classes, dict) or set(classes) != set(case_limits):
|
|
raise ValueError("release control failure-class inventory is invalid")
|
|
if (not isinstance(expected_operations, dict)
|
|
or set(expected_operations) != set(case_limits)):
|
|
raise ValueError("release control operation inventory is invalid")
|
|
for case_id, failure_class in classes.items():
|
|
_identifier(case_id, "release control case ID")
|
|
_identifier(failure_class, "release control failure class")
|
|
operations = expected_operations[case_id]
|
|
if (not isinstance(operations, list) or not operations
|
|
or len(operations) != len(set(operations))
|
|
or len(operations) > case_limits[case_id]
|
|
or any(operation not in OPERATIONS for operation in operations)):
|
|
raise ValueError("release control expected operations are invalid")
|
|
return normalized
|
|
|
|
|
|
def _case_attempt_projection(records: Mapping[tuple[str, str], dict], case_id: str) -> list[dict]:
|
|
return [
|
|
{
|
|
"operation": operation,
|
|
"request_sha256": record["claim"]["request_sha256"],
|
|
"response_sha256": (
|
|
record["finish"]["response_sha256"]
|
|
if record["finish"] is not None else None
|
|
),
|
|
"status": record["finish"]["status"] if record["finish"] is not None else None,
|
|
}
|
|
for (current_case, operation), record in sorted(records.items())
|
|
if current_case == case_id
|
|
]
|
|
|
|
|
|
def _operation_projection(value: list[str] | tuple[str, ...]) -> list[str]:
|
|
if not isinstance(value, (list, tuple)) or not value:
|
|
raise TypeError("expected operations must be a non-empty list or tuple")
|
|
result: list[str] = []
|
|
for operation in value:
|
|
if operation not in OPERATIONS or operation in result:
|
|
raise ValueError("expected operations contain an invalid or duplicate operation")
|
|
result.append(operation)
|
|
return result
|
|
|
|
|
|
def _evidence_hashes(value: Mapping[str, str]) -> dict[str, str]:
|
|
if not isinstance(value, Mapping):
|
|
raise TypeError("case evidence hashes must be a mapping")
|
|
result: dict[str, str] = {}
|
|
for name, digest in value.items():
|
|
name = _identifier(name, "case evidence name")
|
|
result[name] = _sha(digest, "case evidence SHA-256")
|
|
return dict(sorted(result.items()))
|
|
|
|
|
|
def _historical_usage(value: Mapping[str, object] | None) -> dict[str, object]:
|
|
if value is None:
|
|
return {
|
|
"provider_call_count": 0,
|
|
"repair_budget_charged": 0,
|
|
"attempts": [],
|
|
"attempts_sha256": None,
|
|
"receipt_sha256": None,
|
|
}
|
|
expected = {
|
|
"provider_call_count", "repair_budget_charged",
|
|
"attempts", "receipt_sha256",
|
|
}
|
|
if not isinstance(value, Mapping) or set(value) != expected:
|
|
raise ValueError("historical usage shape is invalid")
|
|
provider_calls = value["provider_call_count"]
|
|
repair_calls = value["repair_budget_charged"]
|
|
for label, count, ceiling in (
|
|
("provider_call_count", provider_calls, MAX_PREALLOCATED_ATTEMPTS),
|
|
("repair_budget_charged", repair_calls, MAX_REPAIR_ATTEMPTS),
|
|
):
|
|
if not isinstance(count, int) or isinstance(count, bool) or not 0 <= count <= ceiling:
|
|
raise ValueError(f"historical {label} is invalid")
|
|
if repair_calls > provider_calls:
|
|
raise ValueError("historical repair budget cannot exceed provider calls")
|
|
attempts = value["attempts"]
|
|
if not isinstance(attempts, list) or len(attempts) != provider_calls:
|
|
raise ValueError("historical attempts do not match provider call count")
|
|
normalized_attempts = []
|
|
attempt_ids: set[str] = set()
|
|
claim_hashes: set[str] = set()
|
|
attempt_keys = {
|
|
"attempt_id", "plan_sha256", "case_id", "operation",
|
|
"provider_status", "business_status", "elapsed_seconds", "timed_out",
|
|
"request_sha256", "response_sha256", "provider", "model",
|
|
"job_id", "audit_run_id",
|
|
"identity_path", "identity_file_sha256", "ledger_identity_sha256",
|
|
"claim_path", "claim_sha256", "finish_path", "finish_sha256",
|
|
"before_path", "before_sha256", "after_path", "after_sha256",
|
|
"disposition",
|
|
}
|
|
for attempt in attempts:
|
|
if not isinstance(attempt, Mapping) or set(attempt) != attempt_keys:
|
|
raise ValueError("historical attempt shape is invalid")
|
|
attempt_id = _identifier(attempt["attempt_id"], "historical attempt ID")
|
|
case_id = _identifier(attempt["case_id"], "historical case ID")
|
|
operation = attempt["operation"]
|
|
provider_status = attempt["provider_status"]
|
|
business_status = attempt["business_status"]
|
|
elapsed_seconds = attempt["elapsed_seconds"]
|
|
timed_out = attempt["timed_out"]
|
|
disposition = attempt["disposition"]
|
|
if operation not in OPERATIONS:
|
|
raise ValueError("historical attempt operation is invalid")
|
|
if provider_status not in TERMINAL_STATUSES:
|
|
raise ValueError("historical provider status is invalid")
|
|
if business_status not in {
|
|
"succeeded", "failed", "cancelled", "awaiting_human", "resolved",
|
|
}:
|
|
raise ValueError("historical business status is invalid")
|
|
if (not isinstance(elapsed_seconds, (int, float))
|
|
or isinstance(elapsed_seconds, bool) or elapsed_seconds < 0):
|
|
raise ValueError("historical elapsed seconds is invalid")
|
|
if not isinstance(timed_out, bool) or (timed_out and provider_status != "failed"):
|
|
raise ValueError("historical timeout flag is invalid")
|
|
for label in (
|
|
"plan_sha256", "request_sha256", "response_sha256",
|
|
"identity_file_sha256", "ledger_identity_sha256", "claim_sha256",
|
|
"finish_sha256", "before_sha256", "after_sha256",
|
|
):
|
|
_sha(attempt[label], f"historical {label}")
|
|
for label in ("identity_path", "claim_path", "finish_path", "before_path", "after_path"):
|
|
path_value = attempt[label]
|
|
if (not isinstance(path_value, str) or not path_value
|
|
or len(path_value) > 2048 or "\x00" in path_value):
|
|
raise ValueError(f"historical {label} is invalid")
|
|
if attempt["provider"] != "codex-cli-development" or attempt["model"] != "gpt-5.6-sol":
|
|
raise ValueError("historical provider/model is invalid")
|
|
for label in ("job_id", "audit_run_id"):
|
|
if (not isinstance(attempt[label], str) or not attempt[label]
|
|
or len(attempt[label]) > 256 or "\x00" in attempt[label]):
|
|
raise ValueError(f"historical {label} is invalid")
|
|
if not isinstance(disposition, Mapping) or set(disposition) != {
|
|
"resolution_status", "release_eligible", "reason",
|
|
"replacement_case_id", "replacement_operation",
|
|
}:
|
|
raise ValueError("historical disposition shape is invalid")
|
|
if disposition["resolution_status"] not in {"resolved", "requires_replacement"}:
|
|
raise ValueError("historical disposition status is invalid")
|
|
if not isinstance(disposition["release_eligible"], bool):
|
|
raise ValueError("historical release eligibility is invalid")
|
|
if (not isinstance(disposition["reason"], str)
|
|
or not disposition["reason"].strip() or len(disposition["reason"]) > 1000):
|
|
raise ValueError("historical disposition reason is invalid")
|
|
successful_provider_business = (
|
|
provider_status == "succeeded"
|
|
and business_status in {"succeeded", "resolved"}
|
|
and not timed_out
|
|
)
|
|
if disposition["release_eligible"] and not successful_provider_business:
|
|
raise ValueError(
|
|
"historical provider/business failure cannot be release eligible"
|
|
)
|
|
if (not successful_provider_business
|
|
and disposition["resolution_status"] != "requires_replacement"):
|
|
raise ValueError("historical provider/business failure requires replacement")
|
|
replacement_case_id = disposition["replacement_case_id"]
|
|
replacement_operation = disposition["replacement_operation"]
|
|
if disposition["resolution_status"] == "requires_replacement":
|
|
_identifier(replacement_case_id, "historical replacement case ID")
|
|
if replacement_operation not in OPERATIONS or disposition["release_eligible"]:
|
|
raise ValueError("historical replacement disposition is invalid")
|
|
elif replacement_case_id is not None or replacement_operation is not None:
|
|
raise ValueError("resolved historical disposition cannot declare replacement")
|
|
if attempt_id in attempt_ids or attempt["claim_sha256"] in claim_hashes:
|
|
raise ValueError("historical attempts must be unique")
|
|
attempt_ids.add(attempt_id)
|
|
claim_hashes.add(attempt["claim_sha256"])
|
|
normalized_attempts.append({**dict(attempt),
|
|
"attempt_id": attempt_id, "case_id": case_id, "operation": operation,
|
|
"elapsed_seconds": round(float(elapsed_seconds), 3),
|
|
"disposition": dict(disposition),
|
|
})
|
|
return {
|
|
"provider_call_count": provider_calls,
|
|
"repair_budget_charged": repair_calls,
|
|
"attempts": normalized_attempts,
|
|
"attempts_sha256": _digest(normalized_attempts),
|
|
"receipt_sha256": _sha(value["receipt_sha256"], "historical receipt_sha256"),
|
|
}
|
|
|
|
|
|
def _identifier(value: object, label: str) -> str:
|
|
if not isinstance(value, str) or not _ID_RE.fullmatch(value) or value in {".", ".."}:
|
|
raise ValueError(f"{label} is not a safe identifier")
|
|
return value
|
|
|
|
|
|
def _sha(value: object, label: str) -> str:
|
|
if not isinstance(value, str) or not _SHA_RE.fullmatch(value):
|
|
raise ValueError(f"{label} must be a lowercase SHA-256 value")
|
|
return value
|
|
|
|
|
|
def _digest(value: object) -> str:
|
|
return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True,
|
|
separators=(",", ":"), allow_nan=False).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _read_json(path: Path) -> object:
|
|
def pairs(items):
|
|
result = {}
|
|
for key, value in items:
|
|
if key in result:
|
|
raise CallLedgerError("ledger JSON contains duplicate keys")
|
|
result[key] = value
|
|
return result
|
|
|
|
def constant(value):
|
|
raise CallLedgerError(f"ledger JSON contains non-finite value {value}")
|
|
|
|
return json.loads(path.read_text(encoding="utf-8"),
|
|
object_pairs_hook=pairs, parse_constant=constant)
|
|
|
|
|
|
def _write_x(path: Path, value: object) -> None:
|
|
with path.open("x", encoding="utf-8", newline="\n") as stream:
|
|
json.dump(value, stream, ensure_ascii=False, sort_keys=True,
|
|
separators=(",", ":"), allow_nan=False)
|
|
stream.write("\n")
|
|
stream.flush()
|
|
os.fsync(stream.fileno())
|
|
|
|
|
|
def _reject_link_or_reparse(path: Path, label: str) -> None:
|
|
"""Reject filesystem indirection for ledger-owned directories."""
|
|
try:
|
|
metadata = path.lstat()
|
|
except FileNotFoundError:
|
|
return
|
|
attributes = getattr(metadata, "st_file_attributes", 0)
|
|
reparse_flag = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
|
|
if stat.S_ISLNK(metadata.st_mode) or attributes & reparse_flag:
|
|
raise CallLedgerError(f"{label} must not be a symlink or reparse point")
|
|
|
|
|
|
def _reject_path_chain(path: Path, label: str) -> None:
|
|
"""Reject a link/junction anywhere in an authority path."""
|
|
current = path
|
|
while True:
|
|
_reject_link_or_reparse(current, label)
|
|
if current.parent == current:
|
|
return
|
|
current = current.parent
|