Codex
1333 lines
54 KiB
Python
1333 lines
54 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from typing import Optional
|
|
from urllib.parse import quote
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
STATUS_PATH = ROOT / ".claude" / "monitoring_alert_rollout_status.json"
|
|
DOC_PATH = ROOT / "docs" / "archive" / "AI_ICT_下一阶段开发任务清单_v1.md"
|
|
SESSION_PATH = ROOT / "SESSION.md"
|
|
TASKS_PATH = ROOT / "TASKS.md"
|
|
ENV_LOCAL_PATH = ROOT / ".env.local"
|
|
ENV_PATH = ROOT / ".env"
|
|
ENV_EXAMPLE_PATH = ROOT / ".env.example"
|
|
DEFAULT_TIMEOUT_SECONDS = 900
|
|
|
|
|
|
def _venv_python_command() -> str:
|
|
windows = ROOT / ".venv" / "Scripts" / "python.exe"
|
|
unix = ROOT / ".venv" / "bin" / "python"
|
|
if windows.exists():
|
|
return '"/d/ai-exchange/.venv/Scripts/python.exe"'
|
|
if unix.exists():
|
|
return ".venv/bin/python"
|
|
return "python"
|
|
|
|
|
|
PYTHON_CMD = _venv_python_command()
|
|
|
|
|
|
def _cmd(script_name: str) -> str:
|
|
return f"{PYTHON_CMD} scripts/{script_name}"
|
|
|
|
|
|
def _dedupe(items: list[str]) -> list[str]:
|
|
seen = set()
|
|
out = []
|
|
for item in items:
|
|
if item in seen:
|
|
continue
|
|
seen.add(item)
|
|
out.append(item)
|
|
return out
|
|
|
|
|
|
def _truncate_output(text: str, limit: int = 240) -> str:
|
|
cleaned = str(text or "").strip().replace("\r", " ").replace("\n", " ")
|
|
return cleaned[:limit]
|
|
|
|
|
|
def _get_bool(env, key: str, default: bool = False) -> bool:
|
|
value = env.get(key)
|
|
if value is None or value == "":
|
|
return default
|
|
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _get_int(env, key: str, default: int) -> int:
|
|
value = env.get(key)
|
|
if value is None or value == "":
|
|
return default
|
|
try:
|
|
return int(str(value).strip())
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
def _now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def _read_env_file_value(path: Path, key: str) -> Optional[str]:
|
|
if not path.exists():
|
|
return None
|
|
prefix = f"{key}="
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
stripped = line.strip()
|
|
if not stripped or stripped.startswith("#"):
|
|
continue
|
|
if stripped.startswith(prefix):
|
|
return stripped[len(prefix):].strip()
|
|
return None
|
|
|
|
|
|
def _compose_database_url_from_env(env: dict) -> Optional[str]:
|
|
explicit = str(env.get("DATABASE_URL") or "").strip()
|
|
if explicit:
|
|
return explicit
|
|
host = str(env.get("DATABASE_HOST") or "").strip()
|
|
port = str(env.get("DATABASE_PORT") or "").strip()
|
|
name = str(env.get("DATABASE_NAME") or "").strip()
|
|
user = str(env.get("DATABASE_USER") or "").strip()
|
|
password = str(env.get("DATABASE_PASSWORD") or "")
|
|
if not all([host, port, name, user]):
|
|
return None
|
|
return f"postgresql+psycopg://{quote(user, safe='')}:{quote(password, safe='')}@{host}:{port}/{name}"
|
|
|
|
|
|
def _merge_env_file_values(env: dict, path: Path, keys: list[str]) -> dict:
|
|
merged = dict(env)
|
|
for key in keys:
|
|
if merged.get(key):
|
|
continue
|
|
value = _read_env_file_value(path, key)
|
|
if value is not None:
|
|
merged[key] = value
|
|
return merged
|
|
|
|
|
|
def _seed_database_env(env: dict) -> dict:
|
|
seeded = dict(env or {})
|
|
component_keys = ["DATABASE_HOST", "DATABASE_PORT", "DATABASE_NAME", "DATABASE_USER", "DATABASE_PASSWORD"]
|
|
seeded = _merge_env_file_values(seeded, ENV_LOCAL_PATH, component_keys + ["DATABASE_URL"])
|
|
local_composed = _compose_database_url_from_env(seeded)
|
|
if local_composed:
|
|
seeded["DATABASE_URL"] = local_composed
|
|
return seeded
|
|
seeded = _merge_env_file_values(seeded, ENV_PATH, component_keys + ["DATABASE_URL"])
|
|
env_composed = _compose_database_url_from_env(seeded)
|
|
if env_composed:
|
|
seeded["DATABASE_URL"] = env_composed
|
|
return seeded
|
|
seeded = _merge_env_file_values(seeded, ENV_EXAMPLE_PATH, ["DATABASE_URL"])
|
|
fallback = _compose_database_url_from_env(seeded)
|
|
if fallback:
|
|
seeded["DATABASE_URL"] = fallback
|
|
return seeded
|
|
|
|
|
|
N1_VALIDATION_COMMANDS = [
|
|
_cmd("run_launch_readiness.py"),
|
|
_cmd("run_live_readiness.py"),
|
|
_cmd("run_review_packet.py"),
|
|
_cmd("run_trading_runtime.py"),
|
|
_cmd("run_offline_checks.py"),
|
|
]
|
|
N2_VALIDATION_COMMANDS = [
|
|
_cmd("run_decision_packet.py"),
|
|
_cmd("run_alert_dispatcher.py"),
|
|
_cmd("run_trading_runtime.py"),
|
|
_cmd("run_offline_checks.py"),
|
|
]
|
|
N3_VALIDATION_COMMANDS = [
|
|
_cmd("run_decision_packet.py"),
|
|
_cmd("run_trading_runtime.py"),
|
|
_cmd("run_review_packet.py"),
|
|
_cmd("run_offline_checks.py"),
|
|
]
|
|
N4_VALIDATION_COMMANDS = [
|
|
_cmd("run_trading_runtime.py"),
|
|
_cmd("run_review_packet.py"),
|
|
_cmd("run_offline_checks.py"),
|
|
]
|
|
|
|
PHASES = [
|
|
{
|
|
"code": "N1",
|
|
"title": "Launch Readiness 收尾",
|
|
"goal": "收敛 assisted-trading readiness 剩余 blocker 与 warning，并硬化 readiness 展示。",
|
|
"tasks": [
|
|
"N1-1 明确 launch readiness 判定口径",
|
|
"N1-2 收敛 trust / live / real-trade 样本 warning",
|
|
"N1-3 workbench / API 展示硬化",
|
|
],
|
|
"files": [
|
|
"src/services/readiness/launch_readiness_service.py",
|
|
"src/config/settings.py",
|
|
"src/api/server.py",
|
|
"src/api/trading_runtime.py",
|
|
"src/web/app.js",
|
|
"src/web/index.html",
|
|
"SESSION.md",
|
|
"TASKS.md",
|
|
],
|
|
"validation_commands": N1_VALIDATION_COMMANDS,
|
|
"deliverables": [
|
|
"clear assisted-vs-real-live readiness boundary",
|
|
"reduced launch blockers and warnings",
|
|
"actionable next_actions output",
|
|
"hardened launch readiness workbench panel",
|
|
],
|
|
},
|
|
{
|
|
"code": "N2",
|
|
"title": "最小主动消息告警闭环",
|
|
"goal": "把当前仅落库的 decision alerts 打通到真实外部消息通道。",
|
|
"tasks": [
|
|
"N2-1 新增 alert delivery 持久化",
|
|
"N2-2 新增 notifier 基础设施",
|
|
"N2-3 新增 AlertDispatcherService",
|
|
"N2-4 API / CLI / runtime 接入",
|
|
],
|
|
"files": [
|
|
"migrations/*_create_alert_deliveries.sql",
|
|
"src/repositories/alert_delivery_repository.py",
|
|
"src/services/notifications/__init__.py",
|
|
"src/services/notifications/telegram_notifier.py",
|
|
"src/services/notifications/webhook_notifier.py",
|
|
"src/services/alerts/__init__.py",
|
|
"src/services/alerts/alert_dispatcher_service.py",
|
|
"src/services/alerts/message_template_service.py",
|
|
"src/services/alerts/channel_router_service.py",
|
|
"src/services/alerts/delivery_policy_service.py",
|
|
"src/api/alerts.py",
|
|
"src/api/server.py",
|
|
"src/services/runtime/trading_runtime_service.py",
|
|
"scripts/run_alert_dispatcher.py",
|
|
],
|
|
"validation_commands": N2_VALIDATION_COMMANDS,
|
|
"deliverables": [
|
|
"alert_deliveries persistence",
|
|
"single-channel notifier",
|
|
"queued-to-delivered alert dispatcher",
|
|
"dispatch API and CLI",
|
|
"runtime alert_dispatch task",
|
|
],
|
|
},
|
|
{
|
|
"code": "N3",
|
|
"title": "告警质量与 runtime 硬化",
|
|
"goal": "在外发闭环接通后治理噪声、分级状态并统一决策包展示。",
|
|
"tasks": [
|
|
"N3-1 告警节流与重复抑制",
|
|
"N3-2 runtime blockers 分级",
|
|
"N3-3 决策包展示一致性硬化",
|
|
],
|
|
"files": [
|
|
"src/services/alerts/delivery_policy_service.py",
|
|
"src/services/decision/decision_packet_service.py",
|
|
"src/services/runtime/trading_runtime_service.py",
|
|
"src/services/readiness/launch_readiness_service.py",
|
|
"src/api/trading_runtime.py",
|
|
"src/web/app.js",
|
|
],
|
|
"validation_commands": N3_VALIDATION_COMMANDS,
|
|
"deliverables": [
|
|
"deduped and throttled alerts",
|
|
"graded runtime status semantics",
|
|
"consistent packet wording across API, UI, notifier",
|
|
],
|
|
},
|
|
{
|
|
"code": "N4",
|
|
"title": "视觉盯盘增强预埋",
|
|
"goal": "只固化视觉层模型与扩展位，不全面实现截图与视觉复核主线。",
|
|
"tasks": [
|
|
"N4-1 固化视觉数据模型设计",
|
|
"N4-2 仅做代码预留点",
|
|
],
|
|
"files": [
|
|
"docs/archive/AI_ICT_盯盘告警架构图_v1.md",
|
|
"docs/archive/AI_ICT_盯盘告警技术设计稿_v1.md",
|
|
"docs/archive/AI_ICT_盯盘告警开发任务清单_v1.md",
|
|
"src/services/decision/decision_packet_service.py",
|
|
"src/services/runtime/trading_runtime_service.py",
|
|
],
|
|
"validation_commands": N4_VALIDATION_COMMANDS,
|
|
"deliverables": [
|
|
"stable visual trigger taxonomy",
|
|
"reserved visual payload fields",
|
|
"reserved runtime extension points",
|
|
],
|
|
},
|
|
]
|
|
|
|
MAJOR_BLOCKERS = {
|
|
"runtime_env_missing": [
|
|
"validationerror",
|
|
"field required",
|
|
"database_url",
|
|
"database url",
|
|
"input_value={}"
|
|
],
|
|
"missing_credentials": ["missing_credentials", "缺少凭据", "token", "chat_id", "webhook"],
|
|
"external_channel_auth_required": ["401", "403", "forbidden", "unauthorized", "auth"],
|
|
"unsafe_migration_failure": ["migration", "sql", "alembic"],
|
|
"vendor_or_network_validation_blocked": ["network", "timed out", "connection", "proxy", "dns", "429", "vendor"],
|
|
"live_trading_boundary_unclear": ["live trading", "real live", "submit strategy", "boundary"],
|
|
}
|
|
|
|
|
|
def build_rollout_plan() -> dict:
|
|
return {
|
|
"generated_ts": _now_iso(),
|
|
"doc_path": str(DOC_PATH),
|
|
"major_blockers": list(MAJOR_BLOCKERS.keys()),
|
|
"phases": PHASES,
|
|
"automation_policy": {
|
|
"default_behavior": "run validations and advance when phase passes",
|
|
"pause_only_for": list(MAJOR_BLOCKERS.keys()),
|
|
"run_mode": "run-until-blocked",
|
|
"repair_mode": "repair-loop",
|
|
"bridge_mode": "repair-bridge",
|
|
},
|
|
}
|
|
|
|
|
|
def read_status() -> dict:
|
|
if not STATUS_PATH.exists():
|
|
return {
|
|
"state": "not_started",
|
|
"updated_ts": _now_iso(),
|
|
"current_phase": PHASES[0]["code"],
|
|
"completed_phases": [],
|
|
"blocked_reason": None,
|
|
"note": "Rollout has not started yet.",
|
|
"doc_path": str(DOC_PATH),
|
|
"phases": PHASES,
|
|
"history": [],
|
|
"last_run": None,
|
|
"repair_loop": None,
|
|
}
|
|
payload = json.loads(STATUS_PATH.read_text(encoding="utf-8"))
|
|
payload.setdefault("doc_path", str(DOC_PATH))
|
|
payload.setdefault("phases", PHASES)
|
|
payload.setdefault("history", [])
|
|
payload.setdefault("last_run", None)
|
|
payload.setdefault("repair_loop", None)
|
|
return payload
|
|
|
|
|
|
def write_status(payload: dict) -> None:
|
|
STATUS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
STATUS_PATH.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
|
def start_rollout() -> dict:
|
|
payload = {
|
|
"state": "running",
|
|
"started_ts": _now_iso(),
|
|
"updated_ts": _now_iso(),
|
|
"current_phase": PHASES[0]["code"],
|
|
"completed_phases": [],
|
|
"blocked_reason": None,
|
|
"note": "Autonomous rollout started. Continue until a major blocker is recorded.",
|
|
"phases": PHASES,
|
|
"doc_path": str(DOC_PATH),
|
|
"history": [],
|
|
"last_run": None,
|
|
"repair_loop": None,
|
|
}
|
|
write_status(payload)
|
|
return payload
|
|
|
|
|
|
def _phase_by_code(code: str) -> Optional[dict]:
|
|
for phase in PHASES:
|
|
if phase["code"] == code:
|
|
return phase
|
|
return None
|
|
|
|
|
|
def _next_phase_code(completed: list[str]) -> Optional[str]:
|
|
for phase in PHASES:
|
|
if phase["code"] not in completed:
|
|
return phase["code"]
|
|
return None
|
|
|
|
|
|
def advance_rollout() -> dict:
|
|
status = read_status()
|
|
current = status.get("current_phase")
|
|
if current is None:
|
|
return status
|
|
completed = list(status.get("completed_phases") or [])
|
|
if current not in completed:
|
|
completed.append(current)
|
|
next_phase = _next_phase_code(completed)
|
|
history = list(status.get("history") or [])
|
|
history.append({"ts": _now_iso(), "event": "advance", "from": current, "to": next_phase})
|
|
status["completed_phases"] = completed
|
|
status["current_phase"] = next_phase
|
|
status["updated_ts"] = _now_iso()
|
|
status["state"] = "completed" if next_phase is None else "running"
|
|
status["blocked_reason"] = None
|
|
status["history"] = history[-100:]
|
|
status["note"] = "All rollout phases are completed." if next_phase is None else f"Advanced to {next_phase}."
|
|
write_status(status)
|
|
return status
|
|
|
|
|
|
def block_rollout(*, blocker: str, note: str) -> dict:
|
|
status = read_status()
|
|
history = list(status.get("history") or [])
|
|
history.append({"ts": _now_iso(), "event": "blocked", "phase": status.get("current_phase"), "blocker": blocker, "note": note})
|
|
status["state"] = "blocked"
|
|
status["blocked_reason"] = blocker
|
|
status["updated_ts"] = _now_iso()
|
|
status["note"] = note or "Rollout paused because a major blocker was recorded."
|
|
status["history"] = history[-100:]
|
|
write_status(status)
|
|
return status
|
|
|
|
|
|
def complete_rollout() -> dict:
|
|
status = read_status()
|
|
history = list(status.get("history") or [])
|
|
history.append({"ts": _now_iso(), "event": "complete"})
|
|
status["state"] = "completed"
|
|
status["current_phase"] = None
|
|
status["completed_phases"] = [phase["code"] for phase in PHASES]
|
|
status["blocked_reason"] = None
|
|
status["updated_ts"] = _now_iso()
|
|
status["note"] = "Rollout marked completed."
|
|
status["history"] = history[-100:]
|
|
write_status(status)
|
|
return status
|
|
|
|
|
|
def _run_command(*, command: str, timeout_seconds: int, env: Optional[dict] = None) -> dict:
|
|
started = _now_iso()
|
|
runtime_env = dict(os.environ)
|
|
if env is not None:
|
|
runtime_env.update(env)
|
|
try:
|
|
completed = subprocess.run(
|
|
["bash", "-lc", command],
|
|
cwd=str(ROOT),
|
|
env=runtime_env,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout_seconds,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
return {
|
|
"command": command,
|
|
"started_ts": started,
|
|
"finished_ts": _now_iso(),
|
|
"returncode": int(completed.returncode),
|
|
"stdout": str(completed.stdout or "")[-8000:],
|
|
"stderr": str(completed.stderr or "")[-8000:],
|
|
}
|
|
except subprocess.TimeoutExpired as exc:
|
|
return {
|
|
"command": command,
|
|
"started_ts": started,
|
|
"finished_ts": _now_iso(),
|
|
"returncode": 124,
|
|
"stdout": str(exc.stdout or "")[-8000:],
|
|
"stderr": str(exc.stderr or "")[-8000:],
|
|
}
|
|
|
|
|
|
def _combined_output(run: dict) -> str:
|
|
return f"{run.get('stdout', '')}\n{run.get('stderr', '')}"
|
|
|
|
|
|
def _detect_major_blocker(run: dict) -> Optional[str]:
|
|
haystack = _combined_output(run).lower()
|
|
if "adapter_real_live_not_ready" in haystack or "live readiness: false" in haystack:
|
|
return None
|
|
if "validationerror" in haystack and "database_url" in haystack and "field required" in haystack:
|
|
return "runtime_env_missing"
|
|
for code, patterns in MAJOR_BLOCKERS.items():
|
|
if any(pattern.lower() in haystack for pattern in patterns):
|
|
return code
|
|
return None
|
|
|
|
|
|
def _seed_rollout_env(env: dict) -> dict:
|
|
seeded = dict(env or {})
|
|
override_pairs = [
|
|
("MONITORING_ALERT_DATABASE_URL", "DATABASE_URL"),
|
|
("MONITORING_ALERT_DATABASE_HOST", "DATABASE_HOST"),
|
|
("MONITORING_ALERT_DATABASE_PORT", "DATABASE_PORT"),
|
|
("MONITORING_ALERT_DATABASE_NAME", "DATABASE_NAME"),
|
|
("MONITORING_ALERT_DATABASE_USER", "DATABASE_USER"),
|
|
("MONITORING_ALERT_DATABASE_PASSWORD", "DATABASE_PASSWORD"),
|
|
("MONITORING_ALERT_BROKER_READ_PROXY_URL", "BROKER_READ_PROXY_URL"),
|
|
("MONITORING_ALERT_OKX_API_KEY", "OKX_API_KEY"),
|
|
("MONITORING_ALERT_OKX_API_SECRET", "OKX_API_SECRET"),
|
|
("MONITORING_ALERT_OKX_API_PASSPHRASE", "OKX_API_PASSPHRASE"),
|
|
]
|
|
for source_key, target_key in override_pairs:
|
|
if seeded.get(source_key):
|
|
seeded[target_key] = seeded[source_key]
|
|
seeded = _seed_database_env(seeded)
|
|
return seeded
|
|
|
|
|
|
def _resolve_rollout_env(env: Optional[dict]) -> dict:
|
|
base = dict(os.environ)
|
|
if env is not None:
|
|
base.update(env)
|
|
return _seed_rollout_env(base)
|
|
|
|
|
|
def _is_runtime_env_missing_result(phase_result: dict) -> bool:
|
|
return str(phase_result.get("blocker") or "") == "runtime_env_missing"
|
|
|
|
|
|
def _retry_phase_with_seeded_env(*, phase: dict, phase_result: dict, timeout_seconds: int, env: dict) -> dict:
|
|
if not _is_runtime_env_missing_result(phase_result):
|
|
return phase_result
|
|
seeded_env = _seed_rollout_env(env)
|
|
if seeded_env == env:
|
|
return phase_result
|
|
retried = _execute_phase(phase=phase, timeout_seconds=timeout_seconds, env=seeded_env)
|
|
retried["summary"] = f"{retried.get('summary')} (retried with seeded runtime env)"
|
|
return retried
|
|
|
|
|
|
def _classify_runtime_exit_as_needs_work(phase_result: dict) -> dict:
|
|
commands = list(phase_result.get("commands") or [])
|
|
if not commands:
|
|
return phase_result
|
|
failing = next((item for item in commands if int(item.get("returncode") or 0) != 0), None)
|
|
if failing is None:
|
|
return phase_result
|
|
output = _combined_output(failing).lower()
|
|
if "trading runtime:" in output and "blockers:" in output:
|
|
adjusted = dict(phase_result)
|
|
adjusted["result"] = "needs_work"
|
|
adjusted["blocker"] = None
|
|
adjusted["summary"] = f"Validation failed for phase {phase_result.get('phase')} on command: {phase_result.get('failed_command')}"
|
|
return adjusted
|
|
return phase_result
|
|
|
|
|
|
def _finalize_phase_result(*, phase: dict, phase_result: dict) -> dict:
|
|
result = _classify_runtime_exit_as_needs_work(phase_result)
|
|
guidance = _build_next_step_guidance(
|
|
phase=phase,
|
|
result=result["result"],
|
|
blocker=result.get("blocker"),
|
|
failed_command=result.get("failed_command"),
|
|
command_results=list(result.get("commands") or []),
|
|
)
|
|
repair_metadata = _build_repair_metadata(
|
|
phase=phase,
|
|
result=result["result"],
|
|
blocker=result.get("blocker"),
|
|
failed_command=result.get("failed_command"),
|
|
command_results=list(result.get("commands") or []),
|
|
guidance=guidance,
|
|
)
|
|
result["next_step_guidance"] = guidance
|
|
result["repair_metadata"] = repair_metadata
|
|
return result
|
|
|
|
|
|
def _prioritize_files_for_failed_command(phase: dict, failed_command: Optional[str]) -> list[str]:
|
|
files = list(phase.get("files") or [])
|
|
if not failed_command:
|
|
return files[:6]
|
|
command = failed_command.lower()
|
|
ranked = []
|
|
for file_path in files:
|
|
score = 0
|
|
lower_path = file_path.lower()
|
|
if "launch_readiness" in command and "readiness" in lower_path:
|
|
score += 3
|
|
if "alert_dispatcher" in command and any(token in lower_path for token in ["alerts", "notifications", "runtime"]):
|
|
score += 3
|
|
if "trading_runtime" in command and "runtime" in lower_path:
|
|
score += 3
|
|
if "review_packet" in command and any(token in lower_path for token in ["decision", "operator", "api"]):
|
|
score += 3
|
|
ranked.append((score, file_path))
|
|
ranked.sort(key=lambda item: (-item[0], item[1]))
|
|
return [item[1] for item in ranked[:6]]
|
|
|
|
|
|
def _summarize_command_output(command: dict) -> str:
|
|
for field in ("stderr", "stdout"):
|
|
text = str(command.get(field) or "").strip()
|
|
if text:
|
|
return _truncate_output(text.splitlines()[0].strip())
|
|
return ""
|
|
|
|
|
|
def _infer_fix_hints(*, phase: dict, failed_command: Optional[str], output: str) -> list[str]:
|
|
hints = []
|
|
haystack = output.lower()
|
|
command = str(failed_command or "").lower()
|
|
if "modulenotfounderror" in haystack or "importerror" in haystack:
|
|
hints.append("Start with missing imports or missing modules in the phase's core files.")
|
|
if "no such file or directory" in haystack:
|
|
hints.append("Check whether the referenced script, file, or generated artifact exists at the expected path.")
|
|
if "assertionerror" in haystack or "failed" in haystack:
|
|
hints.append("Read the failing assertion/test output and adjust phase logic before broad re-runs.")
|
|
if "not implemented" in haystack:
|
|
hints.append("Implement the missing phase behavior in the listed service files before re-running validations.")
|
|
if "run_launch_readiness.py" in command:
|
|
hints.append("Prioritize readiness service, settings, and workbench readiness display files.")
|
|
if "run_alert_dispatcher.py" in command:
|
|
hints.append("Prioritize alert delivery repository, notifier, dispatcher service, and runtime dispatch hook.")
|
|
if "run_trading_runtime.py" in command:
|
|
hints.append("Prioritize runtime task state, scheduling flow, and API/runtime status alignment.")
|
|
if "run_review_packet.py" in command:
|
|
hints.append("Prioritize decision packet wording consistency and review packet aggregation paths.")
|
|
if not hints:
|
|
hints.append(f"Focus on the active phase files first: {', '.join((phase.get('files') or [])[:4])}.")
|
|
return _dedupe(hints)
|
|
|
|
|
|
def _should_treat_as_informational_failure(*, phase: dict, command: str, run: dict) -> bool:
|
|
if phase.get("code") != "N1":
|
|
return False
|
|
if "run_live_readiness.py" not in str(command or ""):
|
|
return False
|
|
output = _combined_output(run)
|
|
return "adapter_real_live_not_ready" in output or "Live readiness: False" in output
|
|
|
|
|
|
def _build_next_step_guidance(*, phase: dict, result: str, blocker: Optional[str], failed_command: Optional[str], command_results: list[dict]) -> dict:
|
|
guidance = {
|
|
"priority_tasks": list(phase.get("tasks") or [])[:3],
|
|
"suggested_files": _prioritize_files_for_failed_command(phase, failed_command),
|
|
"suggested_commands": [],
|
|
"reasoning": [],
|
|
}
|
|
if result == "passed":
|
|
guidance["reasoning"].append("Current phase validations passed; advance to the next phase.")
|
|
return guidance
|
|
if blocker is not None:
|
|
guidance["reasoning"].append(f"Major blocker detected: {blocker}.")
|
|
guidance["reasoning"].append("Pause implementation and resolve the external or boundary issue first.")
|
|
if failed_command:
|
|
guidance["suggested_commands"].append(failed_command)
|
|
return guidance
|
|
failed_run = next((item for item in command_results if item.get("returncode") != 0), None)
|
|
if failed_command:
|
|
guidance["suggested_commands"].append(failed_command)
|
|
if failed_run is not None:
|
|
guidance["reasoning"].append("Inspect the first failing validation command and fix the underlying code issue before re-running the phase.")
|
|
guidance["reasoning"].extend(_infer_fix_hints(phase=phase, failed_command=failed_command, output=_combined_output(failed_run)))
|
|
preview = _summarize_command_output(failed_run)
|
|
if preview:
|
|
guidance["reasoning"].append(f"First failing signal: {preview}")
|
|
if not guidance["suggested_commands"]:
|
|
guidance["suggested_commands"] = list(phase.get("validation_commands") or [])[:2]
|
|
return guidance
|
|
|
|
|
|
def _build_repair_prompt_context(*, phase: dict, result: str, failed_command: Optional[str], failed_run: Optional[dict], guidance: dict, failure_class: str, recommended_focus: str, phase_patch_targets: list[str]) -> dict:
|
|
command_preview = _summarize_command_output(failed_run or {}) if failed_run is not None else ""
|
|
prompt_lines = [
|
|
f"Active phase: {phase.get('code')} {phase.get('title')}",
|
|
f"Goal: {phase.get('goal')}",
|
|
f"Failure class: {failure_class}",
|
|
f"Recommended focus: {recommended_focus}",
|
|
f"Validation entrypoint: {failed_command or 'none'}",
|
|
]
|
|
if command_preview:
|
|
prompt_lines.append(f"First failing signal: {command_preview}")
|
|
if phase_patch_targets:
|
|
prompt_lines.append(f"Patch targets: {', '.join(phase_patch_targets)}")
|
|
if guidance.get("reasoning"):
|
|
prompt_lines.append("Repair guidance:")
|
|
prompt_lines.extend([f"- {item}" for item in guidance.get("reasoning") or []])
|
|
return {
|
|
"mode": "repair_loop",
|
|
"phase": phase.get("code"),
|
|
"phase_title": phase.get("title"),
|
|
"result": result,
|
|
"failure_class": failure_class,
|
|
"recommended_focus": recommended_focus,
|
|
"patch_targets": list(phase_patch_targets),
|
|
"validation_entrypoint": failed_command,
|
|
"phase_tasks": list(phase.get("tasks") or []),
|
|
"phase_files": list(phase.get("files") or []),
|
|
"suggested_files": list(guidance.get("suggested_files") or []),
|
|
"suggested_commands": list(guidance.get("suggested_commands") or []),
|
|
"reasoning": list(guidance.get("reasoning") or []),
|
|
"failing_command_preview": command_preview,
|
|
"failing_stdout_tail": _truncate_output(str((failed_run or {}).get("stdout") or ""), 1200),
|
|
"failing_stderr_tail": _truncate_output(str((failed_run or {}).get("stderr") or ""), 1200),
|
|
"prompt_text": "\n".join(prompt_lines),
|
|
}
|
|
|
|
|
|
def _classify_failure(*, result: str, blocker: Optional[str], failed_command: Optional[str], output: str) -> str:
|
|
if result == "passed":
|
|
return "none"
|
|
if blocker is not None:
|
|
return f"blocker:{blocker}"
|
|
haystack = output.lower()
|
|
command = str(failed_command or "").lower()
|
|
if "modulenotfounderror" in haystack or "importerror" in haystack:
|
|
return "missing_dependency_or_import"
|
|
if "no such file or directory" in haystack:
|
|
return "missing_file_or_artifact"
|
|
if "assertionerror" in haystack or "failed" in haystack:
|
|
return "validation_failure"
|
|
if "not implemented" in haystack:
|
|
return "missing_implementation"
|
|
if "run_launch_readiness.py" in command:
|
|
return "readiness_gap"
|
|
if "run_alert_dispatcher.py" in command:
|
|
return "alert_dispatch_gap"
|
|
if "run_trading_runtime.py" in command:
|
|
return "runtime_gap"
|
|
if "run_review_packet.py" in command:
|
|
return "decision_packet_gap"
|
|
return "general_phase_failure"
|
|
|
|
|
|
def _recommend_focus_area(*, phase: dict, failed_command: Optional[str], failure_class: str) -> str:
|
|
command = str(failed_command or "").lower()
|
|
if failure_class.startswith("blocker:"):
|
|
return "resolve_external_blocker"
|
|
if "launch_readiness" in command or failure_class == "readiness_gap":
|
|
return "readiness_and_boundary_hardening"
|
|
if "alert_dispatcher" in command or failure_class == "alert_dispatch_gap":
|
|
return "alert_delivery_and_dispatch"
|
|
if "trading_runtime" in command or failure_class == "runtime_gap":
|
|
return "runtime_scheduler_and_status"
|
|
if "review_packet" in command or failure_class == "decision_packet_gap":
|
|
return "decision_packet_and_operator_view"
|
|
return f"phase:{phase.get('code', 'unknown').lower()}"
|
|
|
|
|
|
def _build_phase_patch_targets(*, phase: dict, failed_command: Optional[str]) -> list[str]:
|
|
return _prioritize_files_for_failed_command(phase, failed_command)[:4]
|
|
|
|
|
|
def _build_repair_metadata(*, phase: dict, result: str, blocker: Optional[str], failed_command: Optional[str], command_results: list[dict], guidance: dict) -> dict:
|
|
failed_run = next((item for item in command_results if item.get("returncode") != 0), None)
|
|
combined = _combined_output(failed_run) if failed_run is not None else ""
|
|
failure_class = _classify_failure(result=result, blocker=blocker, failed_command=failed_command, output=combined)
|
|
recommended_focus = _recommend_focus_area(phase=phase, failed_command=failed_command, failure_class=failure_class)
|
|
phase_patch_targets = _build_phase_patch_targets(phase=phase, failed_command=failed_command)
|
|
return {
|
|
"failure_class": failure_class,
|
|
"recommended_focus": recommended_focus,
|
|
"phase_patch_targets": phase_patch_targets,
|
|
"phase_validation_entrypoint": failed_command,
|
|
"repair_loop_ready": result in {"needs_work", "blocked"},
|
|
"guidance_version": "v1",
|
|
"suggested_files": list(guidance.get("suggested_files") or []),
|
|
"repair_prompt_context": _build_repair_prompt_context(
|
|
phase=phase,
|
|
result=result,
|
|
failed_command=failed_command,
|
|
failed_run=failed_run,
|
|
guidance=guidance,
|
|
failure_class=failure_class,
|
|
recommended_focus=recommended_focus,
|
|
phase_patch_targets=phase_patch_targets,
|
|
),
|
|
}
|
|
|
|
|
|
def _execute_phase(*, phase: dict, timeout_seconds: int, env: Optional[dict] = None) -> dict:
|
|
started_ts = _now_iso()
|
|
runtime_env = _resolve_rollout_env(env)
|
|
command_results = []
|
|
informational_failures = []
|
|
result = "passed"
|
|
blocker = None
|
|
failed_command = None
|
|
summary = f"Phase {phase['code']} passed all validation commands."
|
|
for command in phase.get("validation_commands") or []:
|
|
run = _run_command(command=command, timeout_seconds=timeout_seconds, env=runtime_env)
|
|
command_results.append(run)
|
|
if run["returncode"] == 0:
|
|
continue
|
|
if _should_treat_as_informational_failure(phase=phase, command=command, run=run):
|
|
informational_failures.append(command)
|
|
continue
|
|
failed_command = command
|
|
detected_blocker = _detect_major_blocker(run)
|
|
if detected_blocker is not None:
|
|
result = "blocked"
|
|
blocker = detected_blocker
|
|
summary = f"Major blocker detected while running: {command}"
|
|
break
|
|
result = "needs_work"
|
|
summary = f"Validation failed for phase {phase['code']} on command: {command}"
|
|
break
|
|
phase_result = {
|
|
"phase": phase["code"],
|
|
"title": phase["title"],
|
|
"goal": phase["goal"],
|
|
"started_ts": started_ts,
|
|
"finished_ts": _now_iso(),
|
|
"result": result,
|
|
"blocker": blocker,
|
|
"summary": summary,
|
|
"tasks": list(phase.get("tasks") or []),
|
|
"files": list(phase.get("files") or []),
|
|
"commands": command_results,
|
|
"deliverables": list(phase.get("deliverables") or []),
|
|
"failed_command": failed_command,
|
|
}
|
|
if phase_result["result"] == "passed" and informational_failures:
|
|
phase_result["summary"] = f"Phase {phase['code']} passed core validations with informational failures: {', '.join(informational_failures)}"
|
|
phase_result = _retry_phase_with_seeded_env(phase=phase, phase_result=phase_result, timeout_seconds=timeout_seconds, env=runtime_env)
|
|
return _finalize_phase_result(phase=phase, phase_result=phase_result)
|
|
|
|
def run_until_blocked(env=None) -> dict:
|
|
env = dict(os.environ if env is None else env)
|
|
status = read_status()
|
|
if status.get("state") == "not_started":
|
|
status = start_rollout()
|
|
max_rounds = max(_get_int(env, "MONITORING_ALERT_ROLLOUT_MAX_ROUNDS", 10), 1)
|
|
timeout_seconds = max(_get_int(env, "MONITORING_ALERT_COMMAND_TIMEOUT_SECONDS", DEFAULT_TIMEOUT_SECONDS), 30)
|
|
for _ in range(max_rounds):
|
|
current = status.get("current_phase")
|
|
if current is None:
|
|
status["state"] = "completed"
|
|
status["note"] = "All phases completed."
|
|
write_status(status)
|
|
return status
|
|
phase = _phase_by_code(current)
|
|
if phase is None:
|
|
raise RuntimeError(f"Unknown current phase: {current}")
|
|
phase_result = _execute_phase(phase=phase, timeout_seconds=timeout_seconds, env=env)
|
|
status["last_run"] = phase_result
|
|
history = list(status.get("history") or [])
|
|
history.append({"ts": _now_iso(), "event": "phase_run", "phase": current, "result": phase_result["result"], "blocker": phase_result.get("blocker")})
|
|
status["history"] = history[-100:]
|
|
status["updated_ts"] = _now_iso()
|
|
if phase_result["result"] == "passed":
|
|
status = advance_rollout()
|
|
status["last_run"] = phase_result
|
|
write_status(status)
|
|
return status
|
|
if phase_result["result"] == "blocked":
|
|
status["state"] = "blocked"
|
|
status["blocked_reason"] = phase_result.get("blocker")
|
|
status["note"] = phase_result.get("summary") or "Major blocker detected."
|
|
status["last_run"] = phase_result
|
|
write_status(status)
|
|
return status
|
|
status["state"] = "needs_work"
|
|
status["blocked_reason"] = None
|
|
status["note"] = phase_result.get("summary") or "Phase validation failed and needs implementation work."
|
|
status["last_run"] = phase_result
|
|
write_status(status)
|
|
return status
|
|
status["state"] = "needs_work"
|
|
status["note"] = f"Stopped after reaching MONITORING_ALERT_ROLLOUT_MAX_ROUNDS={max_rounds}."
|
|
write_status(status)
|
|
return status
|
|
|
|
|
|
def _build_bridge_input(loop_payload: dict) -> dict:
|
|
context = dict(loop_payload.get("repair_prompt_context") or {})
|
|
return {
|
|
"protocol_version": "repair-bridge/v1",
|
|
"mode": "repair_loop",
|
|
"phase": loop_payload.get("phase"),
|
|
"phase_title": loop_payload.get("phase_title"),
|
|
"failure_class": loop_payload.get("failure_class"),
|
|
"recommended_focus": loop_payload.get("recommended_focus"),
|
|
"phase_patch_targets": list(loop_payload.get("phase_patch_targets") or []),
|
|
"validation_entrypoint": loop_payload.get("validation_entrypoint"),
|
|
"repair_prompt_context": context,
|
|
}
|
|
|
|
|
|
def _parse_bridge_output(run: dict) -> dict:
|
|
stdout = str(run.get("stdout") or "").strip()
|
|
if not stdout:
|
|
return {"protocol_version": "repair-bridge/v1", "parsed": False, "error": "empty_stdout"}
|
|
try:
|
|
payload = json.loads(stdout)
|
|
return {"protocol_version": payload.get("protocol_version", "repair-bridge/v1"), "parsed": True, "payload": payload}
|
|
except json.JSONDecodeError:
|
|
return {"protocol_version": "repair-bridge/v1", "parsed": False, "error": "invalid_json_stdout", "stdout_preview": _truncate_output(stdout, 800)}
|
|
|
|
|
|
def _validate_bridge_output_payload(payload: dict) -> list[str]:
|
|
errors = []
|
|
for key in ["status", "summary", "changed_files"]:
|
|
if key not in payload:
|
|
errors.append(f"missing_field:{key}")
|
|
if "changed_files" in payload and not isinstance(payload.get("changed_files"), list):
|
|
errors.append("invalid_field:changed_files")
|
|
return errors
|
|
|
|
|
|
def _bridge_output_summary(bridge_output: dict) -> str:
|
|
if not bridge_output.get("parsed"):
|
|
return f"bridge output parse failed: {bridge_output.get('error')}"
|
|
payload = dict(bridge_output.get("payload") or {})
|
|
validation_errors = _validate_bridge_output_payload(payload)
|
|
if validation_errors:
|
|
return f"bridge output schema issues: {', '.join(validation_errors)}"
|
|
return str(payload.get("summary") or "bridge output parsed")
|
|
|
|
|
|
def _bridge_changed_files(bridge_output: dict) -> list[str]:
|
|
if not bridge_output.get("parsed"):
|
|
return []
|
|
payload = dict(bridge_output.get("payload") or {})
|
|
changed_files = payload.get("changed_files")
|
|
if not isinstance(changed_files, list):
|
|
return []
|
|
return [str(item) for item in changed_files]
|
|
|
|
|
|
def _bridge_tests_run(bridge_output: dict) -> list[str]:
|
|
if not bridge_output.get("parsed"):
|
|
return []
|
|
payload = dict(bridge_output.get("payload") or {})
|
|
tests_run = payload.get("tests_run")
|
|
if not isinstance(tests_run, list):
|
|
return []
|
|
return [str(item) for item in tests_run]
|
|
|
|
|
|
def _bridge_success_guess(bridge_output: dict) -> bool:
|
|
if not bridge_output.get("parsed"):
|
|
return False
|
|
payload = dict(bridge_output.get("payload") or {})
|
|
if _validate_bridge_output_payload(payload):
|
|
return False
|
|
return str(payload.get("status") or "").lower() in {"applied", "success", "fixed"}
|
|
|
|
|
|
def _should_rerun_after_bridge(bridge_output: dict, *, env: dict) -> bool:
|
|
if not _get_bool(env, "MONITORING_ALERT_AUTO_RERUN_ENABLED", True):
|
|
return False
|
|
if not bridge_output.get("parsed"):
|
|
return False
|
|
payload = dict(bridge_output.get("payload") or {})
|
|
if _validate_bridge_output_payload(payload):
|
|
return False
|
|
status = str(payload.get("status") or "").lower()
|
|
success_guess = payload.get("success_guess")
|
|
changed_files = _bridge_changed_files(bridge_output)
|
|
if status not in {"applied", "success", "fixed"}:
|
|
return False
|
|
if success_guess is False:
|
|
return False
|
|
return bool(changed_files)
|
|
|
|
|
|
def _select_rerun_commands(bridge_output: dict) -> list[str]:
|
|
tests_run = _bridge_tests_run(bridge_output)
|
|
if tests_run:
|
|
return tests_run
|
|
changed_files = _bridge_changed_files(bridge_output)
|
|
commands = []
|
|
if any("src/services/alerts/" in item or "src/api/alerts.py" in item for item in changed_files):
|
|
commands.append(_cmd("run_alert_dispatcher.py"))
|
|
if any("src/services/runtime/" in item or "src/api/trading_runtime.py" in item for item in changed_files):
|
|
commands.append(_cmd("run_trading_runtime.py"))
|
|
if any("src/services/readiness/" in item or "src/web/" in item or "src/config/settings.py" in item for item in changed_files):
|
|
commands.append(_cmd("run_launch_readiness.py"))
|
|
if any("src/services/decision/" in item or "src/services/operator/" in item for item in changed_files):
|
|
commands.append(_cmd("run_review_packet.py"))
|
|
if changed_files and all(item.startswith("docs/") for item in changed_files):
|
|
return []
|
|
return _dedupe(commands)
|
|
|
|
|
|
def _run_focused_rerun(*, commands: list[str]) -> list[dict]:
|
|
return [_run_command(command=command, timeout_seconds=DEFAULT_TIMEOUT_SECONDS) for command in commands]
|
|
|
|
|
|
def _bridge_result_passed(payload: dict) -> bool:
|
|
result = dict(payload.get("bridge_result") or {})
|
|
return int(result.get("returncode") or 1) == 0
|
|
|
|
|
|
def _run_post_repair_validation(*, env: dict, bridge_output: dict) -> dict:
|
|
rerun_commands = _select_rerun_commands(bridge_output)
|
|
focused_payload = None
|
|
if rerun_commands:
|
|
focused = _run_focused_rerun(commands=rerun_commands)
|
|
focused_payload = {"state": "focused_rerun_completed", "commands": focused, "selected_commands": rerun_commands}
|
|
if any(item.get("returncode") != 0 for item in focused):
|
|
focused_payload["state"] = "needs_work"
|
|
focused_payload["note"] = "Focused rerun failed after repair."
|
|
return focused_payload
|
|
rerun_env = dict(env)
|
|
rerun_env["MONITORING_ALERT_ROLLOUT_MODE"] = "run-until-blocked"
|
|
rerun_payload = run_until_blocked(env=rerun_env)
|
|
if focused_payload:
|
|
rerun_payload = {**rerun_payload, "focused_rerun": focused_payload}
|
|
return rerun_payload
|
|
|
|
|
|
def _should_advance_after_rerun(payload: dict) -> bool:
|
|
rerun_state = dict(payload.get("rerun_state_snapshot") or {})
|
|
return rerun_state.get("state") in {"running", "completed"}
|
|
|
|
|
|
def _build_repair_bridge_protocol_doc() -> dict:
|
|
return {
|
|
"protocol_version": "repair-bridge/v1",
|
|
"stdin": {"required_fields": ["protocol_version", "mode", "phase", "phase_title", "failure_class", "recommended_focus", "phase_patch_targets", "validation_entrypoint", "repair_prompt_context"]},
|
|
"stdout": {"required_fields": ["status", "summary", "changed_files"], "optional_fields": ["notes", "success_guess", "tests_run"]},
|
|
}
|
|
|
|
|
|
def _build_repair_bridge_example_output() -> dict:
|
|
return {
|
|
"protocol_version": "repair-bridge/v1",
|
|
"status": "applied",
|
|
"summary": "Implemented missing alert dispatcher module and updated runtime hook.",
|
|
"changed_files": ["src/services/alerts/alert_dispatcher_service.py", "src/services/runtime/trading_runtime_service.py"],
|
|
"notes": ["Focused on N2 alert delivery gap"],
|
|
"success_guess": True,
|
|
"tests_run": [_cmd("run_alert_dispatcher.py")],
|
|
}
|
|
|
|
|
|
def _bridge_repair_signal(payload: dict) -> dict:
|
|
bridge_output = dict(payload.get("bridge_output") or {})
|
|
return {
|
|
"summary": _bridge_output_summary(bridge_output),
|
|
"success_guess": _bridge_success_guess(bridge_output),
|
|
"changed_files": _bridge_changed_files(bridge_output),
|
|
"tests_run": _bridge_tests_run(bridge_output),
|
|
}
|
|
|
|
|
|
def _bridge_agent_prompt(loop_payload: dict) -> str:
|
|
return json.dumps(_build_bridge_input(loop_payload), ensure_ascii=False, indent=2)
|
|
|
|
|
|
def _run_bridge_command(*, command: str, bridge_input: dict) -> dict:
|
|
started = _now_iso()
|
|
stdin_text = json.dumps(bridge_input, ensure_ascii=False)
|
|
try:
|
|
completed = subprocess.run(
|
|
["bash", "-lc", command],
|
|
cwd=str(ROOT),
|
|
input=stdin_text,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=DEFAULT_TIMEOUT_SECONDS,
|
|
encoding="utf-8",
|
|
errors="replace",
|
|
)
|
|
return {
|
|
"command": command,
|
|
"started_ts": started,
|
|
"finished_ts": _now_iso(),
|
|
"returncode": int(completed.returncode),
|
|
"stdin": _truncate_output(stdin_text, 2000),
|
|
"stdout": str(completed.stdout or "")[-8000:],
|
|
"stderr": str(completed.stderr or "")[-8000:],
|
|
}
|
|
except subprocess.TimeoutExpired as exc:
|
|
return {
|
|
"command": command,
|
|
"started_ts": started,
|
|
"finished_ts": _now_iso(),
|
|
"returncode": 124,
|
|
"stdin": _truncate_output(stdin_text, 2000),
|
|
"stdout": str(exc.stdout or "")[-8000:],
|
|
"stderr": str(exc.stderr or "")[-8000:],
|
|
}
|
|
|
|
|
|
def _build_repair_agent_prompt(loop_payload: dict) -> str:
|
|
return _bridge_agent_prompt(loop_payload)
|
|
|
|
|
|
def _rerun_hint_payload(loop_payload: dict) -> dict:
|
|
return {
|
|
"phase": loop_payload.get("phase"),
|
|
"validation_entrypoint": loop_payload.get("validation_entrypoint"),
|
|
"suggested_commands": list((loop_payload.get("repair_prompt_context") or {}).get("suggested_commands") or []),
|
|
}
|
|
|
|
|
|
def _repair_bridge_summary(payload: dict) -> str:
|
|
if not payload.get("bridge_invoked"):
|
|
return "repair bridge ready but not invoked"
|
|
result = dict(payload.get("bridge_result") or {})
|
|
summary = f"repair bridge invoked with returncode={result.get('returncode')}"
|
|
bridge_output = dict(payload.get("bridge_output") or {})
|
|
if bridge_output:
|
|
summary += f"; {_bridge_output_summary(bridge_output)}"
|
|
if payload.get("post_repair_rerun") is not None:
|
|
summary += f"; post-repair rerun state={dict(payload.get('post_repair_rerun') or {}).get('state')}"
|
|
return summary
|
|
|
|
|
|
def _rerun_state_snapshot(payload: dict) -> dict:
|
|
rerun = dict(payload.get("post_repair_rerun") or {})
|
|
return {"state": rerun.get("state"), "current_phase": rerun.get("current_phase"), "note": rerun.get("note")}
|
|
|
|
|
|
def _enrich_bridge_payload(payload: dict) -> dict:
|
|
loop_payload = dict(payload)
|
|
enriched = {
|
|
**payload,
|
|
"repair_agent_prompt": _build_repair_agent_prompt(loop_payload),
|
|
"rerun_hint": _rerun_hint_payload(loop_payload),
|
|
"summary": _repair_bridge_summary(payload),
|
|
"bridge_protocol": _build_repair_bridge_protocol_doc(),
|
|
}
|
|
if payload.get("bridge_invoked"):
|
|
enriched["bridge_passed"] = _bridge_result_passed(payload)
|
|
enriched["bridge_output_signal"] = _bridge_repair_signal(payload)
|
|
if payload.get("post_repair_rerun") is not None:
|
|
enriched["rerun_summary"] = _repair_bridge_summary(payload)
|
|
enriched["rerun_state_snapshot"] = _rerun_state_snapshot(payload)
|
|
return enriched
|
|
|
|
|
|
def run_repair_loop(env=None) -> dict:
|
|
env = dict(os.environ if env is None else env)
|
|
status = read_status()
|
|
last_run = dict(status.get("last_run") or {})
|
|
repair_metadata = dict(last_run.get("repair_metadata") or {})
|
|
prompt_context = dict(repair_metadata.get("repair_prompt_context") or {})
|
|
if status.get("state") != "needs_work" or not prompt_context:
|
|
return {"state": "repair_loop_unavailable", "updated_ts": _now_iso(), "reason": "No needs_work phase with repair prompt/context is available.", "current_phase": status.get("current_phase")}
|
|
iteration = int(((status.get("repair_loop") or {}).get("iteration") or 0)) + 1
|
|
payload = {
|
|
"state": "repair_loop_ready",
|
|
"updated_ts": _now_iso(),
|
|
"iteration": iteration,
|
|
"phase": last_run.get("phase"),
|
|
"phase_title": last_run.get("title"),
|
|
"failure_class": repair_metadata.get("failure_class"),
|
|
"recommended_focus": repair_metadata.get("recommended_focus"),
|
|
"phase_patch_targets": list(repair_metadata.get("phase_patch_targets") or []),
|
|
"validation_entrypoint": repair_metadata.get("phase_validation_entrypoint"),
|
|
"repair_prompt_context": prompt_context,
|
|
"auto_repair_enabled": _get_bool(env, "MONITORING_ALERT_AUTO_REPAIR_ENABLED", False),
|
|
"next_action": "hand_off_to_repair_agent",
|
|
}
|
|
_write_repair_loop_state(payload)
|
|
return payload
|
|
|
|
|
|
def run_repair_bridge(env=None) -> dict:
|
|
env = dict(os.environ if env is None else env)
|
|
loop_payload = run_repair_loop(env=env)
|
|
if loop_payload.get("state") != "repair_loop_ready":
|
|
return loop_payload
|
|
bridge_command = str(env.get("MONITORING_ALERT_REPAIR_BRIDGE_COMMAND") or "").strip()
|
|
auto_repair_enabled = _get_bool(env, "MONITORING_ALERT_AUTO_REPAIR_ENABLED", False)
|
|
if not auto_repair_enabled or not bridge_command:
|
|
payload = _enrich_bridge_payload({**loop_payload, "state": "repair_bridge_ready", "bridge_invoked": False, "next_action": "invoke_external_repair_agent"})
|
|
_write_repair_loop_state(payload)
|
|
return payload
|
|
bridge_input = _build_bridge_input(loop_payload)
|
|
run = _run_bridge_command(command=bridge_command, bridge_input=bridge_input)
|
|
bridge_output = _parse_bridge_output(run)
|
|
payload = {
|
|
**loop_payload,
|
|
"state": "repair_bridge_invoked",
|
|
"bridge_invoked": True,
|
|
"bridge_command": bridge_command,
|
|
"bridge_input": bridge_input,
|
|
"bridge_result": run,
|
|
"bridge_output": bridge_output,
|
|
"next_action": "rerun_validation_after_repair",
|
|
}
|
|
if _should_rerun_after_bridge(bridge_output, env=env):
|
|
payload["post_repair_rerun"] = _run_post_repair_validation(env=env, bridge_output=bridge_output)
|
|
payload = _enrich_bridge_payload(payload)
|
|
if payload.get("post_repair_rerun") is not None:
|
|
payload["next_action"] = "phase_advanced" if _should_advance_after_rerun(payload) else "inspect_rerun_result"
|
|
_write_repair_loop_state(payload)
|
|
return payload
|
|
|
|
|
|
def _upsert_anchor_block(content: str, header: str, body: str) -> str:
|
|
start = content.find(header)
|
|
block = f"{header}\n{body.rstrip()}\n"
|
|
if start == -1:
|
|
return content.rstrip() + "\n\n" + block
|
|
next_index = content.find("\n## ", start + len(header))
|
|
if next_index == -1:
|
|
return content[:start] + block
|
|
return content[:start] + block + content[next_index + 1:]
|
|
|
|
|
|
def _write_session_anchor(status: dict) -> None:
|
|
if not SESSION_PATH.exists():
|
|
return
|
|
current_phase = status.get("current_phase") or "none"
|
|
repair_loop = dict(status.get("repair_loop") or {})
|
|
body = [f"- state: {status.get('state')}", f"- current_phase: {current_phase}", f"- updated_ts: {status.get('updated_ts')}"]
|
|
if repair_loop:
|
|
body.append(f"- repair_iteration: {repair_loop.get('iteration')}")
|
|
body.append(f"- repair_focus: {repair_loop.get('recommended_focus')}")
|
|
body.append(f"- repair_next_action: {repair_loop.get('next_action')}")
|
|
content = SESSION_PATH.read_text(encoding="utf-8")
|
|
SESSION_PATH.write_text(_upsert_anchor_block(content, "## Automation Status", "\n".join(body)), encoding="utf-8")
|
|
|
|
|
|
def _write_tasks_anchor(status: dict) -> None:
|
|
if not TASKS_PATH.exists():
|
|
return
|
|
current_phase = status.get("current_phase") or "none"
|
|
repair_loop = dict(status.get("repair_loop") or {})
|
|
body = [f"- [ ] state: {status.get('state')}", f"- [ ] current_phase: {current_phase}", f"- [ ] updated_ts: {status.get('updated_ts')}"]
|
|
if repair_loop:
|
|
body.append(f"- [ ] repair_iteration: {repair_loop.get('iteration')}")
|
|
body.append(f"- [ ] repair_focus: {repair_loop.get('recommended_focus')}")
|
|
body.append(f"- [ ] repair_next_action: {repair_loop.get('next_action')}")
|
|
content = TASKS_PATH.read_text(encoding="utf-8")
|
|
TASKS_PATH.write_text(_upsert_anchor_block(content, "## Automation Status", "\n".join(body)), encoding="utf-8")
|
|
|
|
|
|
def _write_status_anchors(status: dict) -> None:
|
|
_write_session_anchor(status)
|
|
_write_tasks_anchor(status)
|
|
|
|
|
|
def _append_patch_round(status: dict, payload: dict) -> None:
|
|
repair_loop = dict(status.get("repair_loop") or {})
|
|
rounds = list(repair_loop.get("patch_rounds") or [])
|
|
bridge_output = dict(payload.get("bridge_output") or {})
|
|
bridge_payload = dict(bridge_output.get("payload") or {}) if bridge_output.get("parsed") else {}
|
|
rounds.append({
|
|
"ts": _now_iso(),
|
|
"iteration": payload.get("iteration"),
|
|
"changed_files": _bridge_changed_files(bridge_output),
|
|
"tests_run": _bridge_tests_run(bridge_output),
|
|
"success_guess": bridge_payload.get("success_guess"),
|
|
"rerun_state": dict(payload.get("rerun_state_snapshot") or {}).get("state"),
|
|
"phase_advanced": payload.get("next_action") == "phase_advanced",
|
|
})
|
|
repair_loop["patch_rounds"] = rounds[-20:]
|
|
status["repair_loop"] = repair_loop
|
|
|
|
|
|
def _sync_patch_rounds_to_payload(status: dict, payload: dict) -> dict:
|
|
repair_loop = dict(status.get("repair_loop") or {})
|
|
if repair_loop.get("patch_rounds"):
|
|
payload["patch_rounds"] = list(repair_loop.get("patch_rounds") or [])
|
|
return payload
|
|
|
|
|
|
def _write_repair_loop_state(payload: dict) -> None:
|
|
status = read_status()
|
|
status["repair_loop"] = payload
|
|
if payload.get("bridge_invoked"):
|
|
_append_patch_round(status, payload)
|
|
payload = _sync_patch_rounds_to_payload(status, payload)
|
|
status["repair_loop"] = payload
|
|
history = list(status.get("history") or [])
|
|
history.append({"ts": _now_iso(), "event": "repair_bridge" if payload.get("bridge_invoked") else "repair_loop_ready", "phase": payload.get("phase"), "iteration": payload.get("iteration")})
|
|
status["history"] = history[-100:]
|
|
status["updated_ts"] = _now_iso()
|
|
write_status(status)
|
|
_write_status_anchors(status)
|
|
|
|
|
|
def render_text(payload: dict) -> str:
|
|
lines = [f"Monitoring alert rollout state: {payload.get('state', 'unknown')}"]
|
|
if payload.get("current_phase"):
|
|
lines.append(f"Current phase: {payload['current_phase']}")
|
|
completed = payload.get("completed_phases") or []
|
|
lines.append(f"Completed phases: {', '.join(completed) if completed else 'none'}")
|
|
if payload.get("blocked_reason"):
|
|
lines.append(f"Blocked reason: {payload['blocked_reason']}")
|
|
if payload.get("note"):
|
|
lines.append(f"Note: {payload['note']}")
|
|
lines.append(f"Task doc: {payload.get('doc_path', str(DOC_PATH))}")
|
|
last_run = payload.get("last_run") or {}
|
|
if last_run:
|
|
lines.append("Last run:")
|
|
lines.append(f"- Phase: {last_run.get('phase')} {last_run.get('title', '')}")
|
|
lines.append(f"- Result: {last_run.get('result')}")
|
|
lines.append(f"- Summary: {last_run.get('summary')}")
|
|
for command in last_run.get("commands") or []:
|
|
preview = _summarize_command_output(command)
|
|
suffix = f" :: {preview}" if preview else ""
|
|
lines.append(f" - [{command.get('returncode')}] {command.get('command')}{suffix}")
|
|
guidance = last_run.get("next_step_guidance") or {}
|
|
if guidance:
|
|
lines.append("Next-step guidance:")
|
|
if guidance.get("priority_tasks"):
|
|
lines.append(f"- Priority tasks: {', '.join(guidance['priority_tasks'])}")
|
|
if guidance.get("suggested_files"):
|
|
lines.append(f"- Suggested files: {', '.join(guidance['suggested_files'])}")
|
|
if guidance.get("suggested_commands"):
|
|
lines.append(f"- Suggested commands: {', '.join(guidance['suggested_commands'])}")
|
|
for item in guidance.get("reasoning") or []:
|
|
lines.append(f"- {item}")
|
|
repair_loop = payload.get("repair_loop") or {}
|
|
if repair_loop:
|
|
lines.append("Repair loop state:")
|
|
lines.append(f"- Iteration: {repair_loop.get('iteration')}")
|
|
lines.append(f"- Phase: {repair_loop.get('phase')} {repair_loop.get('phase_title', '')}")
|
|
lines.append(f"- Failure class: {repair_loop.get('failure_class')}")
|
|
lines.append(f"- Recommended focus: {repair_loop.get('recommended_focus')}")
|
|
if repair_loop.get("summary"):
|
|
lines.append(f"- Bridge summary: {repair_loop.get('summary')}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def main(env=None) -> int:
|
|
env = os.environ if env is None else env
|
|
output_json = _get_bool(env, "MONITORING_ALERT_ROLLOUT_OUTPUT_JSON", False)
|
|
mode = str(env.get("MONITORING_ALERT_ROLLOUT_MODE", "plan")).strip().lower()
|
|
if mode == "plan":
|
|
payload = build_rollout_plan()
|
|
elif mode == "status":
|
|
payload = read_status()
|
|
elif mode == "start":
|
|
payload = start_rollout()
|
|
elif mode == "advance":
|
|
payload = advance_rollout()
|
|
elif mode == "block":
|
|
blocker = str(env.get("MONITORING_ALERT_BLOCKER", "manual_blocker")).strip() or "manual_blocker"
|
|
note = str(env.get("MONITORING_ALERT_BLOCKER_NOTE", "")).strip()
|
|
payload = block_rollout(blocker=blocker, note=note)
|
|
elif mode == "complete":
|
|
payload = complete_rollout()
|
|
elif mode == "run-until-blocked":
|
|
payload = run_until_blocked(env=env)
|
|
elif mode == "repair-loop":
|
|
payload = run_repair_loop(env=env)
|
|
elif mode == "repair-bridge":
|
|
payload = run_repair_bridge(env=env)
|
|
else:
|
|
raise SystemExit(f"Unsupported MONITORING_ALERT_ROLLOUT_MODE: {mode}")
|
|
if output_json:
|
|
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
|
else:
|
|
print(render_text(payload))
|
|
return 0 if payload.get("state") not in {"blocked", "failed"} else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
|