Codex
371 lines
15 KiB
Python
371 lines
15 KiB
Python
from datetime import datetime, timedelta, timezone
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://user:pass@localhost:5432/ai_ict_test")
|
|
|
|
from scripts import run_trading_runtime
|
|
from src.api.server import route_request
|
|
from src.services.runtime import TradingRuntimeScheduler, TradingRuntimeSupervisor
|
|
from src.services.runtime.runtime_task_watchdog_service import RuntimeTaskWatchdogService
|
|
|
|
|
|
class FakeInstrumentRepository:
|
|
def fetch_instrument_by_symbol(self, symbol):
|
|
return {"id": 1 if symbol == "BTC-USDT" else 2, "symbol": symbol}
|
|
|
|
|
|
class FakeTradingPlanService:
|
|
def __init__(self):
|
|
self.repository = FakeInstrumentRepository()
|
|
self.build_calls = 0
|
|
|
|
def build_plan(self, **kwargs):
|
|
self.build_calls += 1
|
|
return type("Plan", (), {"status": "active"})()
|
|
|
|
def persist_plan(self, candidate):
|
|
return 101
|
|
|
|
|
|
class FakeMarketState:
|
|
def __init__(self, *, freshness="market_data_fresh", stream="market_stream_connected", gap="market_data_gap_clear"):
|
|
self.freshness = freshness
|
|
self.stream = stream
|
|
self.gap = gap
|
|
|
|
def to_dict(self):
|
|
return {
|
|
"freshness": {"code": self.freshness},
|
|
"stream_state": {"code": self.stream},
|
|
"data_gap_state": {"code": self.gap},
|
|
}
|
|
|
|
|
|
class FakeMarketStateService:
|
|
def __init__(self, market_state=None):
|
|
self.market_state = market_state or FakeMarketState()
|
|
|
|
def build_market_state(self, **kwargs):
|
|
return self.market_state
|
|
|
|
def persist_market_state_snapshot(self, **kwargs):
|
|
return 301
|
|
|
|
|
|
class FakeBrokerSnapshot:
|
|
status = "broker_state_ok"
|
|
error = None
|
|
|
|
def to_dict(self):
|
|
return {
|
|
"status": self.status,
|
|
"error": self.error,
|
|
"account": {"equity": 1000.0},
|
|
"positions": [],
|
|
"orders": [],
|
|
"fills": [],
|
|
"alerts": [],
|
|
}
|
|
|
|
|
|
class FakeBrokerReadModelService:
|
|
def fetch_snapshot(self, **kwargs):
|
|
return FakeBrokerSnapshot()
|
|
|
|
|
|
class FakeSetupWatcherService:
|
|
def build_candidate(self, **kwargs):
|
|
return type("Candidate", (), {"status": "blocked"})()
|
|
|
|
def persist_candidate(self, candidate):
|
|
return 401 if candidate is not None else None
|
|
|
|
|
|
class FakeDecisionPacketService:
|
|
def build_persist_and_alert(self, **kwargs):
|
|
return {
|
|
"id": 501,
|
|
"status": "actionable",
|
|
"action_recommendation": "confirm_ticket",
|
|
"alerts": [
|
|
{
|
|
"alert_type": "candidate_actionable",
|
|
"payload": {
|
|
"visual_trigger_type": "candidate_actionable_watch",
|
|
"visual_trigger_policy": {
|
|
"status": "triggered",
|
|
"reason": "high_value_actionable_candidate",
|
|
"trigger_types": ["candidate_actionable_watch"],
|
|
},
|
|
"visual_watch_reservation": {"status": "reserved", "watch_run_id": 801},
|
|
},
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
class FakeReconciliationService:
|
|
def reconcile_instrument(self, **kwargs):
|
|
return {
|
|
"status": "ok",
|
|
"matched_execution_ids": [91],
|
|
"imported_execution_ids": [92],
|
|
"journal_updates": [11, 12],
|
|
}
|
|
|
|
|
|
class FakeAlertDispatcherService:
|
|
def dispatch_pending(self, **kwargs):
|
|
return {
|
|
"attempted": 0,
|
|
"items": [],
|
|
"silent_window": {
|
|
"active": False,
|
|
"window_label": "00:00-06:00",
|
|
"next_allowed_ts": None,
|
|
},
|
|
"suppressed_silent_window": 0,
|
|
"suppressed_noise_policy": 0,
|
|
"workbench_only": 0,
|
|
"policy_external_allowed": 0,
|
|
}
|
|
|
|
|
|
class FakeVisualReviewService:
|
|
def __init__(self, payload=None):
|
|
self.payload = payload or {"status": "idle", "attempted": 0, "reviewed": 0, "items": [], "pending_watch_run_ids": []}
|
|
self.calls = []
|
|
|
|
def review_pending_watch_runs(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
return self.payload
|
|
|
|
|
|
class FakeVisualCaptureRetentionService:
|
|
def __init__(self, payload=None):
|
|
self.payload = payload or {
|
|
"status": "completed",
|
|
"root": "captures",
|
|
"retention_hours": 168,
|
|
"eligible_file_count": 2,
|
|
"deleted_file_count": 2,
|
|
"deleted_bytes": 100,
|
|
"retained_file_count": 1,
|
|
"reason": None,
|
|
}
|
|
self.calls = []
|
|
|
|
def cleanup_captures(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
return dict(self.payload)
|
|
|
|
|
|
class FakeResearchCouncilService:
|
|
def __init__(self, *, should_fail=False):
|
|
self.should_fail = should_fail
|
|
self.calls = []
|
|
|
|
def build_and_persist(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
if self.should_fail:
|
|
raise RuntimeError("council unavailable")
|
|
return {
|
|
"id": 601,
|
|
"instrument_id": kwargs["instrument_id"],
|
|
"status": "completed",
|
|
"payload": {
|
|
"decision": {
|
|
"overall_stance": "decision_support_only",
|
|
"committee_confidence": 0.64,
|
|
"recommended_watch_items": ["wait_for_candidate_confirmation"],
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
class TradingRuntimeSchedulerTests(unittest.TestCase):
|
|
def test_run_cycle_records_successful_task_state(self) -> None:
|
|
scheduler = TradingRuntimeScheduler(
|
|
market_state_service=FakeMarketStateService(),
|
|
trading_plan_service=FakeTradingPlanService(),
|
|
setup_watcher_service=FakeSetupWatcherService(),
|
|
broker_read_model_service=FakeBrokerReadModelService(),
|
|
reconciliation_service=FakeReconciliationService(),
|
|
decision_packet_service=FakeDecisionPacketService(),
|
|
alert_dispatcher_service=FakeAlertDispatcherService(),
|
|
visual_review_service=FakeVisualReviewService(
|
|
{
|
|
"status": "completed",
|
|
"attempted": 1,
|
|
"reviewed": 1,
|
|
"items": [
|
|
{
|
|
"watch_run_id": 801,
|
|
"visual_state": "confirm",
|
|
"snapshot_id": 333,
|
|
}
|
|
],
|
|
"pending_watch_run_ids": [801],
|
|
"trigger_type_counts": {"candidate_actionable_watch": 1},
|
|
"review_state_counts": {"confirm": 1},
|
|
}
|
|
),
|
|
visual_capture_retention_service=FakeVisualCaptureRetentionService(),
|
|
research_council_service=FakeResearchCouncilService(),
|
|
)
|
|
with patch("src.services.runtime.trading_runtime_service.settings.trading_runtime_symbols", "BTC-USDT"):
|
|
payload = scheduler.run_cycle()
|
|
|
|
self.assertEqual(payload["scheduler_state"], "running")
|
|
self.assertEqual(payload["current_blockers"], [])
|
|
self.assertEqual(payload["tasks"]["plan"]["last_status"], "ok")
|
|
self.assertEqual(payload["tasks"]["broker_sync"]["details"]["status"], "broker_state_ok")
|
|
self.assertEqual(payload["tasks"]["reconciliation"]["details"]["matched_execution_ids"], [91])
|
|
self.assertEqual(payload["tasks"]["decision_packet"]["details"]["packet_id"], 501)
|
|
self.assertEqual(payload["tasks"]["research_council"]["details"]["run_id"], 601)
|
|
self.assertEqual(payload["tasks"]["research_council"]["last_status"], "ok")
|
|
self.assertEqual(payload["tasks"]["decision_packet"]["details"]["visual_trigger_types"], ["candidate_actionable_watch"])
|
|
self.assertTrue(payload["tasks"]["decision_packet"]["details"]["visual_watch_reserved"])
|
|
self.assertEqual(payload["tasks"]["decision_packet"]["details"]["visual_trigger_policy_statuses"], ["triggered"])
|
|
self.assertEqual(payload["tasks"]["decision_packet"]["details"]["visual_triggered_alert_count"], 1)
|
|
self.assertEqual(payload["tasks"]["visual_watch"]["details"]["visual_reviews_created"], 1)
|
|
self.assertEqual(payload["tasks"]["visual_watch"]["details"]["last_visual_state"], "confirm")
|
|
self.assertEqual(payload["tasks"]["visual_watch"]["details"]["trigger_type_counts"]["candidate_actionable_watch"], 1)
|
|
self.assertEqual(payload["tasks"]["visual_capture_cleanup"]["details"]["deleted_file_count"], 2)
|
|
self.assertFalse(payload["tasks"]["alert_dispatch"]["details"]["silent_window_active"])
|
|
self.assertEqual(payload["tasks"]["alert_dispatch"]["details"]["suppressed_noise_policy"], 0)
|
|
self.assertEqual(payload["runtime_watchdog"]["stalled_task_count"], 0)
|
|
self.assertEqual(payload["latest_broker_state_by_symbol"]["BTC-USDT"]["account"]["equity"], 1000.0)
|
|
self.assertTrue(payload["tasks"]["watch"]["next_run_ts"])
|
|
|
|
def test_run_cycle_enters_degraded_mode_when_market_truth_is_bad(self) -> None:
|
|
scheduler = TradingRuntimeScheduler(
|
|
market_state_service=FakeMarketStateService(FakeMarketState(stream="market_stream_disconnected")),
|
|
trading_plan_service=FakeTradingPlanService(),
|
|
setup_watcher_service=FakeSetupWatcherService(),
|
|
broker_read_model_service=FakeBrokerReadModelService(),
|
|
reconciliation_service=FakeReconciliationService(),
|
|
decision_packet_service=FakeDecisionPacketService(),
|
|
visual_review_service=FakeVisualReviewService(),
|
|
visual_capture_retention_service=FakeVisualCaptureRetentionService(),
|
|
research_council_service=FakeResearchCouncilService(),
|
|
)
|
|
with patch("src.services.runtime.trading_runtime_service.settings.trading_runtime_symbols", "BTC-USDT"):
|
|
payload = scheduler.run_cycle()
|
|
|
|
self.assertTrue(payload["degraded_mode_state"]["is_degraded"])
|
|
self.assertIn("market_stream_degraded:BTC-USDT:market_stream_disconnected", payload["current_blockers"])
|
|
self.assertIn("watch_skipped_market_degraded:BTC-USDT", payload["current_noisy_incidents"])
|
|
self.assertGreaterEqual(len(payload["incidents"]), 1)
|
|
severities = {item["severity"] for item in payload["incidents"]}
|
|
self.assertIn("blocker", severities)
|
|
self.assertIn("noisy_incident", severities)
|
|
|
|
def test_supervisor_run_once_exposes_supervisor_state(self) -> None:
|
|
scheduler = TradingRuntimeScheduler(
|
|
market_state_service=FakeMarketStateService(),
|
|
trading_plan_service=FakeTradingPlanService(),
|
|
setup_watcher_service=FakeSetupWatcherService(),
|
|
broker_read_model_service=FakeBrokerReadModelService(),
|
|
reconciliation_service=FakeReconciliationService(),
|
|
decision_packet_service=FakeDecisionPacketService(),
|
|
alert_dispatcher_service=FakeAlertDispatcherService(),
|
|
visual_review_service=FakeVisualReviewService(),
|
|
visual_capture_retention_service=FakeVisualCaptureRetentionService(),
|
|
research_council_service=FakeResearchCouncilService(),
|
|
)
|
|
supervisor = TradingRuntimeSupervisor(scheduler=scheduler)
|
|
with patch("src.services.runtime.trading_runtime_service.settings.trading_runtime_symbols", "BTC-USDT"):
|
|
payload = supervisor.run_once()
|
|
|
|
self.assertEqual(payload["supervisor_state"], "stopped")
|
|
self.assertEqual(payload["current_blockers"], [])
|
|
|
|
def test_status_snapshot_surfaces_watchdog_warnings_for_stalled_tasks(self) -> None:
|
|
scheduler = TradingRuntimeScheduler(
|
|
market_state_service=FakeMarketStateService(),
|
|
trading_plan_service=FakeTradingPlanService(),
|
|
setup_watcher_service=FakeSetupWatcherService(),
|
|
broker_read_model_service=FakeBrokerReadModelService(),
|
|
reconciliation_service=FakeReconciliationService(),
|
|
decision_packet_service=FakeDecisionPacketService(),
|
|
alert_dispatcher_service=FakeAlertDispatcherService(),
|
|
visual_review_service=FakeVisualReviewService(),
|
|
visual_capture_retention_service=FakeVisualCaptureRetentionService(),
|
|
research_council_service=FakeResearchCouncilService(),
|
|
runtime_task_watchdog_service=RuntimeTaskWatchdogService(stale_multiplier=1, min_stuck_seconds=1),
|
|
)
|
|
stale_ts = (datetime.now(timezone.utc) - timedelta(seconds=30)).isoformat()
|
|
scheduler.tasks["watch"].last_status = "running"
|
|
scheduler.tasks["watch"].run_count = 1
|
|
scheduler.tasks["watch"].last_started_ts = stale_ts
|
|
|
|
payload = scheduler.status_snapshot()
|
|
|
|
self.assertEqual(payload["runtime_watchdog"]["stalled_task_count"], 1)
|
|
self.assertIn("runtime_task_stalled:watch:running_too_long", payload["current_warnings"])
|
|
|
|
|
|
class TradingRuntimeApiTests(unittest.TestCase):
|
|
def test_routes_runtime_status_and_run_once(self) -> None:
|
|
status, payload = route_request(
|
|
"/trading/runtime-status",
|
|
{},
|
|
dependencies={"trading_runtime_status": lambda: {"scheduler_state": "running", "current_blockers": []}},
|
|
)
|
|
run_status, run_payload = route_request(
|
|
"/trading/runtime/run-once",
|
|
{},
|
|
method="POST",
|
|
body={},
|
|
dependencies={"trading_runtime_run_once": lambda: {"scheduler_state": "running", "tasks": {}}},
|
|
)
|
|
|
|
self.assertEqual(status, 200)
|
|
self.assertEqual(payload["scheduler_state"], "running")
|
|
self.assertEqual(run_status, 200)
|
|
self.assertIn("tasks", run_payload)
|
|
|
|
def test_routes_reconciliation_run(self) -> None:
|
|
status, payload = route_request(
|
|
"/trading/reconcile",
|
|
{},
|
|
method="POST",
|
|
body={"instrument_id": 1},
|
|
dependencies={"trading_reconciliation_run": lambda **kwargs: {"status": "ok", "matched_execution_ids": [1]}},
|
|
)
|
|
|
|
self.assertEqual(status, 200)
|
|
self.assertEqual(payload["matched_execution_ids"], [1])
|
|
|
|
def test_routes_runtime_watchdog(self) -> None:
|
|
status, payload = route_request(
|
|
"/trading/runtime-watchdog",
|
|
{"run_once": ["1"]},
|
|
dependencies={"trading_runtime_watchdog": lambda run_once=False: {"stalled_task_count": 0, "run_once": run_once}},
|
|
)
|
|
|
|
self.assertEqual(status, 200)
|
|
self.assertTrue(payload["run_once"])
|
|
|
|
|
|
class TradingRuntimeScriptTests(unittest.TestCase):
|
|
def test_script_outputs_json(self) -> None:
|
|
service = type("Service", (), {"run_once": lambda self: {"scheduler_state": "running", "current_blockers": []}})()
|
|
output = io.StringIO()
|
|
with patch("scripts.run_trading_runtime.get_trading_runtime_service", return_value=service), contextlib.redirect_stdout(output):
|
|
exit_code = run_trading_runtime.main(env={"TRADING_RUNTIME_OUTPUT_JSON": "1"})
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
payload = json.loads(output.getvalue())
|
|
self.assertEqual(payload["scheduler_state"], "running")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|