yuqei
411 lines
18 KiB
Python
411 lines
18 KiB
Python
import contextlib
|
|
import io
|
|
import json
|
|
import os
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://user:pass@localhost:5432/ai_ict_test")
|
|
|
|
from scripts import run_autonomous_cycle
|
|
|
|
|
|
class AutonomousCycleTests(unittest.TestCase):
|
|
def test_schema_readiness_reports_missing_columns(self) -> None:
|
|
result = run_autonomous_cycle.evaluate_schema_readiness(
|
|
required_schema={"candles": ("id", "ts_open", "close"), "trade_signals": ("id",)},
|
|
existing_tables={"candles"},
|
|
existing_columns={"candles": {"id", "ts_open"}},
|
|
)
|
|
|
|
self.assertFalse(result["ready"])
|
|
self.assertEqual(result["missing_tables"], ["trade_signals"])
|
|
self.assertEqual(result["missing_columns"], {"candles": ["close"]})
|
|
|
|
def test_build_required_schema_reads_model_columns(self) -> None:
|
|
schema = run_autonomous_cycle.build_required_schema(required_tables=("candles", "trade_signals"))
|
|
|
|
self.assertIn("ts_open", schema["candles"])
|
|
self.assertIn("evidence", schema["trade_signals"])
|
|
|
|
def test_waits_for_database_after_offline_checks_pass(self) -> None:
|
|
calls = []
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": False, "error_type": "OperationalError", "error": "down"},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(len(calls), 1)
|
|
self.assertIn("Autonomous mode: waiting_for_database", output.getvalue())
|
|
self.assertIn("Next actions: wait_for_database_recovery", output.getvalue())
|
|
self.assertIn("Action plan [no-db]: wait_for_database_recovery", output.getvalue())
|
|
self.assertIn("Database available: False", output.getvalue())
|
|
|
|
def test_require_db_returns_failure_when_database_is_unavailable(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={"AUTONOMOUS_REQUIRE_DB": "1"},
|
|
runner=lambda command, **kwargs: SimpleNamespace(returncode=0, stdout='{"passed": true}', stderr=""),
|
|
db_probe=lambda: {"available": False, "error_type": "OperationalError", "error": "down"},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
|
|
def test_runs_pipeline_when_database_is_available(self) -> None:
|
|
calls = []
|
|
pipeline_env = {}
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_pipeline.py":
|
|
pipeline_env.update(kwargs["env"])
|
|
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": True, "missing_tables": []},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(calls[-1][-1], "scripts/run_pipeline.py")
|
|
self.assertEqual(pipeline_env["BACKTEST_REQUIRE_QUALITY_GATE"], "1")
|
|
self.assertEqual(pipeline_env["PIPELINE_OUTPUT_JSON"], "1")
|
|
self.assertIn("Autonomous mode: pipeline_completed", output.getvalue())
|
|
|
|
def test_preserves_explicit_pipeline_quality_gate_override(self) -> None:
|
|
pipeline_env = {}
|
|
|
|
def fake_runner(command, **kwargs):
|
|
if command[-1] == "scripts/run_pipeline.py":
|
|
pipeline_env.update(kwargs["env"])
|
|
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={"BACKTEST_REQUIRE_QUALITY_GATE": "0"},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": True, "missing_tables": [], "missing_columns": {}},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(pipeline_env["BACKTEST_REQUIRE_QUALITY_GATE"], "0")
|
|
|
|
def test_reports_pipeline_failure_when_guarded_pipeline_fails(self) -> None:
|
|
def fake_runner(command, **kwargs):
|
|
if command[-1] == "scripts/run_pipeline.py":
|
|
return SimpleNamespace(returncode=2, stdout="Database unavailable for pipeline sample: OperationalError\n", stderr="")
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": True, "missing_tables": [], "missing_columns": {}},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertIn("Autonomous mode: pipeline_failed", output.getvalue())
|
|
self.assertIn("inspect_pipeline_output_and_quality_gates", output.getvalue())
|
|
|
|
def test_retries_database_probe_until_available(self) -> None:
|
|
probe_results = [
|
|
{"available": False, "error_type": "OperationalError", "error": "down"},
|
|
{"available": True, "value": 1},
|
|
]
|
|
sleeps = []
|
|
|
|
def fake_probe():
|
|
return probe_results.pop(0)
|
|
|
|
calls = []
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_pipeline.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={"AUTONOMOUS_MAX_DB_PROBES": "2", "AUTONOMOUS_PROBE_INTERVAL_SECONDS": "0.5"},
|
|
runner=fake_runner,
|
|
db_probe=fake_probe,
|
|
schema_probe=lambda: {"ready": True, "missing_tables": []},
|
|
sleeper=sleeps.append,
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(sleeps, [0.5])
|
|
self.assertEqual(calls[-1][-1], "scripts/run_pipeline.py")
|
|
self.assertIn("Autonomous mode: pipeline_completed", output.getvalue())
|
|
|
|
def test_stops_when_database_schema_is_not_ready(self) -> None:
|
|
calls = []
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={"AUTONOMOUS_APPLY_MIGRATIONS": "0"},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": False, "missing_tables": ["candles"]},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertEqual(len(calls), 1)
|
|
self.assertIn("Autonomous mode: database_schema_not_ready", output.getvalue())
|
|
self.assertIn("Missing tables: candles", output.getvalue())
|
|
|
|
def test_runs_migrations_and_continues_when_schema_becomes_ready(self) -> None:
|
|
calls = []
|
|
schema_results = [
|
|
{"ready": False, "missing_tables": ["candles"], "missing_columns": {}},
|
|
{"ready": True, "missing_tables": [], "missing_columns": {}},
|
|
]
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true}', stderr="")
|
|
if command[-1] == "scripts/run_pipeline.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: schema_results.pop(0),
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertIn("scripts/run_migrations.py", [command[-1] for command in calls])
|
|
self.assertEqual(calls[-1][-1], "scripts/run_pipeline.py")
|
|
self.assertIn("Autonomous mode: pipeline_completed", output.getvalue())
|
|
|
|
def test_migration_failure_stops_before_pipeline(self) -> None:
|
|
calls = []
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_migrations.py":
|
|
return SimpleNamespace(returncode=3, stdout="", stderr="migration failed")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": False, "missing_tables": ["candles"], "missing_columns": {}},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 3)
|
|
self.assertIn("Autonomous mode: migration_failed", output.getvalue())
|
|
self.assertIn("inspect_migration_output_and_schema", output.getvalue())
|
|
self.assertEqual(calls[-1][-1], "scripts/run_migrations.py")
|
|
|
|
def test_stops_when_database_readiness_fails(self) -> None:
|
|
calls = []
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(returncode=2, stdout='{"passed": false, "reasons": ["not_enough_candles"]}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": True, "missing_tables": [], "missing_columns": {}},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertEqual(calls[-1][-1], "scripts/run_database_readiness.py")
|
|
self.assertIn("Autonomous mode: database_readiness_failed", output.getvalue())
|
|
|
|
def test_runs_backfill_and_continues_when_only_candles_are_missing(self) -> None:
|
|
calls = []
|
|
backfill_env = {}
|
|
readiness_results = [
|
|
SimpleNamespace(
|
|
returncode=2,
|
|
stdout='{"passed": false, "reasons": ["not_enough_candles"], "next_actions": ["run_okx_backfill_or_load_replay_sample"]}',
|
|
stderr="",
|
|
),
|
|
SimpleNamespace(returncode=0, stdout='{"passed": true, "reasons": []}', stderr=""),
|
|
]
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return readiness_results.pop(0)
|
|
if command[-1] == "scripts/run_okx_backfill.py":
|
|
backfill_env.update(kwargs["env"])
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "total_fetched": 100, "total_wrote": 100}', stderr="")
|
|
if command[-1] == "scripts/run_pipeline.py":
|
|
return SimpleNamespace(returncode=0, stdout='{"ok": true}', stderr="")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": True, "missing_tables": [], "missing_columns": {}},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual([command[-1] for command in calls].count("scripts/run_database_readiness.py"), 2)
|
|
self.assertIn("scripts/run_okx_backfill.py", [command[-1] for command in calls])
|
|
self.assertEqual(backfill_env["OKX_BACKFILL_OUTPUT_JSON"], "1")
|
|
self.assertEqual(calls[-1][-1], "scripts/run_pipeline.py")
|
|
self.assertIn("Autonomous mode: pipeline_completed", output.getvalue())
|
|
|
|
def test_backfill_failure_stops_before_pipeline(self) -> None:
|
|
calls = []
|
|
|
|
def fake_runner(command, **kwargs):
|
|
calls.append(command)
|
|
if command[-1] == "scripts/run_database_readiness.py":
|
|
return SimpleNamespace(
|
|
returncode=2,
|
|
stdout='{"passed": false, "reasons": ["not_enough_candles"], "next_actions": ["run_okx_backfill_or_load_replay_sample"]}',
|
|
stderr="",
|
|
)
|
|
if command[-1] == "scripts/run_okx_backfill.py":
|
|
return SimpleNamespace(returncode=7, stdout="", stderr="network failed")
|
|
return SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr="")
|
|
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=fake_runner,
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
schema_probe=lambda: {"ready": True, "missing_tables": [], "missing_columns": {}},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 7)
|
|
self.assertIn("Autonomous mode: backfill_failed", output.getvalue())
|
|
self.assertIn("inspect_backfill_output_and_data_readiness", output.getvalue())
|
|
self.assertNotIn("scripts/run_pipeline.py", [command[-1] for command in calls])
|
|
|
|
def test_backfill_is_not_used_for_blocked_readiness(self) -> None:
|
|
self.assertFalse(run_autonomous_cycle.readiness_needs_candle_backfill({"reasons": ["schema_not_ready", "not_enough_candles"]}))
|
|
self.assertTrue(run_autonomous_cycle.readiness_needs_candle_backfill({"reasons": ["not_enough_candles"]}))
|
|
self.assertTrue(run_autonomous_cycle.readiness_needs_candle_backfill({"reasons": ["candles_too_stale"]}))
|
|
|
|
def test_json_output_is_machine_readable(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={"AUTONOMOUS_OUTPUT_JSON": "1"},
|
|
runner=lambda command, **kwargs: SimpleNamespace(returncode=0, stdout='{"passed": true, "checks": []}', stderr=""),
|
|
db_probe=lambda: {"available": False, "error_type": "OperationalError", "error": "down"},
|
|
)
|
|
|
|
summary = json.loads(output.getvalue())
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(summary["mode"], "waiting_for_database")
|
|
self.assertTrue(summary["passed"])
|
|
self.assertEqual(summary["next_actions"], ["wait_for_database_recovery"])
|
|
self.assertEqual(summary["action_plan"][0]["action"], "wait_for_database_recovery")
|
|
self.assertFalse(summary["action_plan"][0]["requires_database"])
|
|
self.assertEqual(summary["action_plan"][0]["command"][-1], "scripts/run_autonomous_cycle.py")
|
|
self.assertEqual(summary["steps"][0]["name"], "offline_checks")
|
|
|
|
def test_offline_failure_stops_cycle(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_autonomous_cycle.run_autonomous_cycle(
|
|
env={},
|
|
runner=lambda command, **kwargs: SimpleNamespace(returncode=5, stdout="", stderr="failed"),
|
|
db_probe=lambda: {"available": True, "value": 1},
|
|
)
|
|
|
|
self.assertEqual(exit_code, 5)
|
|
self.assertIn("Autonomous mode: offline_checks_failed", output.getvalue())
|
|
|
|
def test_cycle_next_actions_use_readiness_summary_when_available(self) -> None:
|
|
summary = {
|
|
"mode": "database_readiness_failed",
|
|
"steps": [
|
|
{
|
|
"name": "database_readiness",
|
|
"summary": {"next_actions": ["run_okx_backfill_or_load_replay_sample"]},
|
|
}
|
|
],
|
|
}
|
|
|
|
self.assertEqual(
|
|
run_autonomous_cycle.build_cycle_next_actions(summary),
|
|
["run_okx_backfill_or_load_replay_sample"],
|
|
)
|
|
|
|
def test_cycle_action_plan_maps_database_actions(self) -> None:
|
|
plan = run_autonomous_cycle.build_cycle_action_plan(
|
|
["run_migrations_before_pipeline", "run_guarded_pipeline_when_ready"]
|
|
)
|
|
|
|
self.assertEqual(plan[0]["command"][-1], "scripts/run_migrations.py")
|
|
self.assertTrue(plan[0]["requires_database"])
|
|
self.assertEqual(plan[1]["command"][-1], "scripts/run_pipeline.py")
|
|
self.assertTrue(plan[1]["requires_database"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|