import os import contextlib import io import json import unittest from unittest.mock import patch from sqlalchemy.exc import OperationalError os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://user:pass@localhost:5432/ai_ict_test") from scripts import run_backtest from scripts.run_backtest import ( build_backtest_candidate_model_code_from_env, build_backtest_session_codes_from_env, build_backtest_timeframe_set_from_env, build_backtest_mode_from_env, build_backtest_subject_type_from_env, build_cost_config_from_env, build_quality_gate_from_env, build_replay_config_from_env, build_require_quality_gate_from_env, build_backtest_trading_plan_id_from_env, run_backtest_once, ) from src.services.backtest import BacktestOutcome class RunBacktestScriptTests(unittest.TestCase): def test_build_cost_config_from_env_uses_zero_defaults(self) -> None: config = build_cost_config_from_env({}) self.assertEqual(config.fee_rate, 0.0) self.assertEqual(config.slippage_rate, 0.0) self.assertEqual(config.fixed_cost_r, 0.0) def test_empty_env_overrides_ambient_backtest_settings(self) -> None: with patch.dict( os.environ, { "BACKTEST_FEE_RATE": "0.02", "BACKTEST_REQUIRE_QUALITY_GATE": "1", "BACKTEST_MIN_TOTAL_SIGNALS": "999", "BACKTEST_MIN_EXECUTED_SIGNALS": "888", }, clear=False, ): config = build_cost_config_from_env({}) require_gate = build_require_quality_gate_from_env({}) gate = build_quality_gate_from_env({}) self.assertEqual(config.fee_rate, 0.0) self.assertFalse(require_gate) self.assertNotEqual(gate.min_total_signals, 999) self.assertNotEqual(gate.min_executed_signals, 888) def test_build_cost_config_from_env_reads_cost_values(self) -> None: config = build_cost_config_from_env( { "BACKTEST_FEE_RATE": "0.0005", "BACKTEST_SLIPPAGE_RATE": "0.0007", "BACKTEST_FIXED_COST_R": "0.05", } ) self.assertEqual(config.fee_rate, 0.0005) self.assertEqual(config.slippage_rate, 0.0007) self.assertEqual(config.fixed_cost_r, 0.05) def test_build_require_quality_gate_from_env(self) -> None: self.assertFalse(build_require_quality_gate_from_env({})) self.assertFalse(build_require_quality_gate_from_env({"BACKTEST_REQUIRE_QUALITY_GATE": "0"})) self.assertTrue(build_require_quality_gate_from_env({"BACKTEST_REQUIRE_QUALITY_GATE": "true"})) self.assertTrue(build_require_quality_gate_from_env({"BACKTEST_REQUIRE_QUALITY_GATE": "1"})) def test_build_quality_gate_from_env_reads_thresholds(self) -> None: gate = build_quality_gate_from_env( { "BACKTEST_MIN_TOTAL_SIGNALS": "50", "BACKTEST_MIN_EXECUTED_SIGNALS": "20", "BACKTEST_MIN_EXPECTANCY_R": "0.1", "BACKTEST_MIN_PROFIT_FACTOR_R": "1.5", "BACKTEST_MAX_DRAWDOWN_R": "6.0", "BACKTEST_MAX_CONSECUTIVE_LOSSES": "3", } ) self.assertEqual(gate.min_total_signals, 50) self.assertEqual(gate.min_executed_signals, 20) self.assertEqual(gate.min_expectancy_r, 0.1) self.assertEqual(gate.min_profit_factor_r, 1.5) self.assertEqual(gate.max_drawdown_r, 6.0) self.assertEqual(gate.max_consecutive_losses, 3) def test_build_backtest_mode_and_replay_config_from_env(self) -> None: self.assertEqual(build_backtest_mode_from_env({}), "entry_then_outcome") self.assertEqual(build_backtest_mode_from_env({"BACKTEST_MODE": "replay_no_lookahead"}), "replay_no_lookahead") self.assertEqual(build_backtest_subject_type_from_env({}), "plan_replay") self.assertEqual(build_backtest_subject_type_from_env({"BACKTEST_SUBJECT_TYPE": "signal"}), "signal") self.assertEqual(build_backtest_subject_type_from_env({"BACKTEST_SESSION_CODES": "LONDON"}), "signal") self.assertIsNone(build_backtest_trading_plan_id_from_env({})) self.assertEqual(build_backtest_trading_plan_id_from_env({"BACKTEST_TRADING_PLAN_ID": "77"}), 77) self.assertIsNone(build_backtest_candidate_model_code_from_env({})) self.assertEqual(build_backtest_candidate_model_code_from_env({"BACKTEST_CANDIDATE_MODEL_CODE": "OSOK"}), "OSOK") self.assertEqual(build_backtest_session_codes_from_env({}), []) self.assertEqual(build_backtest_session_codes_from_env({"BACKTEST_SESSION_CODES": "LONDON,NY_AM"}), ["LONDON", "NY_AM"]) replay_config = build_replay_config_from_env( { "BACKTEST_REPLAY_LEFT_BARS": "1", "BACKTEST_REPLAY_RIGHT_BARS": "3", } ) self.assertEqual(replay_config.left_bars, 1) self.assertEqual(replay_config.right_bars, 3) def test_build_backtest_timeframe_set_from_env(self) -> None: self.assertEqual( build_backtest_timeframe_set_from_env({}), {"bias_tf": "1m", "setup_tf": "1m", "trigger_tf": "1m"}, ) self.assertEqual( build_backtest_timeframe_set_from_env( { "AI_ICT_BIAS_TIMEFRAME": "1h", "AI_ICT_SETUP_TIMEFRAME": "15m", "AI_ICT_TRIGGER_TIMEFRAME": "5m", } ), {"bias_tf": "1h", "setup_tf": "15m", "trigger_tf": "5m"}, ) def test_run_backtest_once_allows_failed_quality_gate_when_not_required(self) -> None: service = FakeBacktestService([]) output = io.StringIO() with contextlib.redirect_stdout(output): exit_code = run_backtest_once(service=service, instrument_id=3, limit=5, require_quality_gate=False) self.assertEqual(exit_code, 0) self.assertEqual(service.calls[0]["instrument_id"], 3) self.assertEqual(service.calls[0]["limit"], 5) self.assertIsNone(service.calls[0]["quality_gate"]) self.assertIn("Backtest quality gate passed: False", output.getvalue()) def test_run_backtest_once_blocks_failed_quality_gate_when_required(self) -> None: output = io.StringIO() with contextlib.redirect_stdout(output): exit_code = run_backtest_once(service=FakeBacktestService([]), require_quality_gate=True) self.assertEqual(exit_code, 2) self.assertIn("executed_sample_too_small", output.getvalue()) def test_run_backtest_once_passes_when_required_quality_gate_is_healthy(self) -> None: outcomes = [_build_win_outcome(signal_id=index) for index in range(100)] output = io.StringIO() with contextlib.redirect_stdout(output): exit_code = run_backtest_once(service=FakeBacktestService(outcomes), require_quality_gate=True) self.assertEqual(exit_code, 0) self.assertIn("Backtest quality gate passed: True", output.getvalue()) def test_run_backtest_once_outputs_json_success_summary(self) -> None: outcomes = [_build_win_outcome(signal_id=index) for index in range(100)] output = io.StringIO() with contextlib.redirect_stdout(output): exit_code = run_backtest_once(service=FakeBacktestService(outcomes), require_quality_gate=True, output_json=True) payload = json.loads(output.getvalue()) self.assertEqual(exit_code, 0) self.assertTrue(payload["passed"]) self.assertTrue(payload["quality_gate_required"]) self.assertEqual(payload["backtest_run_id"], 99) self.assertEqual(payload["result_count"], 100) self.assertIsNone(payload["timeframe_set"]) self.assertTrue(payload["quality_gate"]["passed"]) self.assertEqual(payload["summary"]["executed_signals"], 100) def test_run_backtest_once_outputs_json_quality_failure(self) -> None: output = io.StringIO() with contextlib.redirect_stdout(output): exit_code = run_backtest_once(service=FakeBacktestService([]), require_quality_gate=True, output_json=True) payload = json.loads(output.getvalue()) self.assertEqual(exit_code, 2) self.assertFalse(payload["passed"]) self.assertFalse(payload["quality_gate"]["passed"]) self.assertIn("executed_sample_too_small", payload["quality_gate"]["reasons"]) def test_run_backtest_once_passes_replay_mode_and_config(self) -> None: service = FakeBacktestService([]) output = io.StringIO() replay_config = build_replay_config_from_env( { "BACKTEST_REPLAY_LEFT_BARS": "1", "BACKTEST_REPLAY_RIGHT_BARS": "1", } ) with contextlib.redirect_stdout(output): exit_code = run_backtest_once( service=service, backtest_mode="replay_no_lookahead", replay_config=replay_config, ) self.assertEqual(exit_code, 0) self.assertEqual(service.calls[0]["mode"], "replay_no_lookahead") self.assertEqual(service.calls[0]["replay_config"].left_bars, 1) self.assertEqual(service.calls[0]["replay_config"].right_bars, 1) def test_run_backtest_once_passes_plan_replay_dimension(self) -> None: service = FakeBacktestService([]) output = io.StringIO() with contextlib.redirect_stdout(output): exit_code = run_backtest_once( service=service, subject_type="plan_replay", trading_plan_id=77, candidate_model_code="OSOK", session_codes=["LONDON"], ) self.assertEqual(exit_code, 0) self.assertEqual(service.calls[0]["subject_type"], "plan_replay") self.assertEqual(service.calls[0]["trading_plan_id"], 77) self.assertEqual(service.calls[0]["candidate_model_code"], "OSOK") self.assertEqual(service.calls[0]["session_codes"], ["LONDON"]) def test_main_reports_database_unavailable(self) -> None: output = io.StringIO() with patch("scripts.run_backtest.BacktestService.run_backtest", side_effect=OperationalError("stmt", {}, Exception())), contextlib.redirect_stdout(output): exit_code = run_backtest.main() self.assertEqual(exit_code, 2) self.assertIn("Database unavailable for backtest sample: OperationalError", output.getvalue()) def test_main_reports_database_unavailable_as_json(self) -> None: output = io.StringIO() with patch("scripts.run_backtest.BacktestService.run_backtest", side_effect=OperationalError("stmt", {}, Exception())), contextlib.redirect_stdout(output): exit_code = run_backtest.main(env={"BACKTEST_OUTPUT_JSON": "1"}) payload = json.loads(output.getvalue()) self.assertEqual(exit_code, 2) self.assertFalse(payload["passed"]) self.assertEqual(payload["error_type"], "OperationalError") self.assertIsNone(payload["quality_gate"]) class FakeBacktestService: def __init__(self, outcomes: list[BacktestOutcome]) -> None: self.outcomes = outcomes self.calls = [] def run_backtest(self, **kwargs): self.calls.append(kwargs) return 99, self.outcomes def _build_win_outcome(signal_id: int) -> BacktestOutcome: return BacktestOutcome( signal_id=signal_id, outcome="win", mfe=2.0, mae=0.0, tp1_hit=True, tp2_hit=True, invalidated_before_entry=False, error_tags=[], meta={ "signal_side": "buy", "session_codes": ["LONDON"], "weekday": "Wednesday", "mfe_r": 2.0, "mae_r": 0.0, "cost_r": 0.0, "r_multiple": 2.0, }, ) if __name__ == "__main__": unittest.main()