Codex
292 lines
12 KiB
Python
292 lines
12 KiB
Python
import os
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from sqlalchemy.exc import OperationalError
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://user:pass@localhost:5432/ai_ict_test")
|
|
|
|
from scripts import run_backtest
|
|
from scripts.run_backtest import (
|
|
build_backtest_candidate_model_code_from_env,
|
|
build_backtest_session_codes_from_env,
|
|
build_backtest_timeframe_set_from_env,
|
|
build_backtest_mode_from_env,
|
|
build_backtest_subject_type_from_env,
|
|
build_cost_config_from_env,
|
|
build_quality_gate_from_env,
|
|
build_replay_config_from_env,
|
|
build_require_quality_gate_from_env,
|
|
build_backtest_trading_plan_id_from_env,
|
|
run_backtest_once,
|
|
)
|
|
from src.services.backtest import BacktestOutcome
|
|
|
|
|
|
class RunBacktestScriptTests(unittest.TestCase):
|
|
def test_build_cost_config_from_env_uses_zero_defaults(self) -> None:
|
|
config = build_cost_config_from_env({})
|
|
|
|
self.assertEqual(config.fee_rate, 0.0)
|
|
self.assertEqual(config.slippage_rate, 0.0)
|
|
self.assertEqual(config.fixed_cost_r, 0.0)
|
|
|
|
def test_empty_env_overrides_ambient_backtest_settings(self) -> None:
|
|
with patch.dict(
|
|
os.environ,
|
|
{
|
|
"BACKTEST_FEE_RATE": "0.02",
|
|
"BACKTEST_REQUIRE_QUALITY_GATE": "1",
|
|
"BACKTEST_MIN_TOTAL_SIGNALS": "999",
|
|
"BACKTEST_MIN_EXECUTED_SIGNALS": "888",
|
|
},
|
|
clear=False,
|
|
):
|
|
config = build_cost_config_from_env({})
|
|
require_gate = build_require_quality_gate_from_env({})
|
|
gate = build_quality_gate_from_env({})
|
|
|
|
self.assertEqual(config.fee_rate, 0.0)
|
|
self.assertFalse(require_gate)
|
|
self.assertNotEqual(gate.min_total_signals, 999)
|
|
self.assertNotEqual(gate.min_executed_signals, 888)
|
|
|
|
def test_build_cost_config_from_env_reads_cost_values(self) -> None:
|
|
config = build_cost_config_from_env(
|
|
{
|
|
"BACKTEST_FEE_RATE": "0.0005",
|
|
"BACKTEST_SLIPPAGE_RATE": "0.0007",
|
|
"BACKTEST_FIXED_COST_R": "0.05",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(config.fee_rate, 0.0005)
|
|
self.assertEqual(config.slippage_rate, 0.0007)
|
|
self.assertEqual(config.fixed_cost_r, 0.05)
|
|
|
|
def test_build_require_quality_gate_from_env(self) -> None:
|
|
self.assertFalse(build_require_quality_gate_from_env({}))
|
|
self.assertFalse(build_require_quality_gate_from_env({"BACKTEST_REQUIRE_QUALITY_GATE": "0"}))
|
|
self.assertTrue(build_require_quality_gate_from_env({"BACKTEST_REQUIRE_QUALITY_GATE": "true"}))
|
|
self.assertTrue(build_require_quality_gate_from_env({"BACKTEST_REQUIRE_QUALITY_GATE": "1"}))
|
|
|
|
def test_build_quality_gate_from_env_reads_thresholds(self) -> None:
|
|
gate = build_quality_gate_from_env(
|
|
{
|
|
"BACKTEST_MIN_TOTAL_SIGNALS": "50",
|
|
"BACKTEST_MIN_EXECUTED_SIGNALS": "20",
|
|
"BACKTEST_MIN_EXPECTANCY_R": "0.1",
|
|
"BACKTEST_MIN_PROFIT_FACTOR_R": "1.5",
|
|
"BACKTEST_MAX_DRAWDOWN_R": "6.0",
|
|
"BACKTEST_MAX_CONSECUTIVE_LOSSES": "3",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(gate.min_total_signals, 50)
|
|
self.assertEqual(gate.min_executed_signals, 20)
|
|
self.assertEqual(gate.min_expectancy_r, 0.1)
|
|
self.assertEqual(gate.min_profit_factor_r, 1.5)
|
|
self.assertEqual(gate.max_drawdown_r, 6.0)
|
|
self.assertEqual(gate.max_consecutive_losses, 3)
|
|
|
|
def test_build_backtest_mode_and_replay_config_from_env(self) -> None:
|
|
self.assertEqual(build_backtest_mode_from_env({}), "entry_then_outcome")
|
|
self.assertEqual(build_backtest_mode_from_env({"BACKTEST_MODE": "replay_no_lookahead"}), "replay_no_lookahead")
|
|
self.assertEqual(build_backtest_subject_type_from_env({}), "plan_replay")
|
|
self.assertEqual(build_backtest_subject_type_from_env({"BACKTEST_SUBJECT_TYPE": "signal"}), "signal")
|
|
self.assertEqual(build_backtest_subject_type_from_env({"BACKTEST_SESSION_CODES": "LONDON"}), "signal")
|
|
self.assertIsNone(build_backtest_trading_plan_id_from_env({}))
|
|
self.assertEqual(build_backtest_trading_plan_id_from_env({"BACKTEST_TRADING_PLAN_ID": "77"}), 77)
|
|
self.assertIsNone(build_backtest_candidate_model_code_from_env({}))
|
|
self.assertEqual(build_backtest_candidate_model_code_from_env({"BACKTEST_CANDIDATE_MODEL_CODE": "OSOK"}), "OSOK")
|
|
self.assertEqual(build_backtest_session_codes_from_env({}), [])
|
|
self.assertEqual(build_backtest_session_codes_from_env({"BACKTEST_SESSION_CODES": "LONDON,NY_AM"}), ["LONDON", "NY_AM"])
|
|
|
|
replay_config = build_replay_config_from_env(
|
|
{
|
|
"BACKTEST_REPLAY_LEFT_BARS": "1",
|
|
"BACKTEST_REPLAY_RIGHT_BARS": "3",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(replay_config.left_bars, 1)
|
|
self.assertEqual(replay_config.right_bars, 3)
|
|
|
|
def test_build_backtest_timeframe_set_from_env(self) -> None:
|
|
self.assertEqual(
|
|
build_backtest_timeframe_set_from_env({}),
|
|
{"bias_tf": "1m", "setup_tf": "1m", "trigger_tf": "1m"},
|
|
)
|
|
self.assertEqual(
|
|
build_backtest_timeframe_set_from_env(
|
|
{
|
|
"AI_ICT_BIAS_TIMEFRAME": "1h",
|
|
"AI_ICT_SETUP_TIMEFRAME": "15m",
|
|
"AI_ICT_TRIGGER_TIMEFRAME": "5m",
|
|
}
|
|
),
|
|
{"bias_tf": "1h", "setup_tf": "15m", "trigger_tf": "5m"},
|
|
)
|
|
|
|
def test_run_backtest_once_allows_failed_quality_gate_when_not_required(self) -> None:
|
|
service = FakeBacktestService([])
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(service=service, instrument_id=3, limit=5, require_quality_gate=False)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(service.calls[0]["instrument_id"], 3)
|
|
self.assertEqual(service.calls[0]["limit"], 5)
|
|
self.assertIsNone(service.calls[0]["quality_gate"])
|
|
self.assertIn("Backtest quality gate passed: False", output.getvalue())
|
|
|
|
def test_run_backtest_once_blocks_failed_quality_gate_when_required(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(service=FakeBacktestService([]), require_quality_gate=True)
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertIn("executed_sample_too_small", output.getvalue())
|
|
|
|
def test_run_backtest_once_passes_when_required_quality_gate_is_healthy(self) -> None:
|
|
outcomes = [_build_win_outcome(signal_id=index) for index in range(100)]
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(service=FakeBacktestService(outcomes), require_quality_gate=True)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertIn("Backtest quality gate passed: True", output.getvalue())
|
|
|
|
def test_run_backtest_once_outputs_json_success_summary(self) -> None:
|
|
outcomes = [_build_win_outcome(signal_id=index) for index in range(100)]
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(service=FakeBacktestService(outcomes), require_quality_gate=True, output_json=True)
|
|
|
|
payload = json.loads(output.getvalue())
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertTrue(payload["passed"])
|
|
self.assertTrue(payload["quality_gate_required"])
|
|
self.assertEqual(payload["backtest_run_id"], 99)
|
|
self.assertEqual(payload["result_count"], 100)
|
|
self.assertIsNone(payload["timeframe_set"])
|
|
self.assertTrue(payload["quality_gate"]["passed"])
|
|
self.assertEqual(payload["summary"]["executed_signals"], 100)
|
|
|
|
def test_run_backtest_once_outputs_json_quality_failure(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(service=FakeBacktestService([]), require_quality_gate=True, output_json=True)
|
|
|
|
payload = json.loads(output.getvalue())
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertFalse(payload["passed"])
|
|
self.assertFalse(payload["quality_gate"]["passed"])
|
|
self.assertIn("executed_sample_too_small", payload["quality_gate"]["reasons"])
|
|
|
|
def test_run_backtest_once_passes_replay_mode_and_config(self) -> None:
|
|
service = FakeBacktestService([])
|
|
output = io.StringIO()
|
|
replay_config = build_replay_config_from_env(
|
|
{
|
|
"BACKTEST_REPLAY_LEFT_BARS": "1",
|
|
"BACKTEST_REPLAY_RIGHT_BARS": "1",
|
|
}
|
|
)
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(
|
|
service=service,
|
|
backtest_mode="replay_no_lookahead",
|
|
replay_config=replay_config,
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(service.calls[0]["mode"], "replay_no_lookahead")
|
|
self.assertEqual(service.calls[0]["replay_config"].left_bars, 1)
|
|
self.assertEqual(service.calls[0]["replay_config"].right_bars, 1)
|
|
|
|
def test_run_backtest_once_passes_plan_replay_dimension(self) -> None:
|
|
service = FakeBacktestService([])
|
|
output = io.StringIO()
|
|
|
|
with contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest_once(
|
|
service=service,
|
|
subject_type="plan_replay",
|
|
trading_plan_id=77,
|
|
candidate_model_code="OSOK",
|
|
session_codes=["LONDON"],
|
|
)
|
|
|
|
self.assertEqual(exit_code, 0)
|
|
self.assertEqual(service.calls[0]["subject_type"], "plan_replay")
|
|
self.assertEqual(service.calls[0]["trading_plan_id"], 77)
|
|
self.assertEqual(service.calls[0]["candidate_model_code"], "OSOK")
|
|
self.assertEqual(service.calls[0]["session_codes"], ["LONDON"])
|
|
|
|
def test_main_reports_database_unavailable(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with patch("scripts.run_backtest.BacktestService.run_backtest", side_effect=OperationalError("stmt", {}, Exception())), contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest.main()
|
|
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertIn("Database unavailable for backtest sample: OperationalError", output.getvalue())
|
|
|
|
def test_main_reports_database_unavailable_as_json(self) -> None:
|
|
output = io.StringIO()
|
|
|
|
with patch("scripts.run_backtest.BacktestService.run_backtest", side_effect=OperationalError("stmt", {}, Exception())), contextlib.redirect_stdout(output):
|
|
exit_code = run_backtest.main(env={"BACKTEST_OUTPUT_JSON": "1"})
|
|
|
|
payload = json.loads(output.getvalue())
|
|
self.assertEqual(exit_code, 2)
|
|
self.assertFalse(payload["passed"])
|
|
self.assertEqual(payload["error_type"], "OperationalError")
|
|
self.assertIsNone(payload["quality_gate"])
|
|
|
|
|
|
class FakeBacktestService:
|
|
def __init__(self, outcomes: list[BacktestOutcome]) -> None:
|
|
self.outcomes = outcomes
|
|
self.calls = []
|
|
|
|
def run_backtest(self, **kwargs):
|
|
self.calls.append(kwargs)
|
|
return 99, self.outcomes
|
|
|
|
|
|
def _build_win_outcome(signal_id: int) -> BacktestOutcome:
|
|
return BacktestOutcome(
|
|
signal_id=signal_id,
|
|
outcome="win",
|
|
mfe=2.0,
|
|
mae=0.0,
|
|
tp1_hit=True,
|
|
tp2_hit=True,
|
|
invalidated_before_entry=False,
|
|
error_tags=[],
|
|
meta={
|
|
"signal_side": "buy",
|
|
"session_codes": ["LONDON"],
|
|
"weekday": "Wednesday",
|
|
"mfe_r": 2.0,
|
|
"mae_r": 0.0,
|
|
"cost_r": 0.0,
|
|
"r_multiple": 2.0,
|
|
},
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|