Add model trust scoring
This commit is contained in:
@@ -0,0 +1,36 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from src.services.trust import ModelTrustService
|
||||
|
||||
|
||||
def main(env=None) -> int:
|
||||
env = os.environ if env is None else env
|
||||
instrument_id = int(env.get("TRUST_SCORE_INSTRUMENT_ID", "1"))
|
||||
output_json = _get_bool(env, "TRUST_SCORE_OUTPUT_JSON", False)
|
||||
payload = ModelTrustService().build_summary(instrument_id=instrument_id)
|
||||
if output_json:
|
||||
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||
else:
|
||||
print(f"Trust score models: {len(payload.get('models') or [])}")
|
||||
print(f"Whitelist: {payload.get('whitelist') or []}")
|
||||
return 0
|
||||
|
||||
|
||||
def _get_bool(env, key: str, default: bool = False) -> bool:
|
||||
value = env.get(key)
|
||||
if value is None or value == "":
|
||||
return default
|
||||
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -28,6 +28,7 @@ from src.api.trading_runtime import (
|
||||
get_trading_plan_by_id,
|
||||
get_trading_runtime_status,
|
||||
get_trading_ticket_by_id,
|
||||
get_trading_trust_score,
|
||||
get_weekly_review,
|
||||
rerun_ticket_risk_check,
|
||||
run_candidate_confirm_check,
|
||||
@@ -54,6 +55,7 @@ __all__ = [
|
||||
"get_trading_market_state",
|
||||
"get_trading_plan_by_id",
|
||||
"get_trading_runtime_status",
|
||||
"get_trading_trust_score",
|
||||
"get_trading_ticket_by_id",
|
||||
"get_trading_executions",
|
||||
"get_trading_journal_summary",
|
||||
|
||||
@@ -29,6 +29,7 @@ from src.api.trading_runtime import (
|
||||
get_trading_plan_by_id,
|
||||
get_trading_runtime_status,
|
||||
get_trading_ticket_by_id,
|
||||
get_trading_trust_score,
|
||||
get_trading_executions,
|
||||
get_weekly_review,
|
||||
rerun_ticket_risk_check,
|
||||
@@ -77,6 +78,7 @@ def build_api_dependencies() -> dict:
|
||||
"trading_reconciliation_run": run_trading_reconciliation,
|
||||
"trading_runtime_status": get_trading_runtime_status,
|
||||
"trading_runtime_run_once": run_trading_runtime_once,
|
||||
"trading_trust_score": get_trading_trust_score,
|
||||
"trading_watch_run": run_trading_watch,
|
||||
"trading_candidate_by_id": get_trading_candidate_by_id,
|
||||
"trading_candidate_confirm_check": run_candidate_confirm_check,
|
||||
@@ -191,6 +193,9 @@ def route_request(
|
||||
if method == "POST" and path == "/trading/reconcile":
|
||||
instrument_id = _require_json_int(body or {}, "instrument_id")
|
||||
return 200, deps["trading_reconciliation_run"](instrument_id=instrument_id)
|
||||
if method == "GET" and path == "/trading/trust-score":
|
||||
instrument_id = _require_int(query, "instrument_id")
|
||||
return 200, deps["trading_trust_score"](instrument_id=instrument_id)
|
||||
if method == "POST" and path == "/trading/watch/run":
|
||||
instrument_id = _require_json_int(body or {}, "instrument_id")
|
||||
candidate = deps["trading_watch_run"](
|
||||
|
||||
@@ -16,6 +16,7 @@ from src.services.reconciliation import BrokerExecutionReconciliationService
|
||||
from src.services.runtime import get_trading_runtime_service
|
||||
from src.services.trade_plan_candidates import SetupWatcherService
|
||||
from src.services.trading_plans import ACTIVE_PLAN_STATUS, TradingPlanService
|
||||
from src.services.trust import ModelTrustService
|
||||
|
||||
|
||||
def generate_trading_plan(
|
||||
@@ -420,6 +421,10 @@ def run_trading_reconciliation(instrument_id: int) -> dict:
|
||||
return BrokerExecutionReconciliationService().reconcile_instrument(instrument_id=instrument_id)
|
||||
|
||||
|
||||
def get_trading_trust_score(instrument_id: int) -> dict:
|
||||
return ModelTrustService().build_summary(instrument_id=instrument_id)
|
||||
|
||||
|
||||
def market_truth_degraded(market_state) -> bool:
|
||||
payload = market_state.to_dict() if hasattr(market_state, "to_dict") else dict(market_state or {})
|
||||
freshness_code = str(((payload.get("freshness") or {}).get("code")) or "")
|
||||
|
||||
@@ -8,6 +8,21 @@ from src.db.session import SessionLocal
|
||||
|
||||
|
||||
class BacktestRepository:
|
||||
def fetch_recent_backtest_run_rows(self, instrument_id: int, limit: int = 20) -> list[dict]:
|
||||
sql = text(
|
||||
"""
|
||||
select id, subject_type, model_code, instrument_id, trading_plan_id, candidate_model_code,
|
||||
timeframe_set, started_ts, ended_ts, config, summary
|
||||
from backtest_runs
|
||||
where instrument_id = :instrument_id
|
||||
order by started_ts desc, id desc
|
||||
limit :limit
|
||||
"""
|
||||
)
|
||||
with SessionLocal() as session:
|
||||
rows = session.execute(sql, {"instrument_id": instrument_id, "limit": limit}).mappings().all()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def fetch_active_trading_plan(self, instrument_id: int, trading_plan_id: int = None) -> Optional[dict]:
|
||||
if trading_plan_id is not None:
|
||||
sql = text(
|
||||
|
||||
@@ -246,7 +246,7 @@ class TradingJournalRepository:
|
||||
where_sql = f"where {' and '.join(filters)}" if filters else ""
|
||||
sql = text(
|
||||
f"""
|
||||
select tje.*, ts.instrument_id
|
||||
select tje.*, ts.instrument_id, ts.model_code
|
||||
from trading_journal_entries tje
|
||||
left join trade_signals ts on ts.id = tje.signal_id
|
||||
{where_sql}
|
||||
@@ -276,7 +276,7 @@ class TradingJournalRepository:
|
||||
where_sql = " and ".join(filters)
|
||||
sql = text(
|
||||
f"""
|
||||
select tje.*, ts.instrument_id
|
||||
select tje.*, ts.instrument_id, ts.model_code
|
||||
from trading_journal_entries tje
|
||||
left join trade_signals ts on ts.id = tje.signal_id
|
||||
where {where_sql}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from src.services.trust.model_trust_service import ModelTrustService
|
||||
|
||||
__all__ = ["ModelTrustService"]
|
||||
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import date
|
||||
from typing import Optional
|
||||
|
||||
from src.repositories.backtest_repository import BacktestRepository
|
||||
from src.repositories.trading_journal_repository import TradingJournalRepository, default_week_range
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TrustBucket:
|
||||
model_code: str
|
||||
classification: str
|
||||
score: float
|
||||
sample_size: int
|
||||
real_trade_count: int
|
||||
framework_execution_rate: float
|
||||
out_of_model_trade_rate: float
|
||||
risk_violation_rate: float
|
||||
expectancy_r: float
|
||||
quality_gate_passed: bool
|
||||
recommendation: str
|
||||
session_recommendations: list[dict]
|
||||
reasons: list[str]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return dict(asdict(self))
|
||||
|
||||
|
||||
class ModelTrustService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
backtest_repository: Optional[BacktestRepository] = None,
|
||||
journal_repository: Optional[TradingJournalRepository] = None,
|
||||
) -> None:
|
||||
self.backtest_repository = backtest_repository or BacktestRepository()
|
||||
self.journal_repository = journal_repository or TradingJournalRepository()
|
||||
|
||||
def build_summary(
|
||||
self,
|
||||
*,
|
||||
instrument_id: int,
|
||||
week_start: Optional[date] = None,
|
||||
week_end: Optional[date] = None,
|
||||
) -> dict:
|
||||
if week_start is None or week_end is None:
|
||||
week_start, week_end = default_week_range()
|
||||
backtest_runs = self.backtest_repository.fetch_recent_backtest_run_rows(instrument_id=instrument_id, limit=20)
|
||||
journal_entries = self.journal_repository.fetch_weekly_journal_entries(
|
||||
week_start=week_start,
|
||||
week_end=week_end,
|
||||
instrument_id=instrument_id,
|
||||
)
|
||||
buckets = build_model_buckets(backtest_runs=backtest_runs, journal_entries=journal_entries)
|
||||
ranked = [bucket.to_dict() for bucket in sorted(buckets.values(), key=lambda item: item.score, reverse=True)]
|
||||
whitelist = [bucket["model_code"] for bucket in ranked if bucket["classification"] == "whitelist"]
|
||||
observe_list = [bucket["model_code"] for bucket in ranked if bucket["classification"] == "observe"]
|
||||
blacklist = [bucket["model_code"] for bucket in ranked if bucket["classification"] == "blacklist"]
|
||||
next_actions = recommend_trust_actions(ranked)
|
||||
return {
|
||||
"instrument_id": instrument_id,
|
||||
"week_start": week_start,
|
||||
"week_end": week_end,
|
||||
"models": ranked,
|
||||
"whitelist": whitelist,
|
||||
"observe_list": observe_list,
|
||||
"blacklist": blacklist,
|
||||
"next_actions": next_actions,
|
||||
}
|
||||
|
||||
|
||||
def build_model_buckets(*, backtest_runs: list[dict], journal_entries: list[dict]) -> dict[str, TrustBucket]:
|
||||
backtest_by_model = {}
|
||||
for row in backtest_runs:
|
||||
model_code = str(row.get("candidate_model_code") or row.get("model_code") or "").strip()
|
||||
if not model_code or model_code in backtest_by_model:
|
||||
continue
|
||||
backtest_by_model[model_code] = row
|
||||
|
||||
journal_by_model = defaultdict(list)
|
||||
for entry in journal_entries:
|
||||
model_code = str(entry.get("model_code") or "").strip()
|
||||
if not model_code:
|
||||
continue
|
||||
journal_by_model[model_code].append(dict(entry))
|
||||
|
||||
model_codes = sorted(set(backtest_by_model) | set(journal_by_model))
|
||||
result = {}
|
||||
for model_code in model_codes:
|
||||
backtest_row = backtest_by_model.get(model_code) or {}
|
||||
summary = dict(backtest_row.get("summary") or {})
|
||||
quality_gate = dict(summary.get("quality_gate") or {})
|
||||
session_performance = dict(summary.get("session_performance") or summary.get("by_session") or {})
|
||||
entries = list(journal_by_model.get(model_code) or [])
|
||||
sample_size = len(entries)
|
||||
real_trade_count = sum(1 for entry in entries if entry.get("real_trade_detected"))
|
||||
framework_execution_rate = ratio(sum(1 for entry in entries if entry.get("framework_execution")), sample_size)
|
||||
out_of_model_trade_rate = ratio(sum(1 for entry in entries if entry.get("out_of_model_trade")), sample_size)
|
||||
risk_violation_rate = ratio(sum(1 for entry in entries if entry.get("risk_violation")), sample_size)
|
||||
expectancy_r = float(summary.get("executed_expectancy_r") or summary.get("expectancy_r") or 0.0)
|
||||
score, reasons = calculate_model_trust_score(
|
||||
quality_gate_passed=bool(quality_gate.get("passed")),
|
||||
sample_size=sample_size,
|
||||
framework_execution_rate=framework_execution_rate,
|
||||
out_of_model_trade_rate=out_of_model_trade_rate,
|
||||
risk_violation_rate=risk_violation_rate,
|
||||
expectancy_r=expectancy_r,
|
||||
real_trade_count=real_trade_count,
|
||||
)
|
||||
classification = classify_model(score=score, quality_gate_passed=bool(quality_gate.get("passed")), sample_size=sample_size)
|
||||
result[model_code] = TrustBucket(
|
||||
model_code=model_code,
|
||||
classification=classification,
|
||||
score=score,
|
||||
sample_size=sample_size,
|
||||
real_trade_count=real_trade_count,
|
||||
framework_execution_rate=framework_execution_rate,
|
||||
out_of_model_trade_rate=out_of_model_trade_rate,
|
||||
risk_violation_rate=risk_violation_rate,
|
||||
expectancy_r=expectancy_r,
|
||||
quality_gate_passed=bool(quality_gate.get("passed")),
|
||||
recommendation=recommend_model_action(classification),
|
||||
session_recommendations=build_session_recommendations(session_performance),
|
||||
reasons=reasons,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def calculate_model_trust_score(
|
||||
*,
|
||||
quality_gate_passed: bool,
|
||||
sample_size: int,
|
||||
framework_execution_rate: float,
|
||||
out_of_model_trade_rate: float,
|
||||
risk_violation_rate: float,
|
||||
expectancy_r: float,
|
||||
real_trade_count: int,
|
||||
) -> tuple[float, list[str]]:
|
||||
score = 50.0
|
||||
reasons = []
|
||||
if quality_gate_passed:
|
||||
score += 20.0
|
||||
reasons.append("backtest_quality_gate_passed")
|
||||
else:
|
||||
score -= 15.0
|
||||
reasons.append("backtest_quality_gate_failed")
|
||||
score += min(expectancy_r * 20.0, 15.0)
|
||||
if expectancy_r <= 0:
|
||||
reasons.append("non_positive_expectancy")
|
||||
score += framework_execution_rate * 15.0
|
||||
if framework_execution_rate < 0.5:
|
||||
reasons.append("framework_execution_rate_low")
|
||||
score -= out_of_model_trade_rate * 20.0
|
||||
if out_of_model_trade_rate > 0.25:
|
||||
reasons.append("out_of_model_trade_rate_high")
|
||||
score -= risk_violation_rate * 20.0
|
||||
if risk_violation_rate > 0:
|
||||
reasons.append("risk_violations_present")
|
||||
if sample_size < 5:
|
||||
score -= 10.0
|
||||
reasons.append("sample_size_small")
|
||||
if real_trade_count == 0:
|
||||
reasons.append("no_real_trade_samples")
|
||||
return max(0.0, min(score, 100.0)), reasons
|
||||
|
||||
|
||||
def classify_model(*, score: float, quality_gate_passed: bool, sample_size: int) -> str:
|
||||
if quality_gate_passed and score >= 70.0 and sample_size >= 5:
|
||||
return "whitelist"
|
||||
if score <= 40.0:
|
||||
return "blacklist"
|
||||
return "observe"
|
||||
|
||||
|
||||
def recommend_model_action(classification: str) -> str:
|
||||
if classification == "whitelist":
|
||||
return "trade_inside_framework"
|
||||
if classification == "blacklist":
|
||||
return "avoid_for_now"
|
||||
return "observe_only"
|
||||
|
||||
|
||||
def build_session_recommendations(session_performance: dict) -> list[dict]:
|
||||
results = []
|
||||
for session_code, stats in session_performance.items():
|
||||
expectancy_r = float(stats.get("executed_expectancy_r") or stats.get("expectancy_r") or 0.0)
|
||||
framework_rate = float(stats.get("framework_execution_rate") or 0.0)
|
||||
risk_count = int(stats.get("risk_violation_count") or 0)
|
||||
score = max(0.0, min(100.0, 50.0 + expectancy_r * 20.0 + framework_rate * 20.0 - risk_count * 10.0))
|
||||
results.append(
|
||||
{
|
||||
"session_code": session_code,
|
||||
"score": score,
|
||||
"classification": "prefer" if score >= 65.0 else "avoid" if score <= 35.0 else "observe",
|
||||
"expectancy_r": expectancy_r,
|
||||
"framework_execution_rate": framework_rate,
|
||||
"risk_violation_count": risk_count,
|
||||
}
|
||||
)
|
||||
return sorted(results, key=lambda item: item["score"], reverse=True)
|
||||
|
||||
|
||||
def recommend_trust_actions(models: list[dict]) -> list[str]:
|
||||
actions = []
|
||||
if any(model["classification"] == "blacklist" for model in models):
|
||||
actions.append("reduce_attention_on_blacklist_models")
|
||||
if any(model["classification"] == "observe" for model in models):
|
||||
actions.append("collect_more_real_trade_samples")
|
||||
if any("no_real_trade_samples" in model.get("reasons", []) for model in models):
|
||||
actions.append("validate_model_with_live_assisted_observation")
|
||||
if not actions:
|
||||
actions.append("retain_current_model_whitelist")
|
||||
return actions[:3]
|
||||
|
||||
|
||||
def ratio(numerator: int, denominator: int) -> float:
|
||||
return numerator / denominator if denominator else 0.0
|
||||
+48
-1
@@ -12,6 +12,7 @@ const overviewCards = document.getElementById('overviewCards');
|
||||
const marketStatePanel = document.getElementById('marketStatePanel');
|
||||
const runtimeStatusPanel = document.getElementById('runtimeStatusPanel');
|
||||
const decisionPacketPanel = document.getElementById('decisionPacketPanel');
|
||||
const trustScorePanel = document.getElementById('trustScorePanel');
|
||||
const preMarketPlanPanel = document.getElementById('preMarketPlanPanel');
|
||||
const tradeCandidatesPanel = document.getElementById('tradeCandidatesPanel');
|
||||
const executionTicketsPanel = document.getElementById('executionTicketsPanel');
|
||||
@@ -352,6 +353,47 @@ function renderDecisionPacket(packet, alertRows) {
|
||||
decisionPacketPanel.appendChild(grid);
|
||||
}
|
||||
|
||||
function renderTrustScore(payload) {
|
||||
clearNode(trustScorePanel);
|
||||
const models = payload?.models || [];
|
||||
trustScorePanel.appendChild(
|
||||
buildInfoGrid([
|
||||
['whitelist', formatList(payload?.whitelist || [])],
|
||||
['observe', formatList(payload?.observe_list || [])],
|
||||
['blacklist', formatList(payload?.blacklist || [])],
|
||||
['next actions', formatList(payload?.next_actions || [])],
|
||||
])
|
||||
);
|
||||
if (!models.length) {
|
||||
trustScorePanel.appendChild(createElement('div', 'empty-state', 'No trust-score samples yet.'));
|
||||
return;
|
||||
}
|
||||
const grid = createElement('div', 'cards');
|
||||
models.forEach((model) => {
|
||||
grid.appendChild(
|
||||
buildItemCard(
|
||||
model.model_code || 'model',
|
||||
model.classification || 'observe',
|
||||
[
|
||||
['score', formatNumber(model.score, 1)],
|
||||
['recommendation', model.recommendation || 'observe_only'],
|
||||
['sample size', String(model.sample_size ?? 0)],
|
||||
['real trades', String(model.real_trade_count ?? 0)],
|
||||
['framework rate', formatPercent(model.framework_execution_rate)],
|
||||
['out-of-model rate', formatPercent(model.out_of_model_trade_rate)],
|
||||
['risk violation rate', formatPercent(model.risk_violation_rate)],
|
||||
['expectancy R', formatNumber(model.expectancy_r, 2)],
|
||||
['quality gate', String(Boolean(model.quality_gate_passed))],
|
||||
['reasons', formatList(model.reasons || [])],
|
||||
],
|
||||
[],
|
||||
formatJsonInline(model.session_recommendations || [])
|
||||
)
|
||||
);
|
||||
});
|
||||
trustScorePanel.appendChild(grid);
|
||||
}
|
||||
|
||||
function renderPreMarketPlan(plan) {
|
||||
clearNode(preMarketPlanPanel);
|
||||
preMarketPlanPanel.appendChild(
|
||||
@@ -716,6 +758,7 @@ function showErrorPanel(message) {
|
||||
marketStatePanel,
|
||||
runtimeStatusPanel,
|
||||
decisionPacketPanel,
|
||||
trustScorePanel,
|
||||
preMarketPlanPanel,
|
||||
tradeCandidatesPanel,
|
||||
executionTicketsPanel,
|
||||
@@ -739,6 +782,7 @@ function renderDashboardData({
|
||||
runtimeStatus,
|
||||
decisionPacket,
|
||||
decisionAlerts,
|
||||
trustScore,
|
||||
tradeCandidates,
|
||||
executionTickets,
|
||||
tradingExecutions,
|
||||
@@ -751,6 +795,7 @@ function renderDashboardData({
|
||||
renderMarketState(marketState, tradingExecutions.items || []);
|
||||
renderRuntimeStatus(runtimeStatus);
|
||||
renderDecisionPacket(decisionPacket, decisionAlerts);
|
||||
renderTrustScore(trustScore);
|
||||
renderPreMarketPlan(tradingPlan);
|
||||
renderCandidates(tradeCandidates.items || []);
|
||||
renderTickets(executionTickets.items || []);
|
||||
@@ -767,7 +812,7 @@ async function refreshDashboard() {
|
||||
const limit = limitInput.value || '20';
|
||||
|
||||
try {
|
||||
const [health, candles, tradingPlan, marketState, runtimeStatus, decisionPacket, decisionAlerts, tradeCandidates, executionTickets, tradingExecutions, journalSummary, journalItems, weeklyReview] = await Promise.all([
|
||||
const [health, candles, tradingPlan, marketState, runtimeStatus, decisionPacket, decisionAlerts, trustScore, tradeCandidates, executionTickets, tradingExecutions, journalSummary, journalItems, weeklyReview] = await Promise.all([
|
||||
fetchJson('/health'),
|
||||
fetchJson(`/candles?${queryString({ instrument_id: instrumentId, timeframe, limit })}`),
|
||||
fetchJson(`/trading/plans/latest?${queryString({ instrument_id: instrumentId })}`).catch(() => null),
|
||||
@@ -775,6 +820,7 @@ async function refreshDashboard() {
|
||||
fetchJson('/trading/runtime-status').catch(() => ({ scheduler_state: 'unavailable', tasks: {}, current_blockers: ['runtime_status_unavailable'] })),
|
||||
fetchJson(`/trading/decision-packet?${queryString({ instrument_id: instrumentId })}`).catch(() => ({ status: 'unavailable', action_recommendation: 'monitor' })),
|
||||
fetchJson(`/trading/alerts?${queryString({ instrument_id: instrumentId, limit: 10 })}`).catch(() => ({ items: [] })),
|
||||
fetchJson(`/trading/trust-score?${queryString({ instrument_id: instrumentId })}`).catch(() => ({ models: [], whitelist: [], observe_list: [], blacklist: [], next_actions: [] })),
|
||||
fetchJson(`/trading/candidates?${queryString({ instrument_id: instrumentId, limit })}`).catch(() => ({ items: [] })),
|
||||
fetchJson(`/trading/tickets?${queryString({ instrument_id: instrumentId, limit })}`).catch(() => ({ items: [] })),
|
||||
fetchJson(`/trading/executions?${queryString({ instrument_id: instrumentId, limit })}`).catch(() => ({ items: [] })),
|
||||
@@ -791,6 +837,7 @@ async function refreshDashboard() {
|
||||
runtimeStatus,
|
||||
decisionPacket,
|
||||
decisionAlerts,
|
||||
trustScore,
|
||||
tradeCandidates,
|
||||
executionTickets,
|
||||
tradingExecutions,
|
||||
|
||||
@@ -69,6 +69,11 @@
|
||||
<div id="decisionPacketPanel" class="detail-panel"></div>
|
||||
</section>
|
||||
|
||||
<section class="panel panel-wide">
|
||||
<h2>Model Trust</h2>
|
||||
<div id="trustScorePanel" class="detail-panel"></div>
|
||||
</section>
|
||||
|
||||
<section class="panel panel-wide">
|
||||
<h2>Pre-Market Plan</h2>
|
||||
<div id="preMarketPlanPanel" class="detail-panel"></div>
|
||||
|
||||
@@ -80,6 +80,16 @@ class ApiServerRoutingTests(unittest.TestCase):
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(payload["imported_execution_ids"], [7])
|
||||
|
||||
def test_routes_trading_trust_score(self) -> None:
|
||||
status, payload = route_request(
|
||||
"/trading/trust-score",
|
||||
{"instrument_id": ["1"]},
|
||||
dependencies={"trading_trust_score": lambda **kwargs: {"whitelist": ["OSOK"], "models": [{"model_code": "OSOK"}]}},
|
||||
)
|
||||
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(payload["whitelist"], ["OSOK"])
|
||||
|
||||
def test_routes_trading_broker_state(self) -> None:
|
||||
status, payload = route_request(
|
||||
"/trading/broker-state",
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
from datetime import datetime, timezone
|
||||
import os
|
||||
import unittest
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://user:pass@localhost:5432/ai_ict_test")
|
||||
|
||||
from src.services.trust import ModelTrustService
|
||||
from src.services.trust.model_trust_service import build_model_buckets
|
||||
|
||||
|
||||
class ModelTrustServiceTests(unittest.TestCase):
|
||||
def test_build_summary_classifies_models(self) -> None:
|
||||
service = ModelTrustService(
|
||||
backtest_repository=_BacktestRepo(
|
||||
[
|
||||
{
|
||||
"model_code": "OSOK",
|
||||
"candidate_model_code": "OSOK",
|
||||
"summary": {
|
||||
"quality_gate": {"passed": True},
|
||||
"executed_expectancy_r": 0.6,
|
||||
"session_performance": {
|
||||
"LONDON": {"executed_expectancy_r": 0.8, "framework_execution_rate": 0.9, "risk_violation_count": 0}
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_code": "REAL_TRADE_IMPORT",
|
||||
"candidate_model_code": "REAL_TRADE_IMPORT",
|
||||
"summary": {"quality_gate": {"passed": False}, "executed_expectancy_r": -0.2},
|
||||
},
|
||||
]
|
||||
),
|
||||
journal_repository=_JournalRepo(
|
||||
[
|
||||
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
|
||||
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
|
||||
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
|
||||
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
|
||||
{"model_code": "OSOK", "framework_execution": False, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
|
||||
{"model_code": "REAL_TRADE_IMPORT", "framework_execution": False, "out_of_model_trade": True, "risk_violation": True, "real_trade_detected": True},
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
payload = service.build_summary(instrument_id=1)
|
||||
|
||||
self.assertIn("OSOK", payload["whitelist"])
|
||||
self.assertIn("REAL_TRADE_IMPORT", payload["blacklist"])
|
||||
self.assertTrue(payload["models"][0]["score"] >= payload["models"][1]["score"])
|
||||
|
||||
def test_build_model_buckets_creates_observe_bucket_when_samples_are_small(self) -> None:
|
||||
buckets = build_model_buckets(
|
||||
backtest_runs=[{"model_code": "OSOK", "candidate_model_code": "OSOK", "summary": {"quality_gate": {"passed": True}, "executed_expectancy_r": 0.2}}],
|
||||
journal_entries=[{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": False}],
|
||||
)
|
||||
|
||||
self.assertEqual(buckets["OSOK"].classification, "observe")
|
||||
self.assertIn("sample_size_small", buckets["OSOK"].reasons)
|
||||
|
||||
|
||||
class _BacktestRepo:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def fetch_recent_backtest_run_rows(self, instrument_id: int, limit: int = 20):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
class _JournalRepo:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def fetch_weekly_journal_entries(self, **kwargs):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -27,6 +27,7 @@ from scripts import (
|
||||
run_trading_runtime,
|
||||
run_sweeps,
|
||||
run_swings,
|
||||
run_trust_score,
|
||||
show_models,
|
||||
)
|
||||
|
||||
@@ -148,6 +149,16 @@ class RuntimeScriptJsonTests(unittest.TestCase):
|
||||
self.assertEqual(exit_code, 0)
|
||||
self.assertEqual(payload["instrument_id"], 2)
|
||||
|
||||
def test_run_trust_score_outputs_json(self) -> None:
|
||||
service = type("Service", (), {"build_summary": lambda self, instrument_id: {"instrument_id": instrument_id, "whitelist": ["OSOK"], "models": []}})()
|
||||
output = io.StringIO()
|
||||
with patch("scripts.run_trust_score.ModelTrustService", return_value=service), contextlib.redirect_stdout(output):
|
||||
exit_code = run_trust_score.main(env={"TRUST_SCORE_OUTPUT_JSON": "1", "TRUST_SCORE_INSTRUMENT_ID": "2"})
|
||||
|
||||
payload = json.loads(output.getvalue())
|
||||
self.assertEqual(exit_code, 0)
|
||||
self.assertEqual(payload["instrument_id"], 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user