Add model trust scoring

This commit is contained in:
Codex
2026-05-08 15:09:35 +08:00
parent e8820e647c
commit 569d9751c7
13 changed files with 441 additions and 3 deletions
+36
View File
@@ -0,0 +1,36 @@
from __future__ import annotations
from pathlib import Path
import json
import os
import sys
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from src.services.trust import ModelTrustService
def main(env=None) -> int:
env = os.environ if env is None else env
instrument_id = int(env.get("TRUST_SCORE_INSTRUMENT_ID", "1"))
output_json = _get_bool(env, "TRUST_SCORE_OUTPUT_JSON", False)
payload = ModelTrustService().build_summary(instrument_id=instrument_id)
if output_json:
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
else:
print(f"Trust score models: {len(payload.get('models') or [])}")
print(f"Whitelist: {payload.get('whitelist') or []}")
return 0
def _get_bool(env, key: str, default: bool = False) -> bool:
value = env.get(key)
if value is None or value == "":
return default
return str(value).strip().lower() in {"1", "true", "yes", "on"}
if __name__ == "__main__":
raise SystemExit(main())
+2
View File
@@ -28,6 +28,7 @@ from src.api.trading_runtime import (
get_trading_plan_by_id,
get_trading_runtime_status,
get_trading_ticket_by_id,
get_trading_trust_score,
get_weekly_review,
rerun_ticket_risk_check,
run_candidate_confirm_check,
@@ -54,6 +55,7 @@ __all__ = [
"get_trading_market_state",
"get_trading_plan_by_id",
"get_trading_runtime_status",
"get_trading_trust_score",
"get_trading_ticket_by_id",
"get_trading_executions",
"get_trading_journal_summary",
+5
View File
@@ -29,6 +29,7 @@ from src.api.trading_runtime import (
get_trading_plan_by_id,
get_trading_runtime_status,
get_trading_ticket_by_id,
get_trading_trust_score,
get_trading_executions,
get_weekly_review,
rerun_ticket_risk_check,
@@ -77,6 +78,7 @@ def build_api_dependencies() -> dict:
"trading_reconciliation_run": run_trading_reconciliation,
"trading_runtime_status": get_trading_runtime_status,
"trading_runtime_run_once": run_trading_runtime_once,
"trading_trust_score": get_trading_trust_score,
"trading_watch_run": run_trading_watch,
"trading_candidate_by_id": get_trading_candidate_by_id,
"trading_candidate_confirm_check": run_candidate_confirm_check,
@@ -191,6 +193,9 @@ def route_request(
if method == "POST" and path == "/trading/reconcile":
instrument_id = _require_json_int(body or {}, "instrument_id")
return 200, deps["trading_reconciliation_run"](instrument_id=instrument_id)
if method == "GET" and path == "/trading/trust-score":
instrument_id = _require_int(query, "instrument_id")
return 200, deps["trading_trust_score"](instrument_id=instrument_id)
if method == "POST" and path == "/trading/watch/run":
instrument_id = _require_json_int(body or {}, "instrument_id")
candidate = deps["trading_watch_run"](
+5
View File
@@ -16,6 +16,7 @@ from src.services.reconciliation import BrokerExecutionReconciliationService
from src.services.runtime import get_trading_runtime_service
from src.services.trade_plan_candidates import SetupWatcherService
from src.services.trading_plans import ACTIVE_PLAN_STATUS, TradingPlanService
from src.services.trust import ModelTrustService
def generate_trading_plan(
@@ -420,6 +421,10 @@ def run_trading_reconciliation(instrument_id: int) -> dict:
return BrokerExecutionReconciliationService().reconcile_instrument(instrument_id=instrument_id)
def get_trading_trust_score(instrument_id: int) -> dict:
return ModelTrustService().build_summary(instrument_id=instrument_id)
def market_truth_degraded(market_state) -> bool:
payload = market_state.to_dict() if hasattr(market_state, "to_dict") else dict(market_state or {})
freshness_code = str(((payload.get("freshness") or {}).get("code")) or "")
+15
View File
@@ -8,6 +8,21 @@ from src.db.session import SessionLocal
class BacktestRepository:
def fetch_recent_backtest_run_rows(self, instrument_id: int, limit: int = 20) -> list[dict]:
sql = text(
"""
select id, subject_type, model_code, instrument_id, trading_plan_id, candidate_model_code,
timeframe_set, started_ts, ended_ts, config, summary
from backtest_runs
where instrument_id = :instrument_id
order by started_ts desc, id desc
limit :limit
"""
)
with SessionLocal() as session:
rows = session.execute(sql, {"instrument_id": instrument_id, "limit": limit}).mappings().all()
return [dict(row) for row in rows]
def fetch_active_trading_plan(self, instrument_id: int, trading_plan_id: int = None) -> Optional[dict]:
if trading_plan_id is not None:
sql = text(
@@ -246,7 +246,7 @@ class TradingJournalRepository:
where_sql = f"where {' and '.join(filters)}" if filters else ""
sql = text(
f"""
select tje.*, ts.instrument_id
select tje.*, ts.instrument_id, ts.model_code
from trading_journal_entries tje
left join trade_signals ts on ts.id = tje.signal_id
{where_sql}
@@ -276,7 +276,7 @@ class TradingJournalRepository:
where_sql = " and ".join(filters)
sql = text(
f"""
select tje.*, ts.instrument_id
select tje.*, ts.instrument_id, ts.model_code
from trading_journal_entries tje
left join trade_signals ts on ts.id = tje.signal_id
where {where_sql}
+3
View File
@@ -0,0 +1,3 @@
from src.services.trust.model_trust_service import ModelTrustService
__all__ = ["ModelTrustService"]
+220
View File
@@ -0,0 +1,220 @@
from __future__ import annotations
from collections import defaultdict
from dataclasses import asdict, dataclass
from datetime import date
from typing import Optional
from src.repositories.backtest_repository import BacktestRepository
from src.repositories.trading_journal_repository import TradingJournalRepository, default_week_range
@dataclass(frozen=True)
class TrustBucket:
model_code: str
classification: str
score: float
sample_size: int
real_trade_count: int
framework_execution_rate: float
out_of_model_trade_rate: float
risk_violation_rate: float
expectancy_r: float
quality_gate_passed: bool
recommendation: str
session_recommendations: list[dict]
reasons: list[str]
def to_dict(self) -> dict:
return dict(asdict(self))
class ModelTrustService:
def __init__(
self,
*,
backtest_repository: Optional[BacktestRepository] = None,
journal_repository: Optional[TradingJournalRepository] = None,
) -> None:
self.backtest_repository = backtest_repository or BacktestRepository()
self.journal_repository = journal_repository or TradingJournalRepository()
def build_summary(
self,
*,
instrument_id: int,
week_start: Optional[date] = None,
week_end: Optional[date] = None,
) -> dict:
if week_start is None or week_end is None:
week_start, week_end = default_week_range()
backtest_runs = self.backtest_repository.fetch_recent_backtest_run_rows(instrument_id=instrument_id, limit=20)
journal_entries = self.journal_repository.fetch_weekly_journal_entries(
week_start=week_start,
week_end=week_end,
instrument_id=instrument_id,
)
buckets = build_model_buckets(backtest_runs=backtest_runs, journal_entries=journal_entries)
ranked = [bucket.to_dict() for bucket in sorted(buckets.values(), key=lambda item: item.score, reverse=True)]
whitelist = [bucket["model_code"] for bucket in ranked if bucket["classification"] == "whitelist"]
observe_list = [bucket["model_code"] for bucket in ranked if bucket["classification"] == "observe"]
blacklist = [bucket["model_code"] for bucket in ranked if bucket["classification"] == "blacklist"]
next_actions = recommend_trust_actions(ranked)
return {
"instrument_id": instrument_id,
"week_start": week_start,
"week_end": week_end,
"models": ranked,
"whitelist": whitelist,
"observe_list": observe_list,
"blacklist": blacklist,
"next_actions": next_actions,
}
def build_model_buckets(*, backtest_runs: list[dict], journal_entries: list[dict]) -> dict[str, TrustBucket]:
backtest_by_model = {}
for row in backtest_runs:
model_code = str(row.get("candidate_model_code") or row.get("model_code") or "").strip()
if not model_code or model_code in backtest_by_model:
continue
backtest_by_model[model_code] = row
journal_by_model = defaultdict(list)
for entry in journal_entries:
model_code = str(entry.get("model_code") or "").strip()
if not model_code:
continue
journal_by_model[model_code].append(dict(entry))
model_codes = sorted(set(backtest_by_model) | set(journal_by_model))
result = {}
for model_code in model_codes:
backtest_row = backtest_by_model.get(model_code) or {}
summary = dict(backtest_row.get("summary") or {})
quality_gate = dict(summary.get("quality_gate") or {})
session_performance = dict(summary.get("session_performance") or summary.get("by_session") or {})
entries = list(journal_by_model.get(model_code) or [])
sample_size = len(entries)
real_trade_count = sum(1 for entry in entries if entry.get("real_trade_detected"))
framework_execution_rate = ratio(sum(1 for entry in entries if entry.get("framework_execution")), sample_size)
out_of_model_trade_rate = ratio(sum(1 for entry in entries if entry.get("out_of_model_trade")), sample_size)
risk_violation_rate = ratio(sum(1 for entry in entries if entry.get("risk_violation")), sample_size)
expectancy_r = float(summary.get("executed_expectancy_r") or summary.get("expectancy_r") or 0.0)
score, reasons = calculate_model_trust_score(
quality_gate_passed=bool(quality_gate.get("passed")),
sample_size=sample_size,
framework_execution_rate=framework_execution_rate,
out_of_model_trade_rate=out_of_model_trade_rate,
risk_violation_rate=risk_violation_rate,
expectancy_r=expectancy_r,
real_trade_count=real_trade_count,
)
classification = classify_model(score=score, quality_gate_passed=bool(quality_gate.get("passed")), sample_size=sample_size)
result[model_code] = TrustBucket(
model_code=model_code,
classification=classification,
score=score,
sample_size=sample_size,
real_trade_count=real_trade_count,
framework_execution_rate=framework_execution_rate,
out_of_model_trade_rate=out_of_model_trade_rate,
risk_violation_rate=risk_violation_rate,
expectancy_r=expectancy_r,
quality_gate_passed=bool(quality_gate.get("passed")),
recommendation=recommend_model_action(classification),
session_recommendations=build_session_recommendations(session_performance),
reasons=reasons,
)
return result
def calculate_model_trust_score(
*,
quality_gate_passed: bool,
sample_size: int,
framework_execution_rate: float,
out_of_model_trade_rate: float,
risk_violation_rate: float,
expectancy_r: float,
real_trade_count: int,
) -> tuple[float, list[str]]:
score = 50.0
reasons = []
if quality_gate_passed:
score += 20.0
reasons.append("backtest_quality_gate_passed")
else:
score -= 15.0
reasons.append("backtest_quality_gate_failed")
score += min(expectancy_r * 20.0, 15.0)
if expectancy_r <= 0:
reasons.append("non_positive_expectancy")
score += framework_execution_rate * 15.0
if framework_execution_rate < 0.5:
reasons.append("framework_execution_rate_low")
score -= out_of_model_trade_rate * 20.0
if out_of_model_trade_rate > 0.25:
reasons.append("out_of_model_trade_rate_high")
score -= risk_violation_rate * 20.0
if risk_violation_rate > 0:
reasons.append("risk_violations_present")
if sample_size < 5:
score -= 10.0
reasons.append("sample_size_small")
if real_trade_count == 0:
reasons.append("no_real_trade_samples")
return max(0.0, min(score, 100.0)), reasons
def classify_model(*, score: float, quality_gate_passed: bool, sample_size: int) -> str:
if quality_gate_passed and score >= 70.0 and sample_size >= 5:
return "whitelist"
if score <= 40.0:
return "blacklist"
return "observe"
def recommend_model_action(classification: str) -> str:
if classification == "whitelist":
return "trade_inside_framework"
if classification == "blacklist":
return "avoid_for_now"
return "observe_only"
def build_session_recommendations(session_performance: dict) -> list[dict]:
results = []
for session_code, stats in session_performance.items():
expectancy_r = float(stats.get("executed_expectancy_r") or stats.get("expectancy_r") or 0.0)
framework_rate = float(stats.get("framework_execution_rate") or 0.0)
risk_count = int(stats.get("risk_violation_count") or 0)
score = max(0.0, min(100.0, 50.0 + expectancy_r * 20.0 + framework_rate * 20.0 - risk_count * 10.0))
results.append(
{
"session_code": session_code,
"score": score,
"classification": "prefer" if score >= 65.0 else "avoid" if score <= 35.0 else "observe",
"expectancy_r": expectancy_r,
"framework_execution_rate": framework_rate,
"risk_violation_count": risk_count,
}
)
return sorted(results, key=lambda item: item["score"], reverse=True)
def recommend_trust_actions(models: list[dict]) -> list[str]:
actions = []
if any(model["classification"] == "blacklist" for model in models):
actions.append("reduce_attention_on_blacklist_models")
if any(model["classification"] == "observe" for model in models):
actions.append("collect_more_real_trade_samples")
if any("no_real_trade_samples" in model.get("reasons", []) for model in models):
actions.append("validate_model_with_live_assisted_observation")
if not actions:
actions.append("retain_current_model_whitelist")
return actions[:3]
def ratio(numerator: int, denominator: int) -> float:
return numerator / denominator if denominator else 0.0
+48 -1
View File
@@ -12,6 +12,7 @@ const overviewCards = document.getElementById('overviewCards');
const marketStatePanel = document.getElementById('marketStatePanel');
const runtimeStatusPanel = document.getElementById('runtimeStatusPanel');
const decisionPacketPanel = document.getElementById('decisionPacketPanel');
const trustScorePanel = document.getElementById('trustScorePanel');
const preMarketPlanPanel = document.getElementById('preMarketPlanPanel');
const tradeCandidatesPanel = document.getElementById('tradeCandidatesPanel');
const executionTicketsPanel = document.getElementById('executionTicketsPanel');
@@ -352,6 +353,47 @@ function renderDecisionPacket(packet, alertRows) {
decisionPacketPanel.appendChild(grid);
}
function renderTrustScore(payload) {
clearNode(trustScorePanel);
const models = payload?.models || [];
trustScorePanel.appendChild(
buildInfoGrid([
['whitelist', formatList(payload?.whitelist || [])],
['observe', formatList(payload?.observe_list || [])],
['blacklist', formatList(payload?.blacklist || [])],
['next actions', formatList(payload?.next_actions || [])],
])
);
if (!models.length) {
trustScorePanel.appendChild(createElement('div', 'empty-state', 'No trust-score samples yet.'));
return;
}
const grid = createElement('div', 'cards');
models.forEach((model) => {
grid.appendChild(
buildItemCard(
model.model_code || 'model',
model.classification || 'observe',
[
['score', formatNumber(model.score, 1)],
['recommendation', model.recommendation || 'observe_only'],
['sample size', String(model.sample_size ?? 0)],
['real trades', String(model.real_trade_count ?? 0)],
['framework rate', formatPercent(model.framework_execution_rate)],
['out-of-model rate', formatPercent(model.out_of_model_trade_rate)],
['risk violation rate', formatPercent(model.risk_violation_rate)],
['expectancy R', formatNumber(model.expectancy_r, 2)],
['quality gate', String(Boolean(model.quality_gate_passed))],
['reasons', formatList(model.reasons || [])],
],
[],
formatJsonInline(model.session_recommendations || [])
)
);
});
trustScorePanel.appendChild(grid);
}
function renderPreMarketPlan(plan) {
clearNode(preMarketPlanPanel);
preMarketPlanPanel.appendChild(
@@ -716,6 +758,7 @@ function showErrorPanel(message) {
marketStatePanel,
runtimeStatusPanel,
decisionPacketPanel,
trustScorePanel,
preMarketPlanPanel,
tradeCandidatesPanel,
executionTicketsPanel,
@@ -739,6 +782,7 @@ function renderDashboardData({
runtimeStatus,
decisionPacket,
decisionAlerts,
trustScore,
tradeCandidates,
executionTickets,
tradingExecutions,
@@ -751,6 +795,7 @@ function renderDashboardData({
renderMarketState(marketState, tradingExecutions.items || []);
renderRuntimeStatus(runtimeStatus);
renderDecisionPacket(decisionPacket, decisionAlerts);
renderTrustScore(trustScore);
renderPreMarketPlan(tradingPlan);
renderCandidates(tradeCandidates.items || []);
renderTickets(executionTickets.items || []);
@@ -767,7 +812,7 @@ async function refreshDashboard() {
const limit = limitInput.value || '20';
try {
const [health, candles, tradingPlan, marketState, runtimeStatus, decisionPacket, decisionAlerts, tradeCandidates, executionTickets, tradingExecutions, journalSummary, journalItems, weeklyReview] = await Promise.all([
const [health, candles, tradingPlan, marketState, runtimeStatus, decisionPacket, decisionAlerts, trustScore, tradeCandidates, executionTickets, tradingExecutions, journalSummary, journalItems, weeklyReview] = await Promise.all([
fetchJson('/health'),
fetchJson(`/candles?${queryString({ instrument_id: instrumentId, timeframe, limit })}`),
fetchJson(`/trading/plans/latest?${queryString({ instrument_id: instrumentId })}`).catch(() => null),
@@ -775,6 +820,7 @@ async function refreshDashboard() {
fetchJson('/trading/runtime-status').catch(() => ({ scheduler_state: 'unavailable', tasks: {}, current_blockers: ['runtime_status_unavailable'] })),
fetchJson(`/trading/decision-packet?${queryString({ instrument_id: instrumentId })}`).catch(() => ({ status: 'unavailable', action_recommendation: 'monitor' })),
fetchJson(`/trading/alerts?${queryString({ instrument_id: instrumentId, limit: 10 })}`).catch(() => ({ items: [] })),
fetchJson(`/trading/trust-score?${queryString({ instrument_id: instrumentId })}`).catch(() => ({ models: [], whitelist: [], observe_list: [], blacklist: [], next_actions: [] })),
fetchJson(`/trading/candidates?${queryString({ instrument_id: instrumentId, limit })}`).catch(() => ({ items: [] })),
fetchJson(`/trading/tickets?${queryString({ instrument_id: instrumentId, limit })}`).catch(() => ({ items: [] })),
fetchJson(`/trading/executions?${queryString({ instrument_id: instrumentId, limit })}`).catch(() => ({ items: [] })),
@@ -791,6 +837,7 @@ async function refreshDashboard() {
runtimeStatus,
decisionPacket,
decisionAlerts,
trustScore,
tradeCandidates,
executionTickets,
tradingExecutions,
+5
View File
@@ -69,6 +69,11 @@
<div id="decisionPacketPanel" class="detail-panel"></div>
</section>
<section class="panel panel-wide">
<h2>Model Trust</h2>
<div id="trustScorePanel" class="detail-panel"></div>
</section>
<section class="panel panel-wide">
<h2>Pre-Market Plan</h2>
<div id="preMarketPlanPanel" class="detail-panel"></div>
+10
View File
@@ -80,6 +80,16 @@ class ApiServerRoutingTests(unittest.TestCase):
self.assertEqual(status, 200)
self.assertEqual(payload["imported_execution_ids"], [7])
def test_routes_trading_trust_score(self) -> None:
status, payload = route_request(
"/trading/trust-score",
{"instrument_id": ["1"]},
dependencies={"trading_trust_score": lambda **kwargs: {"whitelist": ["OSOK"], "models": [{"model_code": "OSOK"}]}},
)
self.assertEqual(status, 200)
self.assertEqual(payload["whitelist"], ["OSOK"])
def test_routes_trading_broker_state(self) -> None:
status, payload = route_request(
"/trading/broker-state",
+79
View File
@@ -0,0 +1,79 @@
from datetime import datetime, timezone
import os
import unittest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://user:pass@localhost:5432/ai_ict_test")
from src.services.trust import ModelTrustService
from src.services.trust.model_trust_service import build_model_buckets
class ModelTrustServiceTests(unittest.TestCase):
def test_build_summary_classifies_models(self) -> None:
service = ModelTrustService(
backtest_repository=_BacktestRepo(
[
{
"model_code": "OSOK",
"candidate_model_code": "OSOK",
"summary": {
"quality_gate": {"passed": True},
"executed_expectancy_r": 0.6,
"session_performance": {
"LONDON": {"executed_expectancy_r": 0.8, "framework_execution_rate": 0.9, "risk_violation_count": 0}
},
},
},
{
"model_code": "REAL_TRADE_IMPORT",
"candidate_model_code": "REAL_TRADE_IMPORT",
"summary": {"quality_gate": {"passed": False}, "executed_expectancy_r": -0.2},
},
]
),
journal_repository=_JournalRepo(
[
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
{"model_code": "OSOK", "framework_execution": False, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": True},
{"model_code": "REAL_TRADE_IMPORT", "framework_execution": False, "out_of_model_trade": True, "risk_violation": True, "real_trade_detected": True},
]
),
)
payload = service.build_summary(instrument_id=1)
self.assertIn("OSOK", payload["whitelist"])
self.assertIn("REAL_TRADE_IMPORT", payload["blacklist"])
self.assertTrue(payload["models"][0]["score"] >= payload["models"][1]["score"])
def test_build_model_buckets_creates_observe_bucket_when_samples_are_small(self) -> None:
buckets = build_model_buckets(
backtest_runs=[{"model_code": "OSOK", "candidate_model_code": "OSOK", "summary": {"quality_gate": {"passed": True}, "executed_expectancy_r": 0.2}}],
journal_entries=[{"model_code": "OSOK", "framework_execution": True, "out_of_model_trade": False, "risk_violation": False, "real_trade_detected": False}],
)
self.assertEqual(buckets["OSOK"].classification, "observe")
self.assertIn("sample_size_small", buckets["OSOK"].reasons)
class _BacktestRepo:
def __init__(self, rows):
self.rows = rows
def fetch_recent_backtest_run_rows(self, instrument_id: int, limit: int = 20):
return list(self.rows)
class _JournalRepo:
def __init__(self, rows):
self.rows = rows
def fetch_weekly_journal_entries(self, **kwargs):
return list(self.rows)
if __name__ == "__main__":
unittest.main()
+11
View File
@@ -27,6 +27,7 @@ from scripts import (
run_trading_runtime,
run_sweeps,
run_swings,
run_trust_score,
show_models,
)
@@ -148,6 +149,16 @@ class RuntimeScriptJsonTests(unittest.TestCase):
self.assertEqual(exit_code, 0)
self.assertEqual(payload["instrument_id"], 2)
def test_run_trust_score_outputs_json(self) -> None:
service = type("Service", (), {"build_summary": lambda self, instrument_id: {"instrument_id": instrument_id, "whitelist": ["OSOK"], "models": []}})()
output = io.StringIO()
with patch("scripts.run_trust_score.ModelTrustService", return_value=service), contextlib.redirect_stdout(output):
exit_code = run_trust_score.main(env={"TRUST_SCORE_OUTPUT_JSON": "1", "TRUST_SCORE_INSTRUMENT_ID": "2"})
payload = json.loads(output.getvalue())
self.assertEqual(exit_code, 0)
self.assertEqual(payload["instrument_id"], 2)
if __name__ == "__main__":
unittest.main()