yuqei
145 lines
4.6 KiB
Python
145 lines
4.6 KiB
Python
from pathlib import Path
|
|
import json
|
|
import os
|
|
import sys
|
|
from http.client import IncompleteRead
|
|
from urllib.error import HTTPError, URLError
|
|
|
|
from sqlalchemy.exc import SQLAlchemyError
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from src.repositories.candle_repository import CandleRepository
|
|
from src.services.market_data.okx_client import OKXClient
|
|
from src.services.market_data.normalizer import normalize_okx_candles
|
|
|
|
|
|
INSTRUMENT_ID_MAP = {
|
|
"BTC-USDT": 1,
|
|
"ETH-USDT": 2,
|
|
}
|
|
|
|
TIMEFRAMES = ["1m", "5m", "15m"]
|
|
|
|
|
|
def build_backfill_config_from_env(env=None) -> dict:
|
|
env = os.environ if env is None else env
|
|
symbol = env.get("OKX_BACKFILL_SYMBOL") or "BTC-USDT"
|
|
inst_id = env.get("OKX_BACKFILL_INST_ID") or f"{symbol}-SWAP"
|
|
timeframes = _get_csv(env, "OKX_BACKFILL_TIMEFRAMES", tuple(TIMEFRAMES))
|
|
return {
|
|
"symbol": symbol,
|
|
"inst_id": inst_id,
|
|
"pages": _get_int(env, "OKX_BACKFILL_PAGES", 20),
|
|
"limit": _get_int(env, "OKX_BACKFILL_LIMIT", 100),
|
|
"timeframes": list(timeframes),
|
|
}
|
|
|
|
|
|
def run_okx_backfill_once(
|
|
client=None,
|
|
repository=None,
|
|
symbol: str = "BTC-USDT",
|
|
inst_id: str = "BTC-USDT-SWAP",
|
|
pages: int = 20,
|
|
limit: int = 100,
|
|
timeframes: list[str] = TIMEFRAMES,
|
|
) -> dict:
|
|
if symbol not in INSTRUMENT_ID_MAP:
|
|
raise ValueError(f"unsupported_okx_backfill_symbol:{symbol}")
|
|
client = client or OKXClient()
|
|
repository = repository or CandleRepository()
|
|
grand_raw = 0
|
|
grand_written = 0
|
|
timeframe_results = []
|
|
|
|
for timeframe in timeframes:
|
|
total_raw = 0
|
|
total_written = 0
|
|
seen_ts = set()
|
|
after = None
|
|
for _ in range(pages):
|
|
raw = client.get_candles(inst_id=inst_id, bar=timeframe, limit=limit, after=after)
|
|
if not raw:
|
|
break
|
|
normalized = normalize_okx_candles(symbol=symbol, venue="OKX", timeframe=timeframe, candles=raw)
|
|
confirmed = []
|
|
for candle in normalized:
|
|
if not candle.confirm or candle.ts_open_ms in seen_ts:
|
|
continue
|
|
seen_ts.add(candle.ts_open_ms)
|
|
confirmed.append(candle)
|
|
written = repository.upsert_many(confirmed, instrument_id=INSTRUMENT_ID_MAP[symbol])
|
|
total_raw += len(raw)
|
|
total_written += written
|
|
next_after = str(min(int(candle.ts) for candle in raw))
|
|
if next_after == after:
|
|
break
|
|
after = next_after
|
|
grand_raw += total_raw
|
|
grand_written += total_written
|
|
timeframe_results.append({"timeframe": timeframe, "fetched": total_raw, "wrote": total_written})
|
|
|
|
return {
|
|
"symbol": symbol,
|
|
"inst_id": inst_id,
|
|
"pages": pages,
|
|
"limit": limit,
|
|
"timeframes": timeframe_results,
|
|
"total_fetched": grand_raw,
|
|
"total_wrote": grand_written,
|
|
}
|
|
|
|
|
|
def main(env=None) -> int:
|
|
env = os.environ if env is None else env
|
|
json_output = _get_bool(env, "OKX_BACKFILL_OUTPUT_JSON", False)
|
|
try:
|
|
summary = run_okx_backfill_once(**build_backfill_config_from_env(env=env))
|
|
except (SQLAlchemyError, URLError, HTTPError, TimeoutError, RuntimeError, OSError, IncompleteRead, ValueError) as exc:
|
|
summary = {
|
|
"passed": False,
|
|
"error_type": exc.__class__.__name__,
|
|
"error": str(exc),
|
|
}
|
|
if json_output:
|
|
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
|
|
else:
|
|
print(f"OKX backfill failed: {exc.__class__.__name__}")
|
|
return 2
|
|
summary["passed"] = True
|
|
if json_output:
|
|
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
|
|
return 0
|
|
for item in summary["timeframes"]:
|
|
print(f"timeframe={item['timeframe']} fetched={item['fetched']} wrote={item['wrote']}")
|
|
print(f"total fetched={summary['total_fetched']} total wrote={summary['total_wrote']}")
|
|
return 0
|
|
|
|
|
|
def _get_csv(env, key: str, default: tuple[str, ...]) -> tuple[str, ...]:
|
|
value = env.get(key)
|
|
if value is None or value.strip() == "":
|
|
return default
|
|
return tuple(item.strip() for item in value.split(",") if item.strip())
|
|
|
|
|
|
def _get_int(env, key: str, default: int) -> int:
|
|
value = env.get(key)
|
|
if value is None or value == "":
|
|
return default
|
|
return int(value)
|
|
|
|
|
|
def _get_bool(env, key: str, default: bool) -> bool:
|
|
value = env.get(key)
|
|
if value is None or value == "":
|
|
return default
|
|
return str(value).strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|