ParallelLLC/algorithmic_trading

1"""Market data loading with a three-tier fallback.2 3Order of preference: live Yahoo download -> on-disk cache -> a deterministic4simulator. The fallback exists because a Hugging Face Space that shows a5stack trace on the first click is a Space nobody shares. When the simulator is6used, :class:`~algotrader.types.MarketData` says so and the UI shows it.7"""8 9from __future__ import annotations10 11import hashlib12import logging13import os14from dataclasses import dataclass15from pathlib import Path16from typing import Optional17 18import numpy as np19import pandas as pd20 21from .types import OHLCV_COLUMNS, MarketData22 23logger = logging.getLogger(__name__)24 25CACHE_DIR = Path(os.environ.get("ALGOTRADER_CACHE", Path.home() / ".cache" / "algotrader"))26NETWORK_ENABLED = os.environ.get("ALGOTRADER_OFFLINE", "").lower() not in ("1", "true", "yes")27 28# Popular tickers get hand-set simulation parameters so the offline demo is at29# least in the right postcode: annual drift, annual vol, and a starting price.30@dataclass(frozen=True)31class SimProfile:32    drift: float33    vol: float34    price: float35 36 37SIM_PROFILES: dict[str, SimProfile] = {38    "AAPL": SimProfile(0.24, 0.29, 190.0),39    "MSFT": SimProfile(0.25, 0.27, 410.0),40    "NVDA": SimProfile(0.55, 0.52, 120.0),41    "TSLA": SimProfile(0.30, 0.58, 250.0),42    "AMZN": SimProfile(0.22, 0.33, 180.0),43    "GOOGL": SimProfile(0.20, 0.31, 170.0),44    "META": SimProfile(0.26, 0.40, 500.0),45    "SPY": SimProfile(0.10, 0.16, 550.0),46    "QQQ": SimProfile(0.14, 0.21, 480.0),47    "BTC-USD": SimProfile(0.45, 0.65, 65000.0),48    "ETH-USD": SimProfile(0.35, 0.75, 3000.0),49    "GLD": SimProfile(0.07, 0.14, 200.0),50    "TLT": SimProfile(0.01, 0.15, 95.0),51}52 53DEFAULT_UNIVERSE = ["SPY", "AAPL", "NVDA", "MSFT", "TSLA", "QQQ", "BTC-USD", "GLD"]54 55 56def _seed_for(symbol: str) -> int:57    """Stable per-symbol seed so a given ticker always simulates identically."""58    digest = hashlib.sha256(symbol.upper().encode()).digest()59    return int.from_bytes(digest[:4], "big")60 61 62def _normalise(df: pd.DataFrame) -> pd.DataFrame:63    """Coerce any loader's output into a clean lowercase OHLCV frame."""64    if isinstance(df.columns, pd.MultiIndex):65        df = df.copy()66        df.columns = [str(c[0]) for c in df.columns]67    df = df.rename(columns={c: str(c).strip().lower().replace(" ", "_") for c in df.columns})68    if "adj_close" in df.columns and "close" not in df.columns:69        df = df.rename(columns={"adj_close": "close"})70    missing = [c for c in OHLCV_COLUMNS if c not in df.columns]71    for col in missing:72        if col == "volume":73            df["volume"] = 0.074        elif "close" in df.columns:75            df[col] = df["close"]76        else:77            raise ValueError(f"Price data is missing required column: {col}")78    df = df.loc[:, list(OHLCV_COLUMNS)].astype(float)79    if not isinstance(df.index, pd.DatetimeIndex):80        df.index = pd.to_datetime(df.index)81    df.index = df.index.tz_localize(None) if df.index.tz is not None else df.index82    df = df[~df.index.duplicated(keep="last")].sort_index()83    df = df[df["close"] > 0].dropna(subset=["close"])84    return df85 86 87def _cache_path(symbol: str, interval: str) -> Path:88    safe = symbol.upper().replace("/", "_")89    return CACHE_DIR / f"{safe}_{interval}.csv"90 91 92def _read_cache(symbol: str, interval: str) -> Optional[pd.DataFrame]:93    path = _cache_path(symbol, interval)94    if not path.exists():95        return None96    try:97        return _normalise(pd.read_csv(path, index_col=0, parse_dates=True))98    except Exception as exc:  # pragma: no cover - corrupted cache is not worth failing over99        logger.warning("Ignoring unreadable cache %s: %s", path, exc)100        return None101 102 103def _write_cache(symbol: str, interval: str, df: pd.DataFrame) -> None:104    try:105        CACHE_DIR.mkdir(parents=True, exist_ok=True)106        df.to_csv(_cache_path(symbol, interval))107    except Exception as exc:  # pragma: no cover - a read-only FS must not break the app108        logger.warning("Could not write cache for %s: %s", symbol, exc)109 110 111def _download(symbol: str, start: str, end: str | None, interval: str) -> Optional[pd.DataFrame]:112    if not NETWORK_ENABLED:113        return None114    try:115        import yfinance as yf116    except ImportError:117        logger.info("yfinance not installed; using offline data")118        return None119    try:120        raw = yf.download(121            symbol,122            start=start,123            end=end,124            interval=interval,125            progress=False,126            auto_adjust=True,127            threads=False,128        )129    except Exception as exc:130        logger.warning("Download failed for %s: %s", symbol, exc)131        return None132    if raw is None or len(raw) == 0:133        logger.warning("Download for %s returned no rows", symbol)134        return None135    try:136        return _normalise(raw)137    except Exception as exc:138        logger.warning("Could not normalise download for %s: %s", symbol, exc)139        return None140 141 142def simulate_ohlcv(143    symbol: str = "SIM",144    start: str = "2015-01-01",145    end: str | None = None,146    interval: str = "1d",147    seed: Optional[int] = None,148) -> pd.DataFrame:149    """Generate a deterministic but realistic-looking OHLCV series.150 151    This is not geometric Brownian motion with a straight face: it uses a152    two-state (calm / stressed) regime switch, Student-t innovations and153    GARCH-ish vol persistence, so the resulting series has fat tails and154    volatility clustering. That matters, because a strategy tested against155    naive GBM looks far better than it deserves to.156    """157    profile = SIM_PROFILES.get(symbol.upper(), SimProfile(0.08, 0.25, 100.0))158    rng = np.random.default_rng(_seed_for(symbol) if seed is None else seed)159 160    freq = {"1d": "B", "1wk": "W-FRI", "1h": "h"}.get(interval, "B")161    index = pd.date_range(start=start, end=end or pd.Timestamp.today().normalize(), freq=freq)162    n = len(index)163    if n < 50:164        raise ValueError("Simulated range is too short to backtest")165 166    ppy = 252 if freq in ("B", "h") else 52167    mu = profile.drift / ppy168    base_vol = profile.vol / np.sqrt(ppy)169 170    # Regime chain: calm state is sticky, stressed state is short and violent.171    p_calm_to_stress, p_stress_to_calm = 0.01, 0.06172    regime = np.zeros(n, dtype=int)173    for i in range(1, n):174        flip = rng.random()175        if regime[i - 1] == 0:176            regime[i] = 1 if flip < p_calm_to_stress else 0177        else:178            regime[i] = 0 if flip < p_stress_to_calm else 1179 180    # Persistent vol around a regime-dependent level.181    vol = np.empty(n)182    level = np.where(regime == 1, base_vol * 2.4, base_vol * 0.9)183    vol[0] = level[0]184    for i in range(1, n):185        vol[i] = 0.92 * vol[i - 1] + 0.08 * level[i]186 187    shocks = rng.standard_t(df=4, size=n) / np.sqrt(2.0)  # unit-ish variance, fat tails188    drift = np.where(regime == 1, mu - 3.0 * base_vol**2, mu)189    log_ret = drift + vol * shocks190    close = profile.price * np.exp(np.cumsum(log_ret))191    close = close * (profile.price / close[-1])  # end near the quoted level192 193    intrabar = vol * rng.uniform(0.3, 1.1, size=n)194    open_ = close * np.exp(-log_ret * rng.uniform(0.2, 0.8, size=n))195    high = np.maximum(open_, close) * np.exp(np.abs(intrabar))196    low = np.minimum(open_, close) * np.exp(-np.abs(intrabar))197    volume = rng.lognormal(mean=15.5, sigma=0.45, size=n) * (1.0 + 3.0 * regime)198 199    return _normalise(200        pd.DataFrame(201            {"open": open_, "high": high, "low": low, "close": close, "volume": volume},202            index=index,203        )204    )205 206 207def load_ohlcv(208    symbol: str = "SPY",209    start: str = "2015-01-01",210    end: str | None = None,211    interval: str = "1d",212    source: str = "yahoo",213) -> MarketData:214    """Load OHLCV for ``symbol``.215 216    Default ``source='yahoo'`` requires a Yahoo download. ``auto`` still falls217    back to cache then the simulator (Hugging Face Space). ``synthetic`` is tests only.218    """219    symbol = (symbol or "SPY").strip().upper()220 221    if source == "synthetic":222        df = simulate_ohlcv(symbol, start, end, interval)223        return MarketData(symbol, df, "synthetic", interval, "Simulated prices (requested).")224 225    if source in ("yahoo", "live", "auto"):226        df = _download(symbol, start, end, interval)227        if df is not None and len(df) > 50:228            _write_cache(symbol, interval, df)229            return MarketData(symbol, df, "yfinance", interval, "Live data from Yahoo Finance.")230        if source in ("yahoo", "live"):231            raise RuntimeError(232                f"Yahoo returned no usable bars for {symbol}. "233                "Check the ticker, date range, and network. "234                "Pass source='synthetic' only for offline tests."235            )236 237    cached = _read_cache(symbol, interval)238    if cached is not None and len(cached) > 50:239        window = cached.loc[str(start) : str(end)] if end else cached.loc[str(start) :]240        if len(window) > 50:241            return MarketData(symbol, window, "bundled", interval, "Cached data (network unavailable).")242 243    df = simulate_ohlcv(symbol, start, end, interval)244    return MarketData(245        symbol,246        df,247        "synthetic",248        interval,249        f"Live data for {symbol} was unavailable, so this run uses a deterministic "250        "market simulator with fat tails and volatility clustering. The statistics "251        "below are still valid — they are just measured on a simulated market.",252    )253