Deepvest/ProfilingAI
0
1import numpy as np2import pandas as pd3from typing import List, Dict, Tuple, Optional, Any4from dataclasses import dataclass5from datetime import datetime, timedelta6import statsmodels.api as sm7from scipy import stats8from sklearn.model_selection import TimeSeriesSplit9from sklearn.mixture import GaussianMixture10import matplotlib.pyplot as plt11from concurrent.futures import ProcessPoolExecutor12import warnings13warnings.filterwarnings('ignore')14 15# Imports from other modules16from src.core import (17 PortfolioOptimizerImpl, 18 OptimizationConstraints, 19 MarketAnalyzerImpl20)21from src.analysis.ml_analyzer import MLEnhancedAnalyzer22from src.core.types import MarketAnalyzerProtocol23from ..strategies.alternative_assets import AlternativeAssetsPortfolio24from ..core.risk_metrics import EnhancedRiskMetrics25from ..core.profiling import PersonalProfile26from ..core.event_types import MarketUpdate, PortfolioAlert27from ..core.types import MarketAnalyzerProtocol as MarketAnalyzer28 29@dataclass30class BacktestResult:31 """Store backtest results"""32 returns: pd.Series33 positions: pd.DataFrame34 trades: pd.DataFrame35 metrics: Dict[str, float]36 risk_metrics: Dict[str, float]37 transaction_costs: pd.Series38 turnover: pd.Series39 drawdowns: pd.Series40 attribution: Dict[str, pd.Series]41 stress_test_results: Dict[str, float]42 43 def to_dict(self) -> Dict:44 """Convert results to dictionary format"""45 return {46 'returns': self.returns.to_dict(),47 'positions': self.positions.to_dict(),48 'trades': self.trades.to_dict(),49 'metrics': self.metrics,50 'risk_metrics': self.risk_metrics,51 'transaction_costs': self.transaction_costs.to_dict(),52 'turnover': self.turnover.to_dict(),53 'drawdowns': self.drawdowns.to_dict(),54 'attribution': {k: v.to_dict() for k, v in self.attribution.items()},55 'stress_test_results': self.stress_test_results56 }57 58@dataclass59class BacktestConfig:60 """Configuration for backtest"""61 start_date: datetime62 end_date: datetime63 initial_capital: float64 rebalance_frequency: str = "monthly"65 transaction_cost_model: str = "percentage"66 transaction_cost_params: Dict[str, float] = None67 include_alternative_assets: bool = True68 risk_free_rate: float = 0.0269 benchmark_index: str = "SPY"70 stress_periods: Dict[str, Tuple[datetime, datetime]] = None71 72 def __post_init__(self):73 """Validate configuration after initialization"""74 if self.transaction_cost_params is None:75 self.transaction_cost_params = {'cost_rate': 0.001}76 77 if self.stress_periods is None:78 self.stress_periods = {79 'covid_crash': (80 datetime(2020, 2, 19),81 datetime(2020, 3, 23)82 ),83 'financial_crisis': (84 datetime(2008, 9, 1),85 datetime(2009, 3, 1)86 )87 }88 89class BacktestEngine:90 """Main backtesting engine with enhanced risk metrics and performance analysis"""91 92 def __init__(self, portfolio_optimizer, market_analyzer, backtest_engine, **kwargs):93 self.backtest_engine = backtest_engine94 95 self.risk_metrics = EnhancedRiskMetrics(returns=None, prices=None)96 if not isinstance(portfolio_optimizer, PortfolioOptimizerImpl):97 raise TypeError("portfolio_optimizer must be instance of PortfolioOptimizerImpl")98 if market_analyzer is None:99 ml_analyzer = MLEnhancedAnalyzer()100 self.market_analyzer = MarketAnalyzerImpl(ml_analyzer=ml_analyzer)101 else:102 self.market_analyzer = market_analyzer103 self.alternative_assets_portfolio = alternative_assets_portfolio104 self.risk_free_rate = risk_free_rate105 106 # État interne du backtest107 self.current_positions = None108 self.transaction_costs = None109 self.current_portfolio_value = None110 self.rebalance_dates = []111 112 # Cache pour les calculs113 self._cache = {}114 115 def _initialize_market_data(self, data: pd.DataFrame):116 """Initialize market data properly"""117 try:118 # Vérifier les données119 if data is None or data.empty:120 raise ValueError("No market data provided")121 122 # Normaliser l'index temporel123 data.index = pd.to_datetime(data.index).tz_localize(None)124 125 # Créer une copie pour éviter les modifications indésirables126 self.market_data = data.copy()127 128 # Calculer les rendements129 self.returns = self.market_data.pct_change().fillna(0)130 131 # S'assurer que les données sont chronologiquement ordonnées132 self.market_data = self.market_data.sort_index()133 self.returns = self.returns.sort_index()134 135 print("Market data initialized successfully")136 return True137 138 except Exception as e:139 print(f"Error initializing market data: {e}")140 return False141 142 143 async def run_backtest(self,144 config: BacktestConfig,145 data: pd.DataFrame,146 constraints: OptimizationConstraints,147 alternative_data: Optional[pd.DataFrame] = None) -> BacktestResult:148 149 # Normaliser les timezones150 data.index = pd.to_datetime(data.index).tz_localize(None)151 config.start_date = pd.to_datetime(config.start_date).tz_localize(None)152 config.end_date = pd.to_datetime(config.end_date).tz_localize(None)153 """Execute backtest with given configuration"""154 try:155 # Validation and initialization of market data156 self._validate_data(data, config)157 self._initialize_market_data(data)158 market_conditions = await self.market_analyzer.analyze_market_conditions(data)159 160 # Initialize backtest state161 self._initialize_backtest(config, data)162 163 # Execute simulation - Passer tous les arguments nécessaires164 results = self._run_simulation(165 config=config,166 constraints=constraints,167 alternative_data=alternative_data,168 data=self.market_data # Ajout de data ici169 )170 171 # Calculate metrics172 metrics = self._calculate_performance_metrics(results)173 risk_metrics = self._calculate_risk_metrics(results)174 attribution = self._perform_attribution_analysis(results)175 176 # Run stress tests177 stress_results = self._run_stress_tests(results, config.stress_periods)178 179 return BacktestResult(180 returns=results['returns'],181 positions=results['positions'],182 trades=results['trades'],183 metrics=metrics,184 risk_metrics=risk_metrics,185 transaction_costs=results['transaction_costs'],186 turnover=results['turnover'],187 drawdowns=self._calculate_drawdowns(results['returns']),188 attribution=attribution,189 stress_test_results=stress_results190 )191 192 except Exception as e:193 print(f"Error during backtest execution: {e}")194 raise195 196 async def _run_simulation(self, config: BacktestConfig, data: pd.DataFrame, constraints: OptimizationConstraints, alternative_data: Optional[pd.DataFrame] = None) -> Dict[str, pd.DataFrame]:197 """Run the main simulation loop"""198 try:199 results = {200 'positions': pd.DataFrame(index=data.index, columns=data.columns, dtype=float),201 'trades': pd.DataFrame(index=data.index, columns=data.columns, dtype=float),202 'returns': pd.Series(index=data.index, dtype=float),203 'transaction_costs': pd.Series(index=data.index, dtype=float),204 'turnover': pd.Series(index=data.index, dtype=float),205 'portfolio_values': pd.Series(index=data.index, dtype=float)206 }207 208 portfolio_value = config.initial_capital209 previous_weights = np.zeros(len(data.columns))210 211 for date in data.index:212 if self._should_rebalance(date, config.rebalance_frequency):213 # Get current market data slice214 current_data = data.loc[:date]215 216 try:217 # Analyze market conditions - MODIFICATION ICI218 market_conditions = await self.market_analyzer.analyze_market_conditions(current_data)219 except Exception as e:220 print(f"Market analysis error: {str(e)}")221 market_conditions = None222 223 # Optimize portfolio224 try:225 new_weights = self.portfolio_optimizer.optimize_portfolio(constraints=constraints)226 except Exception as e:227 print(f"Optimization error: {str(e)}")228 new_weights = previous_weights229 230 # Calculate trades and costs231 trades = self._calculate_trades(previous_weights, new_weights, portfolio_value)232 costs = self._calculate_transaction_costs(trades, data.loc[date], config.transaction_cost_model, config.transaction_cost_params)233 234 portfolio_value *= (1 - costs)235 previous_weights = new_weights236 237 results['trades'].loc[date] = trades238 results['transaction_costs'].loc[date] = costs239 results['turnover'].loc[date] = np.sum(np.abs(trades)) / (2 * portfolio_value)240 241 # Calculate daily returns242 daily_return = self._calculate_daily_returns(date, data, previous_weights)243 portfolio_value *= (1 + daily_return)244 245 results['returns'].loc[date] = daily_return246 results['positions'].loc[date] = previous_weights247 results['portfolio_values'].loc[date] = portfolio_value248 249 return results250 251 except Exception as e:252 print(f"Error in simulation: {str(e)}")253 raise254 255 def _validate_data(self, data: pd.DataFrame, config: BacktestConfig) -> None:256 """Validate input data for backtesting"""257 try:258 # Vérifier si les données sont vides259 if data is None or data.empty:260 raise ValueError("Data cannot be empty")261 262 # Vérifier et corriger l'ordre de l'index si nécessaire263 if not data.index.is_monotonic_increasing:264 print("Warning: Data index not monotonic, sorting data...")265 data.sort_index(inplace=True)266 267 # Convertir les dates en datetime268 data.index = pd.to_datetime(data.index)269 config_start = pd.to_datetime(config.start_date)270 config_end = pd.to_datetime(config.end_date)271 272 # Vérifier la couverture temporelle273 if data.index[0] > config_start or data.index[-1] < config_end:274 raise ValueError(275 f"Data range {data.index[0]} to {data.index[-1]} "276 f"does not cover backtest period {config_start} to {config_end}"277 )278 279 # Vérifier les valeurs manquantes280 if data.isnull().any().any():281 missing_cols = data.columns[data.isnull().any()].tolist()282 raise ValueError(f"Missing values found in columns: {missing_cols}")283 284 except Exception as e:285 print(f"Error validating data: {str(e)}")286 raise287 288 def _initialize_backtest(self, config: BacktestConfig, data: pd.DataFrame) -> None:289 """Initialize backtest state"""290 self.current_positions = pd.Series(0, index=data.columns)291 self.transaction_costs = pd.Series(0, index=data.index)292 self.current_portfolio_value = config.initial_capital293 self._cache = {} # Reset cache294 295 # Generate rebalancing dates296 self.rebalance_dates = self._generate_rebalance_dates(297 data.index, config.rebalance_frequency298 )299 300 301 def _clean_results(self, results: Dict[str, pd.DataFrame]) -> Dict[str, pd.DataFrame]:302 """Nettoie les résultats de la simulation"""303 try:304 # Remplacer les inf et -inf par NaN305 for key, value in results.items():306 if isinstance(value, pd.DataFrame) or isinstance(value, pd.Series):307 results[key] = value.replace([np.inf, -np.inf], np.nan)308 309 # Forward fill pour les positions310 results['positions'] = results['positions'].fillna(method='ffill')311 312 # Remplacer les NaN restants par 0313 for key, value in results.items():314 if isinstance(value, pd.DataFrame) or isinstance(value, pd.Series):315 results[key] = value.fillna(0)316 317 return results318 319 except Exception as e:320 print(f"Error cleaning results: {e}")321 return results322 323 def _get_sector_mapping(self, columns: pd.Index) -> Dict[str, List[str]]:324 """Get sector mapping for assets"""325 try:326 return {'default': columns.tolist()}327 except Exception as e:328 print(f"Error in sector mapping: {str(e)}")329 return {}330 331 def _calculate_value_exposure(self, positions: pd.DataFrame) -> pd.Series:332 """Calculate value factor exposure"""333 try:334 return positions.rolling(window=252).mean()335 except Exception as e:336 print(f"Error calculating value exposure: {str(e)}")337 return pd.Series(index=positions.index)338 339 def _calculate_growth_exposure(self, positions: pd.DataFrame) -> pd.Series:340 """Calculate growth factor exposure"""341 try:342 return positions.pct_change().rolling(window=252).mean()343 except Exception as e:344 print(f"Error calculating growth exposure: {str(e)}")345 return pd.Series(index=positions.index)346 347 def _calculate_momentum_exposure(self, positions: pd.DataFrame) -> pd.Series:348 """Calculate momentum factor exposure"""349 try:350 returns = positions.pct_change()351 return returns.rolling(window=252).mean()352 except Exception as e:353 print(f"Error calculating momentum exposure: {str(e)}")354 return pd.Series(index=positions.index)355 356 def _calculate_quality_exposure(self, positions: pd.DataFrame) -> pd.Series:357 """Calculate quality factor exposure"""358 try:359 returns = positions.pct_change()360 return 1 / returns.rolling(window=63).std()361 except Exception as e:362 print(f"Error calculating quality exposure: {str(e)}")363 return pd.Series(index=positions.index)364 365 def _calculate_size_exposure(self, positions: pd.DataFrame) -> pd.Series:366 """Calculate size factor exposure"""367 try:368 return positions.abs().sum(axis=1)369 except Exception as e:370 print(f"Error calculating size exposure: {str(e)}")371 return pd.Series(index=positions.index)372 373 def _calculate_daily_returns(self, date: datetime, market_returns: pd.DataFrame, positions: np.ndarray) -> float:374 """Calculate daily returns properly handling numpy arrays"""375 try:376 # Convert numpy array to pandas Series with proper index377 positions_series = pd.Series(positions, index=market_returns.columns)378 379 # Get returns for the specific date380 if isinstance(market_returns, pd.DataFrame):381 daily_returns = market_returns.loc[date]382 else:383 return 0.0384 385 # Calculate weighted returns386 return (positions_series * daily_returns).sum()387 388 except Exception as e:389 print(f"Error calculating daily returns for {date}: {e}")390 return 0.0391 392 def _decompose_performance(self, returns: pd.Series, positions: pd.DataFrame) -> Dict[str, pd.Series]:393 """Decompose portfolio performance"""394 try:395 decomposition = {396 'market_effect': returns * positions.mean(),397 'selection_effect': returns * (positions - positions.mean()),398 'interaction_effect': returns * positions.diff()399 }400 return decomposition401 except Exception as e:402 print(f"Error decomposing performance: {str(e)}")403 return {} 404 405 def _calculate_trades(self,406 new_weights: np.ndarray,407 portfolio_value: float,408 prices: pd.Series) -> np.ndarray:409 """Calculate required trades to achieve new weights with advanced handling"""410 current_values = self.current_positions * portfolio_value411 target_values = new_weights * portfolio_value412 413 # Calculer les trades en nombre d'unités414 trades = (target_values - current_values) / prices415 416 # Arrondir au nombre entier d'unités le plus proche417 trades = np.round(trades, decimals=0)418 419 # Recalculer les poids après arrondissement420 actual_values = trades * prices421 actual_weights = actual_values / portfolio_value422 423 # Stocker les poids réels pour utilisation future424 self._cache['actual_weights'] = actual_weights425 426 return trades427 428 def _calculate_performance_metrics(self, results: Dict[str, pd.DataFrame]) -> Dict[str, float]:429 """Calculate comprehensive performance metrics"""430 try:431 # Extraire et nettoyer les returns432 returns = results['returns'].replace([np.inf, -np.inf], np.nan).dropna()433 434 if len(returns) == 0:435 raise ValueError("No valid returns data")436 437 # Calculer les métriques de base438 total_return = (1 + returns).prod() - 1439 440 metrics = {441 'total_return': total_return,442 'cagr': self._calculate_cagr(returns),443 'volatility': returns.std() * np.sqrt(252),444 'sharpe_ratio': self._calculate_sharpe_ratio(returns),445 'sortino_ratio': self._calculate_sortino_ratio(returns),446 'max_drawdown': self._calculate_max_drawdown(returns),447 'win_rate': len(returns[returns > 0]) / len(returns),448 }449 450 # Ajouter des métriques avancées seulement si elles peuvent être calculées451 try:452 metrics.update({453 'calmar_ratio': abs(metrics['cagr'] / metrics['max_drawdown']) if metrics['max_drawdown'] != 0 else np.inf,454 'profit_factor': abs(returns[returns > 0].sum() / returns[returns < 0].sum()) if returns[returns < 0].sum() != 0 else np.inf,455 'avg_win': returns[returns > 0].mean() if len(returns[returns > 0]) > 0 else 0,456 'avg_loss': returns[returns < 0].mean() if len(returns[returns < 0]) > 0 else 0457 })458 except Exception as e:459 print(f"Warning: Could not calculate some advanced metrics: {e}")460 461 # Nettoyer les résultats finaux462 metrics = {k: v for k, v in metrics.items() if not np.isnan(v) and not np.isinf(v)}463 464 return metrics465 466 except Exception as e:467 print(f"Error calculating performance metrics: {e}")468 return {}469 470 471 def _calculate_cagr(self, returns: pd.Series) -> float:472 """Calculate Compound Annual Growth Rate"""473 if returns is None or len(returns) == 0:474 return 0.0475 476 total_return = (1 + returns).prod()477 n_years = len(returns) / 252 # 252 jours de trading par an478 479 if n_years > 0 and total_return > 0:480 return (total_return ** (1/n_years)) - 1481 else:482 return 0.0483 484 def _calculate_sortino_ratio(self, returns: pd.Series) -> float:485 """Calculate Sortino ratio"""486 try:487 excess_returns = returns - self.risk_free_rate/252488 negative_returns = excess_returns[excess_returns < 0]489 downside_std = negative_returns.std() * np.sqrt(252)490 491 if downside_std == 0:492 return 0.0493 494 return excess_returns.mean() * 252 / downside_std495 except Exception as e:496 print(f"Error calculating Sortino ratio: {e}")497 return 0.0498 499 500 def _calculate_var(self, returns: pd.Series, confidence: float = 0.95) -> float:501 """Calculate Value at Risk"""502 try:503 return np.percentile(returns, (1 - confidence) * 100)504 except Exception as e:505 print(f"Error calculating VaR: {e}")506 return 0.0507 508 def _get_benchmark_returns(self, results: Dict[str, pd.DataFrame]) -> pd.Series:509 """Get benchmark returns series"""510 returns = results['returns'] # returns est une Series, pas un DataFrame511 # Retourner une Series vide si pas de benchmark512 return pd.Series(dtype=float) 513 514 def _get_factor_returns(self) -> pd.DataFrame:515 """Récupère les facteurs de risque pour l'attribution de performance"""516 try:517 # Période d'analyse518 dates = self.returns.index519 520 # Facteurs de base521 factors = pd.DataFrame(index=dates)522 523 # Market Factor (Excess Return)524 factors['MKT'] = self.returns.mean(axis=1) - self.risk_free_rate/252525 526 # Size Factor (SMB)527 if hasattr(self, 'market_data') and 'market_cap' in self.market_data:528 factors['SMB'] = self._calculate_size_factor()529 else:530 factors['SMB'] = pd.Series(0, index=dates)531 532 # Value Factor (HML)533 if hasattr(self, 'market_data') and 'book_to_market' in self.market_data:534 factors['HML'] = self._calculate_value_factor()535 else:536 factors['HML'] = pd.Series(0, index=dates)537 538 # Momentum Factor (MOM)539 factors['MOM'] = self._calculate_momentum_factor()540 541 # Volatility Factor (VOL)542 factors['VOL'] = self._calculate_volatility_factor()543 544 return factors545 546 except Exception as e:547 print(f"Erreur dans le calcul des facteurs : {e}")548 # Retourner un DataFrame minimal en cas d'erreur549 return pd.DataFrame({'MKT': self.returns.mean(axis=1)})550 551 def _calculate_risk_metrics(self, results: Dict[str, pd.DataFrame]) -> Dict[str, float]:552 """Calculate comprehensive risk metrics"""553 try:554 returns = results['returns'].replace([np.inf, -np.inf], np.nan).dropna()555 556 if len(returns) == 0:557 raise ValueError("No valid returns data")558 559 risk_metrics = {}560 561 # 1. Métriques Value at Risk562 try:563 risk_metrics.update({564 'var_historical_95': self.risk_metrics.calculate_historical_var(returns, 0.95),565 'var_gaussian_95': self.risk_metrics.calculate_gaussian_var(returns, 0.95),566 'cvar_95': self.risk_metrics.calculate_cvar(returns, 0.95)567 })568 except Exception as e:569 print(f"Error calculating VaR metrics: {e}")570 risk_metrics.update({571 'var_historical_95': 0.0,572 'var_gaussian_95': 0.0,573 'cvar_95': 0.0574 })575 576 # 2. Métriques de Volatilité577 try:578 risk_metrics.update({579 'volatility_annual': returns.std() * np.sqrt(252),580 'downside_deviation': self.risk_metrics.calculate_downside_deviation(returns),581 'upside_volatility': self.risk_metrics.calculate_upside_volatility(returns),582 'volatility_skew': self.risk_metrics.calculate_volatility_skew(returns)583 })584 except Exception as e:585 print(f"Error calculating volatility metrics: {e}")586 risk_metrics.update({587 'volatility_annual': 0.0,588 'downside_deviation': 0.0,589 'upside_volatility': 0.0,590 'volatility_skew': 0.0591 })592 593 # 3. Métriques de Distribution594 try:595 risk_metrics.update({596 'skewness': returns.skew(),597 'kurtosis': returns.kurtosis(),598 'tail_ratio': self.risk_metrics.calculate_tail_ratio(returns),599 'jar_bera_stat': self.risk_metrics.calculate_jarque_bera(returns)600 })601 except Exception as e:602 print(f"Error calculating distribution metrics: {e}")603 risk_metrics.update({604 'skewness': 0.0,605 'kurtosis': 0.0,606 'tail_ratio': 1.0,607 'jar_bera_stat': 0.0608 })609 610 # 4. Métriques de Marché611 try:612 risk_metrics.update({613 'beta': self.risk_metrics.calculate_beta(returns),614 'alpha': self.risk_metrics.calculate_alpha(returns),615 'tracking_error': self.risk_metrics.calculate_tracking_error(returns),616 'r_squared': self.risk_metrics.calculate_r_squared(returns)617 })618 except Exception as e:619 print(f"Error calculating market metrics: {e}")620 risk_metrics.update({621 'beta': 1.0,622 'alpha': 0.0,623 'tracking_error': 0.0,624 'r_squared': 0.0625 })626 627 # 5. Métriques Avancées628 try:629 risk_metrics.update({630 'ulcer_index': self.risk_metrics.calculate_ulcer_index(returns),631 'pain_index': self.risk_metrics.calculate_pain_index(returns),632 'pain_ratio': self.risk_metrics.calculate_pain_ratio(returns),633 'burke_ratio': self.risk_metrics.calculate_burke_ratio(returns)634 })635 except Exception as e:636 print(f"Error calculating advanced metrics: {e}")637 risk_metrics.update({638 'ulcer_index': 0.0,639 'pain_index': 0.0,640 'pain_ratio': 0.0,641 'burke_ratio': 0.0642 })643 644 # 6. Analyse des Régimes645 try:646 regime_metrics = self._calculate_regime_metrics(returns)647 risk_metrics.update(regime_metrics)648 except Exception as e:649 print(f"Error calculating regime metrics: {e}")650 risk_metrics.update({651 'regime_0': {'frequency': 0.0, 'avg_return': 0.0},652 'regime_1': {'frequency': 0.0, 'avg_return': 0.0},653 'regime_2': {'frequency': 0.0, 'avg_return': 0.0}654 })655 656 # Nettoyer les résultats finaux657 risk_metrics = {658 k: v for k, v in risk_metrics.items() 659 if not (isinstance(v, float) and (np.isnan(v) or np.isinf(v)))660 }661 662 return risk_metrics663 664 except Exception as e:665 print(f"Error calculating risk metrics: {e}")666 return {}667 668 def _calculate_rolling_metrics(self, returns: pd.Series, window: int = 252) -> Dict[str, pd.Series]:669 """Calculate rolling performance metrics"""670 try:671 rolling_metrics = {672 'rolling_sharpe': self.risk_metrics.calculate_rolling_sharpe(window),673 'rolling_sortino': self.risk_metrics.calculate_rolling_sortino(window),674 'rolling_volatility': self.risk_metrics.calculate_rolling_volatility(window),675 'rolling_beta': self.risk_metrics.calculate_rolling_beta(window),676 'rolling_var': self.risk_metrics.calculate_rolling_var(window)677 }678 return rolling_metrics679 except Exception as e:680 print(f"Error calculating rolling metrics: {str(e)}")681 raise682 683 def _calculate_regime_metrics(self, returns: pd.Series) -> Dict[str, Dict[str, float]]:684 """Calculate metrics under different market regimes"""685 try:686 # Identifier les régimes de marché687 regimes = self._identify_market_regimes(returns)688 regime_metrics = {}689 690 # Calculer les métriques pour chaque régime691 for regime in np.unique(regimes):692 regime_returns = returns[regimes == regime]693 regime_metrics[f'regime_{regime}'] = {694 'frequency': len(regime_returns) / len(returns),695 'avg_return': regime_returns.mean(),696 'volatility': regime_returns.std() * np.sqrt(252),697 'sharpe': self.risk_metrics.calculate_regime_sharpe(regime_returns),698 'max_drawdown': self.risk_metrics.calculate_regime_drawdown(regime_returns),699 'win_rate': len(regime_returns[regime_returns > 0]) / len(regime_returns)700 }701 702 return regime_metrics703 704 except Exception as e:705 print(f"Error calculating regime metrics: {str(e)}")706 raise707 708 709 def _calculate_recovery_time(self, returns: pd.Series) -> int:710 """Calcule le temps de récupération après un drawdown"""711 if returns is None or len(returns) == 0:712 return 0713 714 cumulative = (1 + returns).cumprod()715 running_max = cumulative.expanding().max()716 drawdowns = cumulative / running_max - 1717 718 recovery_periods = 0719 in_drawdown = False720 721 for i, dd in enumerate(drawdowns):722 if not in_drawdown and dd < 0:723 in_drawdown = True724 start_idx = i725 elif in_drawdown and dd >= 0:726 in_drawdown = False727 recovery_periods = i - start_idx728 729 return recovery_periods730 731 def _identify_market_regimes(self, returns: pd.Series, n_regimes: int = 3) -> np.ndarray:732 try:733 # Préparer les features pour le GMM734 features = self._prepare_regime_features(returns)735 features = np.nan_to_num(features).reshape(-1, 1) # Reshape en 2D736 737 gmm = GaussianMixture(n_components=n_regimes, random_state=42)738 regimes = gmm.fit_predict(features)739 740 return regimes741 except Exception as e:742 print(f"Error identifying market regimes: {e}")743 return np.zeros(len(returns))744 745 def _prepare_regime_features(self, returns: pd.Series) -> np.ndarray:746 """Prepare features for regime detection"""747 try:748 window = 21 # fenêtre de 1 mois749 750 # Calculer les features751 mean = returns.rolling(window=window).mean().fillna(0)752 std = returns.rolling(window=window).std().fillna(0)753 skew = returns.rolling(window=window).skew().fillna(0)754 kurt = returns.rolling(window=window).kurt().fillna(0)755 756 # Combiner les features en matrice 2D757 features = np.column_stack([758 mean.values,759 std.values,760 skew.values,761 kurt.values762 ])763 764 return features765 766 except Exception as e:767 print(f"Error preparing regime features: {e}")768 return np.zeros((len(returns), 4))769 770 def _perform_attribution_analysis(self, results: Dict[str, pd.DataFrame]) -> Dict[str, pd.Series]:771 """Perform detailed performance attribution analysis"""772 try:773 returns = results['returns']774 positions = results['positions']775 776 attribution = {777 # Attribution par facteurs778 'factor_attribution': self._calculate_factor_attribution(returns, positions),779 780 # Attribution par secteur781 'sector_attribution': self._calculate_sector_attribution(returns, positions),782 783 # Attribution par style784 'style_attribution': self._calculate_style_attribution(returns, positions),785 786 # Attribution du risque787 'risk_attribution': self._calculate_risk_attribution(returns, positions),788 789 # Décomposition de la performance790 'performance_decomposition': self._decompose_performance(returns, positions)791 }792 793 return attribution794 795 except Exception as e:796 print(f"Error performing attribution analysis: {str(e)}")797 raise798 799 def _calculate_factor_attribution(self, returns: pd.Series, positions: pd.DataFrame) -> pd.Series:800 """801 Calcule l'attribution de performance basée sur les facteurs802 803 Args:804 returns: Rendements du portefeuille805 positions: Positions du portefeuille806 """807 try:808 # Charger les rendements des facteurs809 factors = pd.DataFrame(index=returns.index)810 811 # Market Factor812 factors['MKT'] = returns.fillna(0).mean() - self.risk_free_rate/252813 814 # Size Factor (SMB)815 factors['SMB'] = positions.fillna(0).rolling(window=21).mean().std()816 817 # Value Factor (HML)818 factors['HML'] = pd.Series(0, index=returns.index) # placeholder819 820 # Momentum Factor (MOM)821 factors['MOM'] = returns.fillna(0).rolling(window=252).mean()822 823 # Volatility Factor (VOL)824 factors['VOL'] = returns.fillna(0).rolling(window=63).std()825 826 # Nettoyer les valeurs aberrantes827 factors = factors.replace([np.inf, -np.inf], np.nan)828 factors = factors.fillna(0)829 returns = returns.fillna(0)830 831 # Régresser les rendements sur les facteurs832 try:833 model = sm.OLS(returns, factors).fit()834 factor_contribution = pd.Series(835 model.params * factors.mean(),836 index=factors.columns837 )838 except:839 factor_contribution = pd.Series(0, index=factors.columns)840 841 return factor_contribution842 843 except Exception as e:844 print(f"Error calculating factor attribution: {str(e)}")845 return pd.Series()846 847 def _calculate_sector_attribution(self, returns: pd.Series, positions: pd.DataFrame) -> pd.Series:848 """Calculate sector-based performance attribution"""849 try:850 # Charger les mappings sectoriels851 sector_mapping = self._get_sector_mapping(positions.columns)852 sector_returns = {}853 854 # Calculer les contributions par secteur855 for sector, assets in sector_mapping.items():856 sector_positions = positions[assets].sum(axis=1)857 sector_ret = (returns * sector_positions).sum()858 sector_returns[sector] = {859 'total_return': sector_ret,860 'average_weight': sector_positions.mean(),861 'contribution': sector_ret * sector_positions.mean()862 }863 864 return pd.DataFrame(sector_returns).T865 866 except Exception as e:867 print(f"Error calculating sector attribution: {str(e)}")868 raise869 870 def _calculate_style_attribution(self, returns: pd.Series, positions: pd.DataFrame) -> pd.Series:871 """Calculate investment style attribution (value, growth, momentum, etc.)"""872 try:873 # Définir les facteurs de style874 style_factors = {875 'value': self._calculate_value_exposure(positions),876 'growth': self._calculate_growth_exposure(positions),877 'momentum': self._calculate_momentum_exposure(positions),878 'quality': self._calculate_quality_exposure(positions),879 'size': self._calculate_size_exposure(positions)880 }881 882 # Calculer l'attribution pour chaque style883 style_attribution = {}884 for style, exposure in style_factors.items():885 style_return = (returns * exposure).sum()886 style_attribution[style] = style_return887 888 return pd.Series(style_attribution)889 890 except Exception as e:891 print(f"Error calculating style attribution: {str(e)}")892 raise893 894 def _calculate_risk_attribution(self, returns: pd.Series, positions: pd.DataFrame) -> pd.Series:895 """Calculate risk-based performance attribution"""896 try:897 # Calculer la contribution au risque de chaque position898 vol = returns.std() * np.sqrt(252)899 pos_vol = positions.std() * np.sqrt(252)900 risk_contribution = pos_vol * positions.corr() * vol901 902 return pd.Series(risk_contribution.sum(), index=positions.columns)903 904 except Exception as e:905 print(f"Error calculating risk attribution: {str(e)}")906 return pd.Series()907 908 def _calculate_transaction_costs(self,909 trades: np.ndarray,910 prices: pd.Series,911 cost_model: str,912 cost_params: Dict[str, float]) -> float:913 """Calculate transaction costs based on specified model"""914 if cost_model == "percentage":915 return np.sum(np.abs(trades * prices)) * cost_params['cost_rate']916 elif cost_model == "fixed":917 return np.sum(np.abs(trades > 0)) * cost_params['fixed_cost']918 else:919 raise ValueError(f"Unknown transaction cost model: {cost_model}")920 921 922 def _calculate_sharpe_ratio(self, returns: pd.Series) -> float:923 """Calculate Sharpe ratio"""924 try:925 excess_returns = returns - self.risk_free_rate/252 # Conversion en taux journalier926 if excess_returns.std() == 0:927 return 0.0928 return np.sqrt(252) * excess_returns.mean() / excess_returns.std()929 except Exception as e:930 print(f"Error calculating Sharpe ratio: {e}")931 return 0.0932 933 def _calculate_liquidity_score(self, results: BacktestResult, profile: PersonalProfile) -> Dict[str, float]:934 """935 Calcule le score de liquidité du portefeuille par rapport au profil investisseur936 937 Args:938 results: Résultats du backtest939 profile: Profil de l'investisseur940 941 Returns:942 Dict contenant les métriques de liquidité943 """944 try:945 # Calculer la liquidité moyenne quotidienne946 daily_volume = results.positions * self.current_portfolio_value947 948 # Calculer le ratio de liquidité (combien de jours pour liquider)949 liquidation_days = daily_volume / (results.positions * 0.1) # Assume 10% max volume quotidien950 951 # Calculer le coût de liquidité estimé952 liquidity_cost = daily_volume * 0.0025 # Estimation du spread bid-ask953 954 # Évaluer l'adéquation avec les besoins de liquidité du profil955 required_liquidity = profile.annual_income / 12 # Besoins mensuels956 957 liquidity_score = {958 'average_days_to_liquidate': liquidation_days.mean().mean(),959 'liquidity_cost_percent': (liquidity_cost.sum() / self.current_portfolio_value),960 'liquidity_coverage_ratio': daily_volume.mean().sum() / required_liquidity,961 'emergency_buffer_ratio': results.positions['cash'].iloc[-1] if 'cash' in results.positions else 0,962 'overall_liquidity_score': self._calculate_overall_liquidity_score(963 liquidation_days.mean().mean(),964 required_liquidity,965 profile.risk_tolerance966 )967 }968 969 return liquidity_score970 971 except Exception as e:972 print(f"Erreur dans le calcul du score de liquidité : {e}")973 return {974 'average_days_to_liquidate': 0,975 'liquidity_cost_percent': 0,976 'liquidity_coverage_ratio': 0,977 'emergency_buffer_ratio': 0,978 'overall_liquidity_score': 0979 }980 981 def _calculate_overall_liquidity_score(self, 982 days_to_liquidate: float,983 required_liquidity: float,984 risk_tolerance: int) -> float:985 """ 986 Calcule le score global de liquidité987 988 Args:989 days_to_liquidate: Nombre moyen de jours pour liquider990 required_liquidity: Liquidité requise991 risk_tolerance: Tolérance au risque du profil (1-5)992 993 Returns:994 Score entre 0 et 1995 """996 # Ajuster les seuils selon la tolérance au risque997 max_acceptable_days = {998 1: 2, # Très conservateur999 2: 5, # Conservateur1000 3: 10, # Modéré1001 4: 15, # Dynamique1002 5: 20 # Agressif1003 }[risk_tolerance]1004 1005 # Calculer le score normalisé1006 days_score = max(0, 1 - (days_to_liquidate / max_acceptable_days))1007 1008 # Ajouter d'autres facteurs si nécessaire1009 # ...1010 1011 return days_score1012 1013 def _run_stress_tests(self, results: Dict[str, pd.DataFrame], stress_periods: Dict[str, Tuple[datetime, datetime]]) -> Dict[str, float]:1014 """Run stress tests with proper timezone handling"""1015 try:1016 returns = results['returns']1017 1018 # Normaliser l'index temporel1019 if returns.index.tz is not None:1020 returns.index = returns.index.tz_localize(None)1021 1022 stress_results = {}1023 1024 for period_name, (start, end) in stress_periods.items():1025 try:1026 # Convertir et normaliser les dates de la période1027 start = pd.to_datetime(start).tz_localize(None)1028 end = pd.to_datetime(end).tz_localize(None)1029 1030 if start > returns.index[-1] or end < returns.index[0]:1031 print(f"No data for stress period {period_name}")1032 continue1033 1034 start = max(start, returns.index[0])1035 end = min(end, returns.index[-1])1036 1037 period_returns = returns[start:end]1038 1039 if len(period_returns) > 0:1040 stress_results[period_name] = {1041 'total_return': (1 + period_returns).prod() - 1,1042 'max_drawdown': self._calculate_max_drawdown(period_returns),1043 'volatility': period_returns.std() * np.sqrt(252),1044 'worst_day': period_returns.min(),1045 'recovery_time': self._calculate_recovery_time(period_returns)1046 }1047 1048 except Exception as e:1049 print(f"Error in stress period {period_name}: {e}")1050 1051 return stress_results1052 1053 except Exception as e:1054 print(f"Error in stress tests: {e}")1055 return {}1056 1057 def _run_historical_stress_tests(self, returns: pd.Series, stress_periods: Dict[str, Tuple[datetime, datetime]]) -> Dict[str, float]:1058 """Run historical stress tests"""1059 historical_results = {}1060 1061 # Vérifier d'abord si nous avons des données1062 if returns is None or len(returns) == 0:1063 print("Warning: No return data available for stress tests")1064 return historical_results1065 1066 # Obtenir la plage de dates disponible1067 data_start = returns.index.min()1068 data_end = returns.index.max()1069 1070 for period_name, (start, end) in stress_periods.items():1071 try:1072 # Convertir les dates1073 start = pd.to_datetime(start)1074 end = pd.to_datetime(end)1075 1076 # Vérifier si la période est dans notre plage de données1077 if start > data_end or end < data_start:1078 print(f"Warning: Stress period {period_name} ({start} to {end}) outside available data range ({data_start} to {data_end})")1079 continue1080 1081 # Ajuster la période si nécessaire1082 adjusted_start = max(start, data_start)1083 adjusted_end = min(end, data_end)1084 1085 # Gérer les fuseaux horaires de manière plus robuste1086 if returns.index.tz is not None:1087 adjusted_start = adjusted_start.tz_localize(returns.index.tz)1088 adjusted_end = adjusted_end.tz_localize(returns.index.tz)1089 1090 period_returns = returns.loc[adjusted_start:adjusted_end]1091 1092 # Vérifier si nous avons assez de données1093 if len(period_returns) < 5: # minimum arbitraire de 5 points de données1094 print(f"Warning: Insufficient data for stress period {period_name}")1095 continue1096 1097 historical_results[period_name] = {1098 'total_return': (1 + period_returns).prod() - 1,1099 'max_drawdown': self.risk_metrics.calculate_max_drawdown(period_returns),1100 'volatility': period_returns.std() * np.sqrt(252),1101 'var_95': np.percentile(period_returns, 5),1102 'worst_day': period_returns.min(),1103 'recovery_days': self._calculate_recovery_time(period_returns)1104 }1105 1106 # Vérifier la validité des résultats1107 for key, value in historical_results[period_name].items():1108 if np.isnan(value) or np.isinf(value):1109 historical_results[period_name][key] = 01110 print(f"Warning: Invalid {key} value for period {period_name}")1111 1112 except Exception as e:1113 print(f"Error in stress period {period_name}: {str(e)}")1114 historical_results[period_name] = {1115 'total_return': 0,1116 'max_drawdown': 0,1117 'volatility': 0,1118 'var_95': 0,1119 'worst_day': 0,1120 'recovery_days': 01121 }1122 1123 if not historical_results:1124 print("Warning: No valid stress test results generated")1125 1126 return historical_results1127 1128 def _run_correlation_stress_tests(self, positions: pd.DataFrame) -> Dict[str, float]:1129 """1130 Run correlation stress tests on portfolio positions1131 1132 Args:1133 positions: Portfolio positions over time1134 1135 Returns:1136 Dictionary of correlation stress test results1137 """1138 try:1139 # Calculer la matrice de corrélation de base1140 base_corr = positions.corr()1141 1142 # Différents scénarios de stress sur les corrélations1143 stress_scenarios = {1144 'high_correlation': {1145 'adjustment': 0.3, # Augmentation des corrélations de 30%1146 'floor': 0.3, # Corrélation minimum1147 'cap': 0.95 # Corrélation maximum1148 },1149 'correlation_breakdown': {1150 'adjustment': -0.5, # Diminution des corrélations de 50%1151 'floor': -0.8, # Corrélation minimum1152 'cap': 0.2 # Corrélation maximum1153 }1154 }1155 1156 results = {}1157 1158 for scenario, params in stress_scenarios.items():1159 # Ajuster la matrice de corrélation1160 stressed_corr = base_corr + params['adjustment']1161 stressed_corr = np.clip(stressed_corr, params['floor'], params['cap'])1162 np.fill_diagonal(stressed_corr, 1.0)1163 1164 # Calculer le risque du portefeuille avec les nouvelles corrélations1165 weights = positions.iloc[-1] # Utiliser les derniers poids1166 portfolio_risk = np.sqrt(1167 weights @ stressed_corr @ weights * 1168 (self.returns.std() * np.sqrt(252))**21169 )1170 1171 results[scenario] = {1172 'portfolio_risk': portfolio_risk,1173 'diversification_score': 1 - (portfolio_risk / (weights * self.returns.std() * np.sqrt(252)).sum()),1174 'max_correlation': stressed_corr.max(),1175 'min_correlation': stressed_corr.min()1176 }1177 1178 return results1179 1180 except Exception as e:1181 print(f"Error in correlation stress tests: {str(e)}")1182 return {1183 'high_correlation': {1184 'portfolio_risk': 0.0,1185 'diversification_score': 0.0,1186 'max_correlation': 0.0,1187 'min_correlation': 0.01188 }1189 }1190 1191 def _run_hypothetical_stress_tests(self, returns: pd.Series) -> Dict[str, float]:1192 """Run hypothetical stress scenarios"""1193 scenarios = {1194 'market_crash': {'shock': -0.20, 'duration': 22}, # -20% sur un mois1195 'volatility_spike': {'vol_multiplier': 3, 'duration': 10},1196 'correlation_breakdown': {'correlation_shock': 0.8, 'duration': 15},1197 'liquidity_crisis': {'bid_ask_multiplier': 5, 'duration': 30}1198 }1199 1200 results = {}