Deepvest/ProfilingAI
0
1from __future__ import annotations2 3import asyncio4import hashlib5import json6import logging7import os8import re9from dataclasses import asdict, dataclass, field10from datetime import datetime, timedelta11from functools import lru_cache12from typing import Any, Dict, List, Optional, Tuple, Union, Set13 14import numpy as np15import torch16from transformers import AutoTokenizer, AutoModelForCausalLM17try:18 from transformers import PhiForCausalLM19except ImportError:20 pass21try:22 from peft import PeftModel, PeftConfig23except ImportError:24 pass25 26# Configure logging27logging.basicConfig(28 level=logging.INFO,29 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'30)31logger = logging.getLogger("DeepVest")32 33# Configuration manager34@dataclass35class DeepVestConfig:36 """Configuration for the DeepVest system"""37 # Model configuration38 model_path: str = "financial_profiler_model"39 use_quantization: bool = True40 cache_size: int = 10041 42 # Database configuration43 db_path: str = "profiles_db"44 use_persistent_storage: bool = True45 46 # Analysis configuration47 default_horizon: int = 548 default_risk_tolerance: int = 349 min_recommendation_count: int = 350 51 # Performance settings52 enable_caching: bool = True53 batch_size: int = 454 parallel_processing: bool = True55 56 # Market data integration57 fetch_market_data: bool = False58 market_data_api_key: Optional[str] = None59 60 # Debug settings61 debug_mode: bool = False62 log_prompts: bool = False63 64 def __post_init__(self):65 """Validate configuration after initialization"""66 if self.debug_mode:67 logging.getLogger("DeepVest").setLevel(logging.DEBUG)68 69# Core data models70@dataclass71class FinancialAnalysis:72 """Detailed financial analysis of an investor profile"""73 income_stability: float = 0.574 debt_ratio: float = 0.075 savings_capacity: float = 0.076 emergency_fund_months: float = 3.077 risk_capacity: float = 0.578 investment_horizon: int = 579 liquidity_needs: float = 0.280 income_sources: Dict[str, float] = field(default_factory=dict)81 fixed_expenses: Dict[str, float] = field(default_factory=dict)82 variable_expenses: Dict[str, float] = field(default_factory=dict)83 credit_score: Optional[int] = None84 tax_bracket: Optional[float] = None85 86 def validate(self) -> bool:87 """Validate financial analysis attributes"""88 try:89 validations = [90 0 <= self.income_stability <= 1,91 self.debt_ratio >= 0,92 0 <= self.savings_capacity <= 1,93 self.emergency_fund_months >= 0,94 0 <= self.risk_capacity <= 1,95 self.investment_horizon > 0,96 0 <= self.liquidity_needs <= 197 ]98 if self.credit_score is not None:99 validations.append(300 <= self.credit_score <= 850)100 if self.tax_bracket is not None:101 validations.append(0 <= self.tax_bracket <= 1)102 103 return all(validations)104 except Exception as e:105 logger.error(f"Error validating financial analysis: {e}")106 return False107 108 def calculate_total_expenses(self) -> float:109 """Calculate total monthly expenses"""110 return (111 sum(self.fixed_expenses.values()) +112 sum(self.variable_expenses.values())113 )114 115 def calculate_savings_rate(self) -> float:116 """Calculate savings rate as a percentage of income"""117 total_income = sum(self.income_sources.values())118 if total_income == 0:119 return 0120 return self.savings_capacity / total_income121 122 def get_financial_health_score(self) -> float:123 """Calculate overall financial health score (0-1)"""124 # Calculate components with different weights125 stability_weight = 0.3126 debt_weight = 0.25127 savings_weight = 0.25128 emergency_weight = 0.2129 130 # Calculate debt score (lower is better)131 debt_score = max(0, 1 - (self.debt_ratio / 0.5))132 133 # Calculate emergency fund score (higher is better)134 emergency_score = min(1, self.emergency_fund_months / 6)135 136 # Calculate savings score137 savings_score = min(1, self.savings_capacity * 3)138 139 # Calculate overall score140 health_score = (141 stability_weight * self.income_stability +142 debt_weight * debt_score +143 savings_weight * savings_score +144 emergency_weight * emergency_score145 )146 147 return min(1, max(0, health_score))148 149@dataclass150class FamilyAnalysis:151 """Detailed analysis of family situation"""152 family_size: int = 1153 dependents: int = 0154 life_stage: str = 'undefined'155 future_events: List[Dict] = field(default_factory=list)156 monthly_obligations: float = 0.0157 insurance_coverage: Dict[str, float] = field(default_factory=dict)158 education_needs: Dict[str, float] = field(default_factory=dict)159 healthcare_needs: Dict[str, float] = field(default_factory=dict)160 retirement_plans: Dict[str, Any] = field(default_factory=dict)161 estate_plans: Dict[str, Any] = field(default_factory=dict)162 163 def validate(self) -> bool:164 """Validate family analysis attributes"""165 try:166 validations = [167 self.family_size >= 1,168 self.dependents >= 0,169 self.monthly_obligations >= 0,170 all(value >= 0 for value in self.insurance_coverage.values()),171 all(value >= 0 for value in self.education_needs.values()),172 all(value >= 0 for value in self.healthcare_needs.values())173 ]174 return all(validations)175 176 except Exception as e:177 logger.error(f"Error validating family analysis: {e}")178 return False179 180 def calculate_total_obligations(self) -> float:181 """Calculate total monthly obligations"""182 return (183 self.monthly_obligations +184 sum(self.insurance_coverage.values()) +185 sum(self.education_needs.values()) +186 sum(self.healthcare_needs.values())187 )188 189 def get_next_life_event(self) -> Optional[Dict]:190 """Get the next upcoming life event"""191 if not self.future_events:192 return None193 194 # Sort events by date and return the closest one195 future_events = [e for e in self.future_events if 'date' in e]196 if not future_events:197 return None198 199 return min(future_events, key=lambda x: x['date'])200 201 def calculate_family_complexity(self) -> float:202 """Calculate family situation complexity (0-1)"""203 # Base complexity based on family size204 base_complexity = min(1, (self.family_size - 1) * 0.2)205 206 # Additional complexity for dependents207 dependent_complexity = min(0.5, self.dependents * 0.1)208 209 # Additional complexity for upcoming events210 events_complexity = min(0.3, len(self.future_events) * 0.05)211 212 # Additional complexity for special needs213 special_needs = len(self.healthcare_needs) > 0 or len(self.education_needs) > 0214 special_complexity = 0.2 if special_needs else 0215 216 return min(1, base_complexity + dependent_complexity + events_complexity + special_complexity)217 218@dataclass219class InvestmentGoals:220 """Detailed investment goals"""221 primary_goals: List[Dict] = field(default_factory=list)222 timeframes: Dict[str, int] = field(default_factory=dict)223 required_returns: Dict[str, float] = field(default_factory=dict)224 priority_order: List[str] = field(default_factory=list)225 target_amounts: Dict[str, float] = field(default_factory=dict)226 goal_progress: Dict[str, float] = field(default_factory=dict)227 risk_tolerance_by_goal: Dict[str, int] = field(default_factory=dict)228 milestones: Dict[str, List[Dict]] = field(default_factory=dict)229 230 def validate(self) -> bool:231 """Validate investment goals attributes"""232 try:233 if self.target_amounts and any(amount < 0 for amount in self.target_amounts.values()):234 return False235 236 if self.required_returns and any(ret < -1 or ret > 2 for ret in self.required_returns.values()):237 return False238 239 if self.risk_tolerance_by_goal and any(tol < 1 or tol > 5 for tol in self.risk_tolerance_by_goal.values()):240 return False241 242 if self.goal_progress and any(prog < 0 or prog > 1 for prog in self.goal_progress.values()):243 return False244 245 return True246 247 except Exception as e:248 logger.error(f"Error validating investment goals: {e}")249 return False250 251 def calculate_total_target_amount(self) -> float:252 """Calculate total target amount across all goals"""253 return sum(self.target_amounts.values())254 255 def get_highest_priority_goal(self) -> Optional[str]:256 """Get the highest priority goal"""257 return self.priority_order[0] if self.priority_order else None258 259 def calculate_weighted_risk_tolerance(self) -> float:260 """Calculate risk tolerance weighted by goal amount"""261 if not self.risk_tolerance_by_goal or not self.target_amounts:262 return 3.0 # Default value263 264 total_amount = self.calculate_total_target_amount()265 if total_amount == 0:266 return 3.0267 268 weighted_tolerance = sum(269 self.risk_tolerance_by_goal.get(goal, 3) * amount / total_amount270 for goal, amount in self.target_amounts.items()271 )272 return weighted_tolerance273 274 def calculate_goal_progress(self, goal_id: str) -> float:275 """Calculate progress percentage for a specific goal"""276 if goal_id not in self.target_amounts or goal_id not in self.goal_progress:277 return 0.0278 return self.goal_progress[goal_id] / self.target_amounts[goal_id]279 280 def get_upcoming_milestones(self, days: int = 90) -> List[Dict]:281 """Get upcoming milestones within specified days"""282 upcoming = []283 now = datetime.now()284 cutoff = now + timedelta(days=days)285 286 for goal, milestones in self.milestones.items():287 for milestone in milestones:288 date = milestone.get('date')289 if date and now <= date <= cutoff:290 upcoming.append({291 'goal': goal,292 **milestone293 })294 295 return sorted(upcoming, key=lambda x: x['date'])296 297 def estimate_goal_feasibility(self, 298 monthly_contribution: float, 299 expected_return: float) -> Dict[str, float]:300 """Estimate feasibility of goals based on contributions and returns"""301 feasibility = {}302 303 for goal_id, target in self.target_amounts.items():304 # Get current progress305 current_amount = self.goal_progress.get(goal_id, 0)306 307 # Get timeframe in months308 months = self.timeframes.get(goal_id, 60) * 12309 310 if months <= 0:311 feasibility[goal_id] = 0312 continue313 314 # Calculate future value of current amount315 future_current = current_amount * (1 + expected_return/12) ** months316 317 # Calculate future value of monthly contributions318 future_contributions = monthly_contribution * ((1 + expected_return/12) ** months - 1) / (expected_return/12)319 320 # Calculate total expected amount321 expected_amount = future_current + future_contributions322 323 # Calculate feasibility as percentage of target324 feasibility[goal_id] = min(1.0, expected_amount / target)325 326 return feasibility327 328@dataclass329class ProfileAnalysisResult:330 """Complete result of LLM profile analysis"""331 # Basic metrics332 risk_score: float333 investment_horizon: int334 primary_goals: List[str]335 336 # Detailed analyses337 constraints: Dict[str, Any]338 recommendations: List[str]339 explanation: str340 341 # Advanced metrics342 risk_decomposition: Dict[str, float] = field(default_factory=dict)343 goal_feasibility: Dict[str, float] = field(default_factory=dict)344 investment_style: Dict[str, float] = field(default_factory=dict)345 market_sensitivity: Dict[str, float] = field(default_factory=dict)346 347 # Recommended allocations348 asset_allocation: Dict[str, float] = field(default_factory=dict)349 geographic_allocation: Dict[str, float] = field(default_factory=dict)350 351 # Alerts and attention points352 alerts: List[Dict[str, Any]] = field(default_factory=list)353 attention_points: List[str] = field(default_factory=list)354 355 # Context and adaptability356 market_context: Dict[str, Any] = field(default_factory=dict)357 adaptability_score: float = 0.5358 stress_test_results: Dict[str, float] = field(default_factory=dict)359 360 def __post_init__(self):361 """Post-initialization validation"""362 if not 0 <= self.risk_score <= 1:363 raise ValueError("Risk score must be between 0 and 1")364 if self.investment_horizon <= 0:365 raise ValueError("Investment horizon must be positive")366 367 def get_risk_category(self) -> str:368 """Determine risk category based on score"""369 if self.risk_score < 0.2:370 return "Très conservateur"371 elif self.risk_score < 0.4:372 return "Conservateur"373 elif self.risk_score < 0.6:374 return "Modéré"375 elif self.risk_score < 0.8:376 return "Dynamique"377 else:378 return "Très dynamique"379 380 def get_critical_alerts(self) -> List[Dict[str, Any]]:381 """Get critical alerts only"""382 return [alert for alert in self.alerts if alert.get('severity') == 'critical']383 384 def get_major_constraints(self) -> List[str]:385 """Identify major constraints"""386 return [387 constraint for constraint, value in self.constraints.items()388 if isinstance(value, (bool, int, float)) and value > 0.7389 ]390 391 def get_primary_investment_style(self) -> str:392 """Determine primary investment style"""393 if not self.investment_style:394 return "Balanced"395 return max(self.investment_style.items(), key=lambda x: x[1])[0]396 397 def calculate_overall_feasibility(self) -> float:398 """Calculate overall goal feasibility"""399 if not self.goal_feasibility:400 return 0.5401 return sum(self.goal_feasibility.values()) / len(self.goal_feasibility)402 403 def get_risk_factors(self) -> List[Tuple[str, float]]:404 """Get main risk factors sorted by importance"""405 return sorted(406 self.risk_decomposition.items(),407 key=lambda x: x[1],408 reverse=True409 )410 411 def get_market_adaptability(self) -> Dict[str, float]:412 """Evaluate adaptability to market conditions"""413 adaptability = {}414 for condition, sensitivity in self.market_sensitivity.items():415 adaptability[condition] = min(416 self.adaptability_score * (1 + sensitivity),417 1.0418 )419 return adaptability420 421 def generate_one_page_summary(self) -> str:422 """Generate a concise one-page summary of the analysis"""423 risk_category = self.get_risk_category()424 425 summary = [426 f"# Profil d'Investissement: {risk_category}",427 f"Score de risque: {self.risk_score:.2f}/1.00 | Horizon: {self.investment_horizon} ans",428 "",429 "## Objectifs Principaux",430 *[f"- {goal}" for goal in self.primary_goals[:3]],431 "",432 "## Allocation d'Actifs Recommandée",433 *[f"- {asset}: {weight:.1%}" for asset, weight in 434 sorted(self.asset_allocation.items(), key=lambda x: x[1], reverse=True)],435 "",436 "## Facteurs de Risque Principaux",437 *[f"- {factor}: {score:.1%}" for factor, score in self.get_risk_factors()[:3]],438 "",439 "## Recommandations Clés",440 *[f"- {rec}" for rec in self.recommendations[:3]],441 "",442 "## Points d'Attention",443 *[f"- {point}" for point in self.attention_points[:3]],444 ]445 446 return "\n".join(summary)447 448 def to_dict(self) -> Dict[str, Any]:449 """Convert analysis to dictionary"""450 return {451 'risk_profile': {452 'score': self.risk_score,453 'category': self.get_risk_category(),454 'decomposition': self.risk_decomposition,455 'stress_tests': self.stress_test_results456 },457 'investment_profile': {458 'horizon': self.investment_horizon,459 'style': self.investment_style,460 'adaptability': self.adaptability_score,461 'market_sensitivity': self.market_sensitivity462 },463 'goals_analysis': {464 'primary_goals': self.primary_goals,465 'feasibility': self.goal_feasibility,466 'constraints': self.constraints467 },468 'recommendations': {469 'suggestions': self.recommendations,470 'attention_points': self.attention_points,471 'alerts': self.alerts472 },473 'allocations': {474 'assets': self.asset_allocation,475 'geography': self.geographic_allocation476 },477 'market_context': self.market_context,478 'explanation': self.explanation479 }480 481@dataclass482class PersonalProfile:483 """Complete and dynamic investor profile with advanced analysis"""484 # Identifiers and metadata485 id: Optional[str] = None486 created_at: datetime = field(default_factory=datetime.now)487 last_updated: datetime = field(default_factory=datetime.now)488 version: int = 1489 490 # Base profile491 risk_tolerance: int = 3492 investment_horizon: int = 5493 income_stability: int = 3494 financial_knowledge: int = 3495 investment_goals: List[str] = field(default_factory=list)496 esg_preferences: bool = False497 constraints: Dict[str, Any] = field(default_factory=dict)498 499 # Personal information500 age: int = 30501 annual_income: float = 50000.0502 total_assets: float = 0.0503 total_debt: float = 0.0504 monthly_savings: float = 0.0 # Added this field for the épargne mensuelle505 occupation: str = ""506 industry: str = ""507 education_level: str = ""508 marital_status: str = "single"509 number_of_dependents: int = 0510 tax_status: str = ""511 residential_status: str = "owner"512 513 # Detailed analyses514 financial_analysis: Optional[FinancialAnalysis] = None515 family_analysis: Optional[FamilyAnalysis] = None516 investment_objectives: Optional[InvestmentGoals] = None517 profile_description: Optional[str] = None518 519 # Dynamic metrics and history520 risk_metrics: Dict[str, float] = field(default_factory=dict)521 performance_metrics: Dict[str, float] = field(default_factory=dict)522 historical_changes: List[Dict[str, Any]] = field(default_factory=list)523 last_review: Optional[datetime] = None524 525 # Investment preferences526 investment_preferences: Dict[str, Any] = field(default_factory=lambda: {527 'asset_classes': [],528 'sectors': [],529 'regions': [],530 'strategies': [],531 'excluded_investments': []532 })533 534 # LLM analysis results535 llm_analysis_results: Optional[ProfileAnalysisResult] = None536 risk_score: float = 0.5537 adaptability_score: float = 0.5538 complexity_tolerance: float = 0.5539 540 def __post_init__(self):541 """Post-initialization setup"""542 # Generate ID if not provided543 if self.id is None:544 self.id = self._generate_profile_id()545 546 # Initialize analyses if not provided547 if self.financial_analysis is None:548 self.financial_analysis = FinancialAnalysis()549 550 if self.family_analysis is None:551 self.family_analysis = FamilyAnalysis()552 553 if self.investment_objectives is None:554 self.investment_objectives = InvestmentGoals()555 556 # Validate profile557 self._validate_profile()558 def _generate_profile_id(self) -> str:559 """Generate unique profile ID"""560 components = [561 str(self.age),562 str(self.annual_income),563 str(self.risk_tolerance),564 self.occupation,565 datetime.now().isoformat()566 ]567 unique_string = "_".join(components)568 return f"PROFILE_{hashlib.sha256(unique_string.encode()).hexdigest()[:16]}"569 570 def _validate_profile(self):571 """Comprehensive profile validation"""572 validations = [573 (1 <= self.risk_tolerance <= 5, "Risk tolerance must be between 1 and 5"),574 (1 <= self.investment_horizon <= 50, "Investment horizon must be between 1 and 50 years"),575 (1 <= self.income_stability <= 5, "Income stability must be between 1 and 5"),576 (1 <= self.financial_knowledge <= 5, "Financial knowledge must be between 1 and 5"),577 (18 <= self.age <= 120, "Age must be between 18 and 120"),578 (self.annual_income >= 0, "Annual income cannot be negative"),579 (self.total_assets >= 0, "Total assets cannot be negative"),580 (self.total_debt >= 0, "Total debt cannot be negative"),581 (0 <= self.risk_score <= 1, "Risk score must be between 0 and 1"),582 (0 <= self.adaptability_score <= 1, "Adaptability score must be between 0 and 1"),583 (0 <= self.complexity_tolerance <= 1, "Complexity tolerance must be between 0 and 1")584 ]585 586 # Advanced validation587 if self.investment_preferences:588 for asset_class in self.investment_preferences.get('asset_classes', []):589 if not isinstance(asset_class, str):590 raise ValueError("Invalid asset class format")591 592 if self.investment_goals and len(self.investment_goals) == 0:593 raise ValueError("At least one investment goal must be specified")594 595 # Check basic constraints596 for validation, message in validations:597 if not validation:598 raise ValueError(message)599 600 # Check detailed analyses601 if self.financial_analysis and not self.financial_analysis.validate():602 raise ValueError("Invalid financial analysis")603 604 if self.family_analysis and not self.family_analysis.validate():605 raise ValueError("Invalid family analysis")606 607 if self.investment_objectives and not self.investment_objectives.validate():608 raise ValueError("Invalid investment objectives")609 610 def calculate_comprehensive_risk_metrics(self) -> Dict[str, float]:611 """Calculate comprehensive risk metrics"""612 try:613 risk_metrics = {614 'total_debt_ratio': self.total_debt / self.annual_income if self.annual_income > 0 else float('inf'),615 'emergency_fund_ratio': self.financial_analysis.emergency_fund_months / 6 if self.financial_analysis else 0,616 'risk_capacity': self.financial_analysis.risk_capacity if self.financial_analysis else 0.5,617 'investment_diversification': len(self.investment_goals) / 5 if self.investment_goals else 0,618 'debt_service_ratio': self._calculate_debt_service_ratio(),619 'savings_rate': self._calculate_savings_rate(),620 'liquidity_ratio': self._calculate_liquidity_ratio(),621 'investment_concentration': self._calculate_investment_concentration()622 }623 624 # Add LLM scores625 if self.llm_analysis_results:626 risk_metrics.update({627 'llm_risk_score': self.llm_analysis_results.risk_score,628 'adaptability': self.llm_analysis_results.adaptability_score629 })630 631 # Calculate overall risk capacity632 financial_health = self.financial_analysis.get_financial_health_score() if self.financial_analysis else 0.5633 risk_metrics['overall_risk_capacity'] = (634 risk_metrics['risk_capacity'] * 0.5 + 635 (1 - risk_metrics['total_debt_ratio']) * 0.3 + 636 financial_health * 0.2637 )638 639 self.risk_metrics = risk_metrics640 return risk_metrics641 642 except Exception as e:643 logger.error(f"Error calculating risk metrics: {e}")644 return {}645 646 def _calculate_debt_service_ratio(self) -> float:647 """Calculate debt service ratio"""648 try:649 if not self.financial_analysis or self.annual_income == 0:650 return 0.0651 652 if isinstance(self.financial_analysis.fixed_expenses, dict) and 'debt_payments' in self.financial_analysis.fixed_expenses:653 if isinstance(self.financial_analysis.fixed_expenses['debt_payments'], dict):654 monthly_debt_payment = sum(self.financial_analysis.fixed_expenses['debt_payments'].values())655 else:656 monthly_debt_payment = self.financial_analysis.fixed_expenses['debt_payments']657 else:658 # Estimate monthly debt payment using total debt and a standard amortization659 monthly_debt_payment = self.total_debt * 0.01 # Rough estimate of 1% monthly payment660 661 monthly_income = self.annual_income / 12662 return monthly_debt_payment / monthly_income if monthly_income > 0 else 0663 664 except Exception as e:665 logger.error(f"Error calculating debt service ratio: {e}")666 return 0.0667 668 def _calculate_savings_rate(self) -> float:669 """Calculate savings rate"""670 try:671 if not self.financial_analysis or self.annual_income == 0:672 return 0.0673 return (self.financial_analysis.savings_capacity * 12) / self.annual_income674 except Exception as e:675 logger.error(f"Error calculating savings rate: {e}")676 return 0.0677 678 def _calculate_liquidity_ratio(self) -> float:679 """Calculate liquidity ratio"""680 try:681 if not self.financial_analysis or self.total_assets == 0:682 return 0.0683 684 liquid_assets = sum(685 amount for type_, amount in self.financial_analysis.income_sources.items()686 if type_ in ['savings', 'investments', 'cash']687 )688 return liquid_assets / self.total_assets689 690 except Exception as e:691 logger.error(f"Error calculating liquidity ratio: {e}")692 return 0.0693 694 def _calculate_investment_concentration(self) -> float:695 """Calculate investment concentration using HHI"""696 try:697 if not self.investment_preferences or not self.investment_preferences.get('asset_classes'):698 return 1.0699 700 asset_classes = self.investment_preferences['asset_classes']701 weights = [1/len(asset_classes)] * len(asset_classes)702 return sum(w * w for w in weights)703 704 except Exception as e:705 logger.error(f"Error calculating investment concentration: {e}")706 return 1.0707 708 def update(self, updates: Dict[str, Any]) -> 'PersonalProfile':709 """Update profile with new information"""710 try:711 # Create copy of current profile data712 current_data = asdict(self)713 714 # Save change history715 self._save_change_history(updates)716 717 # Update fields718 for key, value in updates.items():719 if key in current_data:720 current_data[key] = value721 722 # Update version and timestamps723 current_data['version'] += 1724 current_data['last_updated'] = datetime.now()725 726 # Create new profile with updated data727 updated_profile = self.__class__(**current_data)728 729 # Validate updated profile730 updated_profile._validate_profile()731 732 return updated_profile733 734 except Exception as e:735 logger.error(f"Error updating profile: {e}")736 raise737 738 def _save_change_history(self, updates: Dict[str, Any]):739 """Record change history"""740 change_record = {741 'timestamp': datetime.now().isoformat(),742 'changes': updates,743 'previous_values': {744 key: getattr(self, key) 745 for key in updates.keys() 746 if hasattr(self, key)747 },748 'version': self.version749 }750 self.historical_changes.append(change_record)751 752 def get_change_history(self, field: Optional[str] = None) -> List[Dict[str, Any]]:753 """Get change history for optional field"""754 if field:755 return [756 change for change in self.historical_changes757 if field in change['changes']758 ]759 return self.historical_changes760 761 def export_json(self, filepath: str, include_detailed_analysis: bool = False):762 """Export profile to JSON file"""763 try:764 # Make sure directory exists765 os.makedirs(os.path.dirname(filepath), exist_ok=True)766 767 with open(filepath, 'w', encoding='utf-8') as f:768 json.dump(769 asdict(self) if include_detailed_analysis else self.to_dict(), 770 f, ensure_ascii=False, indent=2, default=str771 )772 logger.info(f"Profile exported successfully to {filepath}")773 774 except Exception as e:775 logger.error(f"Error exporting profile to {filepath}: {e}")776 raise777 778 def to_dict(self) -> Dict[str, Any]:779 """Convert profile to dictionary representation"""780 return {781 'id': self.id,782 'version': self.version,783 'created_at': self.created_at.isoformat(),784 'last_updated': self.last_updated.isoformat(),785 'basic_info': {786 'age': self.age,787 'annual_income': self.annual_income,788 'total_assets': self.total_assets,789 'total_debt': self.total_debt,790 'marital_status': self.marital_status,791 'dependents': self.number_of_dependents792 },793 'investment_profile': {794 'risk_tolerance': self.risk_tolerance,795 'investment_horizon': self.investment_horizon,796 'financial_knowledge': self.financial_knowledge,797 'income_stability': self.income_stability,798 'goals': self.investment_goals,799 'esg_preferences': self.esg_preferences800 },801 'risk_metrics': self.risk_metrics,802 'llm_analysis': self.llm_analysis_results.to_dict() if self.llm_analysis_results else None803 }804 805 @classmethod806 def from_json(cls, filepath: str) -> 'PersonalProfile':807 """Load profile from JSON file"""808 try:809 with open(filepath, 'r', encoding='utf-8') as f:810 data = json.load(f)811 812 # Convert date strings to datetime objects813 for date_field in ['created_at', 'last_updated', 'last_review']:814 if date_field in data and data[date_field]:815 data[date_field] = datetime.fromisoformat(data[date_field])816 817 # Reconstruct complex objects818 if 'financial_analysis' in data and data['financial_analysis']:819 data['financial_analysis'] = FinancialAnalysis(**data['financial_analysis'])820 821 if 'family_analysis' in data and data['family_analysis']:822 data['family_analysis'] = FamilyAnalysis(**data['family_analysis'])823 824 if 'investment_objectives' in data and data['investment_objectives']:825 data['investment_objectives'] = InvestmentGoals(**data['investment_objectives'])826 827 if 'llm_analysis_results' in data and data['llm_analysis_results']:828 data['llm_analysis_results'] = ProfileAnalysisResult(**data['llm_analysis_results'])829 830 # Create and validate profile831 profile = cls(**data)832 profile._validate_profile()833 834 return profile835 836 except Exception as e:837 logger.error(f"Error loading profile from {filepath}: {e}")838 raise839 840class DeepVestLLM:841 """Advanced LLM integration using fine-tuned financial advisor model"""842 843 def __init__(self, config: DeepVestConfig):844 """Initialize with configuration"""845 self.config = config846 self.logger = logging.getLogger("DeepVest.LLM")847 848 # Initialize model and tokenizer849 self.model = None850 self.tokenizer = None851 self.device = self._detect_optimal_device()852 853 # Load model if available854 if os.path.exists(config.model_path):855 self.load_model(config.model_path) # Pass the model_path argument856 else:857 self.logger.warning(f"Model path {config.model_path} not found, using fallback mode")858 859 # Initialize cache860 self.generation_cache = {}861 self.max_cache_entries = config.cache_size862 863 def _detect_optimal_device(self) -> str:864 """Detect the optimal device for model inference"""865 if torch.cuda.is_available():866 self.logger.info("CUDA GPU available, using CUDA")867 return "cuda"868 elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available():869 self.logger.info("Apple MPS available, using MPS")870 return "mps"871 else:872 self.logger.info("No GPU available, using CPU")873 return "cpu"874 875 def load_model(self, model_path=None):876 """Load the fine-tuned model from the configured path"""877 try:878 path_to_use = model_path if model_path else self.config.model_path879 self.logger.info(f"Loading fine-tuned model from {path_to_use}")880 881 # Determine the best device to use882 if torch.cuda.is_available():883 self.logger.info("CUDA available, using GPU")884 device = "cuda"885 elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():886 self.logger.info("Apple MPS available, using MPS")887 device = "mps"888 else:889 self.logger.info("Using CPU")890 device = "cpu"891 892 # Check if config.json exists and has required fields893 config_file = os.path.join(path_to_use, "config.json")894 if os.path.exists(config_file):895 with open(config_file, 'r') as f:896 config = json.load(f)897 898 # If no model_type, add one for Phi model899 if 'model_type' not in config:900 self.logger.warning("Adding missing model_type to config.json")901 config['model_type'] = "phi" # Use phi since you're fine-tuning Phi-1.5902 903 # Add architecture if missing904 if 'architectures' not in config:905 config['architectures'] = ["PhiForCausalLM"]906 907 # Save the modified config908 with open(config_file, 'w') as f:909 json.dump(config, f, indent=2)910 else:911 # Create a basic config.json if it doesn't exist912 self.logger.warning(f"config.json not found in {path_to_use}, creating one")913 config = {914 "model_type": "phi",915 "architectures": ["PhiForCausalLM"],916 "bos_token_id": 1,917 "eos_token_id": 2,918 "pad_token_id": 0,919 "hidden_size": 2048,920 "vocab_size": 51200,921 "torch_dtype": "float16"922 }923 with open(config_file, 'w') as f:924 json.dump(config, f, indent=2)925 926 # Try different loading approaches927 try:928 # First attempt: direct loading with AutoModelForCausalLM929 self.tokenizer = AutoTokenizer.from_pretrained(930 path_to_use,931 trust_remote_code=True932 )933 934 self.model = AutoModelForCausalLM.from_pretrained(935 path_to_use,936 trust_remote_code=True,937 torch_dtype=torch.float16,938 device_map="auto",939 low_cpu_mem_usage=True940 )941 942 except Exception as model_error:943 self.logger.warning(f"First loading attempt failed: {model_error}")944 945 # Second attempt: try loading with specific model class946 try:947 from transformers import PhiForCausalLM948 949 self.tokenizer = AutoTokenizer.from_pretrained(path_to_use, trust_remote_code=True)950 self.model = PhiForCausalLM.from_pretrained(951 path_to_use,952 trust_remote_code=True,953 torch_dtype=torch.float16,954 device_map="auto",955 low_cpu_mem_usage=True956 )957 958 except Exception as specific_error:959 self.logger.warning(f"Second loading attempt failed: {specific_error}")960 961 # Third attempt: try loading as a PEFT/LoRA model962 try:963 from peft import PeftModel, PeftConfig964 965 # Check if we need to load the base model first966 base_model_path = "microsoft/phi-1_5"967 base_model = AutoModelForCausalLM.from_pretrained(968 base_model_path,969 trust_remote_code=True,970 torch_dtype=torch.float16,971 device_map="auto"972 )973 974 self.tokenizer = AutoTokenizer.from_pretrained(base_model_path, trust_remote_code=True)975 self.model = PeftModel.from_pretrained(base_model, path_to_use)976 977 except Exception as peft_error:978 # All attempts failed979 self.logger.error(f"All loading attempts failed. Last error: {peft_error}")980 raise Exception(f"Could not load model: {model_error} -> {specific_error} -> {peft_error}")981 982 # Make sure the tokenizer has a pad token983 if self.tokenizer.pad_token is None:984 self.tokenizer.pad_token = self.tokenizer.eos_token985 986 self.logger.info(f"Model loaded successfully")987 return True988 989 except Exception as e:990 self.logger.error(f"Error loading model: {e}")991 return False992 993 def _get_cache_key(self, prompt: str) -> str:994 """Generate cache key for a prompt"""995 return hashlib.md5(prompt.encode()).hexdigest()996 997 def _manage_cache(self):998 """Manage cache size"""999 if len(self.generation_cache) > self.max_cache_entries:1000 # Sort by access timestamp and remove oldest1001 sorted_entries = sorted(1002 self.generation_cache.items(),1003 key=lambda x: x[1]['timestamp']1004 )1005 # Remove oldest 20%1006 entries_to_remove = int(self.max_cache_entries * 0.2)1007 for i in range(entries_to_remove):1008 if i < len(sorted_entries):1009 del self.generation_cache[sorted_entries[i][0]]1010 1011 self.logger.debug(f"Cache cleanup: removed {entries_to_remove} entries")1012 1013 async def generate_text(self, prompt: str, max_length: int = 1500) -> str:1014 """Generate text using the loaded model"""1015 # Check cache if enabled1016 if self.config.enable_caching:1017 cache_key = self._get_cache_key(prompt)1018 if cache_key in self.generation_cache:1019 cache_entry = self.generation_cache[cache_key]1020 cache_entry['timestamp'] = datetime.now() # Update access timestamp1021 self.logger.debug("Using cached response")1022 return cache_entry['response']1023 1024 if self.config.log_prompts:1025 self.logger.debug(f"Prompt: {prompt}")1026 1027 # Check if model_loaded is True to confirm successful loading1028 if hasattr(self, 'model_loaded') and self.model_loaded and self.model and self.tokenizer:1029 # Generate with loaded model1030 try:1031 inputs = self.tokenizer(prompt, return_tensors="pt", truncation=True, max_length=4096)1032 1033 # Handle device mapping - use the model's device rather than explicitly moving inputs1034 if hasattr(self.model, 'device'):1035 device = self.model.device1036 inputs = {k: v.to(device) for k, v in inputs.items()}1037 1038 with torch.no_grad():1039 outputs = self.model.generate(1040 **inputs,1041 max_length=max_length,1042 do_sample=True,1043 temperature=0.7,1044 top_p=0.92,1045 top_k=50,1046 repetition_penalty=1.15,1047 num_beams=4,1048 pad_token_id=self.tokenizer.eos_token_id1049 )1050 1051 response = self.tokenizer.decode(outputs[0], skip_special_tokens=True)1052 1053 # Extract actual response from the prompt1054 if "[/INST]" in response:1055 response = response.split("[/INST]")[-1].strip()1056 1057 # Cache the result if enabled1058 if self.config.enable_caching:1059 self.generation_cache[cache_key] = {1060 'response': response,1061 'timestamp': datetime.now()1062 }1063 self._manage_cache()1064 1065 return response1066 1067 except Exception as e:1068 self.logger.error(f"Error generating text: {e}")1069 return await self._fallback_generation(prompt)1070 else:1071 # Use fallback generation1072 self.logger.info("Using fallback generation")1073 return await self._fallback_generation(prompt)1074 1075 async def _fallback_generation(self, prompt: str) -> str:1076 """Fallback generation when model is unavailable"""1077 self.logger.info("Using fallback generation")1078 1079 # Extract key information from prompt1080 risk_tolerance = 31081 investment_horizon = 51082 1083 # Extract risk tolerance if present1084 risk_match = re.search(r'[Rr]isque.*?(\d+)', prompt)1085 if risk_match:1086 risk_tolerance = int(risk_match.group(1))1087 1088 # Extract investment horizon if present1089 horizon_match = re.search(r'[Hh]orizon.*?(\d+)', prompt)1090 if horizon_match:1091 investment_horizon = int(horizon_match.group(1))1092 1093 # Generate simple structured response1094 risk_score = risk_tolerance / 51095 1096 fallback_response = f"""1. Score de risque: {risk_score:.2f} - Basé sur la tolérance au risque déclarée.10972. Horizon d'investissement recommandé: {investment_horizon} ans.10983. Objectifs prioritaires: Épargne générale, Croissance du capital.10994. Contraintes: Limitation classique de diversification.1100 11015. Recommandations d'allocation:1102 - Actions: {int(risk_score * 100)}%1103 - Obligations: {int((1-risk_score) * 80)}%1104 - Liquidités: {int((1-risk_score) * 20)}%1105 11066. Recommandations:1107 - Diversifier votre portefeuille1108 - Investir régulièrement1109 - Maintenir une épargne de sécurité suffisante1110 11117. Points d'attention:1112 - Réévaluer régulièrement votre profil de risque1113 - Ajuster votre allocation selon l'évolution de votre situation1114 - Prévoir une réserve d'urgence adéquate1115"""1116 return fallback_response1117 1118 def prepare_analysis_prompt(self, profile_data: Dict[str, Any]) -> str:1119 """Prepare analysis prompt from profile data"""1120 return f"""[INST] Je suis un conseiller financier expert. Analyse ce profil d'investisseur de manière exhaustive.1121 1122Profil détaillé :1123- Âge: {profile_data.get('age', 'Non spécifié')} ans1124- Revenu annuel: {profile_data.get('annual_income', 'Non spécifié')}€1125- Situation familiale: {profile_data.get('marital_status', 'Non spécifié')}1126- Dépendants: {profile_data.get('number_of_dependents', 0)}1127- Épargne mensuelle: {profile_data.get('monthly_savings', 'Non spécifié')}€1128- Dette totale: {profile_data.get('total_debt', 'Non spécifié')}€1129- Patrimoine total: {profile_data.get('total_assets', 'Non spécifié')}€1130- Objectifs: {", ".join(profile_data.get('investment_goals', []))}1131- Tolérance au risque (1-5): {profile_data.get('risk_tolerance', 3)}1132- Horizon d'investissement: {profile_data.get('investment_horizon', 5)} ans1133- Connaissances financières (1-5): {profile_data.get('financial_knowledge', 3)}1134- Stabilité professionnelle: {profile_data.get('income_stability', 3)}1135- Préférences ESG: {"Oui" if profile_data.get('esg_preferences', False) else "Non"}1136- Secteurs préférés: {", ".join(profile_data.get('investment_preferences', {}).get('sectors', []))}1137- Contraintes spécifiques: {", ".join(str(k) for k, v in profile_data.get('constraints', {}).items() if v)}1138 1139Format de réponse requis:11401. Score de risque (entre 0 et 1): [score] - Justification détaillée11412. Horizon d'investissement recommandé: [années] - Justification11423. Objectifs prioritaires: Liste ordonnée avec justification11434. Contraintes et limitations: Analyse détaillée des contraintes11445. Recommandations d'allocation:1145 - Par classe d'actifs (%)1146 - Par zone géographique (%)1147 - Par stratégie d'investissement11486. Analyse des risques:1149 - Décomposition des risques1150 - Points de vigilance1151 - Stress tests recommandés11527. Plan d'action détaillé:1153 - Court terme (0-2 ans)1154 - Moyen terme (2-5 ans)1155 - Long terme (5+ ans)11568. Recommandations spécifiques:1157 - Investissements recommandés1158 - Stratégies de diversification1159 - Protection du capital11609. Explication détaillée:1161 - Raisonnement complet1162 - Points d'attention particuliers1163 - Opportunités identifiées1164 1165Analyse complète: [/INST]"""1166 1167 def prepare_description_prompt(self, profile_data: Dict[str, Any]) -> str:1168 """Prepare description prompt from profile data"""1169 return f"""[INST] En tant que conseiller financier expert, génère une description détaillée et professionnelle de ce profil d'investisseur.1170 1171Profil complet:1172- Âge: {profile_data.get('age', 'Non spécifié')} ans1173- Revenu annuel: {profile_data.get('annual_income', 'Non spécifié')}€1174- Situation familiale: {profile_data.get('marital_status', 'Non spécifié')}1175- Objectifs: {", ".join(profile_data.get('investment_goals', []))}1176- Tolérance au risque: {profile_data.get('risk_tolerance', 3)}/51177- Horizon: {profile_data.get('investment_horizon', 5)} ans1178- Connaissances financières: {profile_data.get('financial_knowledge', 3)}/51179- Épargne mensuelle: {profile_data.get('monthly_savings', 'Non spécifié')}€1180- Patrimoine: {profile_data.get('total_assets', 'Non spécifié')}€1181- Dette: {profile_data.get('total_debt', 'Non spécifié')}€1182- Secteur d'activité: {profile_data.get('industry', 'Non spécifié')}1183- Stabilité professionnelle: {profile_data.get('income_stability', 3)}/51184 1185La description doit couvrir:11861. Analyse du profil de risque et des objectifs11872. Evaluation de la capacité financière11883. Identification des contraintes et opportunités11894. Recommandations d'investissement personnalisées11905. Points de vigilance et attention particulière11916. Stratégie de diversification recommandée11927. Projection et scénarios d'évolution1193 1194Description: [/INST]"""1195 1196 async def generate_batch(self, prompts: List[str], max_length: int = 1500) -> List[str]:1197 """Generate responses for multiple prompts in parallel"""1198 if not self.model or not self.tokenizer:1199 # Fallback for batch generation1200 return [await self._fallback_generation(prompt) for prompt in prompts]