aankitdas/resource-optimization-ml

1import pandas as pd2import numpy as np3from faker import Faker4from datetime import datetime, timedelta5import random6 7# Set random seed for reproducibility8np.random.seed(42)9random.seed(42)10 11fake = Faker()12 13print("Starting Data Generation...")14 15# ==================== PART 1: Generate Services ====================16print("\nGenerating Services Data...")17 18services_data = []19service_templates = [20    "auth", "cache", "database", "api", "notification",21    "search", "recommendation", "payment", "inventory", "profile",22    "order", "analytics", "logging", "metrics", "config",23    "gateway", "queue", "processor", "manager", "service",24    "worker", "scheduler", "validator", "router", "balancer"25]26 27# Generate 150 services by combining templates28service_names = []29for i in range(6):30    for template in service_templates:31        service_names.append(f"{template}-service-{i+1}")32 33for i, name in enumerate(service_names, start=1):34    services_data.append({35        'service_id': i,36        'service_name': name,37        'memory_mb': random.choice([256, 512, 1024, 2048, 4096]),38        'cpu_cores': random.choice([0.5, 1, 2, 4]),39        'latency_critical': random.choice([True, False]),40        'traffic_volume_rps': random.randint(1000, 100000),  # requests per second41        'dependencies': random.randint(0, 5)  # how many other services it depends on42    })43 44services_df = pd.DataFrame(services_data)45services_df.to_csv('data/services.csv', index=False)46print(f"Generated {len(services_df)} services")47print(services_df.head())48 49# ==================== PART 2: Generate Regional Latency ====================50print("\nGenerating Regional Latency Data...")51 52regions = ['us-east-1', 'us-west-2', 'eu-west-1', 'ap-southeast-1', 'ap-northeast-1']53latency_data = []54 55# Create latency matrix (some regions are closer than others)56latency_matrix = {57    ('us-east-1', 'us-west-2'): (60, 80),58    ('us-east-1', 'eu-west-1'): (90, 110),59    ('us-east-1', 'ap-southeast-1'): (180, 220),60    ('us-east-1', 'ap-northeast-1'): (150, 190),61    ('us-west-2', 'eu-west-1'): (130, 160),62    ('us-west-2', 'ap-southeast-1'): (140, 170),63    ('us-west-2', 'ap-northeast-1'): (110, 140),64    ('eu-west-1', 'ap-southeast-1'): (200, 250),65    ('eu-west-1', 'ap-northeast-1'): (180, 230),66    ('ap-southeast-1', 'ap-northeast-1'): (50, 80),67}68 69# Generate latency measurements over time70start_date = datetime(2024, 1, 1)71for days in range(90):  # 3 months72    timestamp = start_date + timedelta(days=days)73    74    for region1 in regions:75        for region2 in regions:76            if region1 == region2:77                latency_data.append({78                    'region1': region1,79                    'region2': region2,80                    'latency_ms': random.gauss(2, 0.5),  # same region: ~2ms81                    'timestamp': timestamp82                })83            elif (region1, region2) in latency_matrix:84                min_lat, max_lat = latency_matrix[(region1, region2)]85                base_latency = np.random.uniform(min_lat, max_lat)86                # Add some noise87                latency = base_latency + random.gauss(0, 5)88                latency_data.append({89                    'region1': region1,90                    'region2': region2,91                    'latency_ms': max(latency, 1),  # ensure positive92                    'timestamp': timestamp93                })94            elif (region2, region1) in latency_matrix:95                min_lat, max_lat = latency_matrix[(region2, region1)]96                base_latency = np.random.uniform(min_lat, max_lat)97                latency = base_latency + random.gauss(0, 5)98                latency_data.append({99                    'region1': region1,100                    'region2': region2,101                    'latency_ms': max(latency, 1),102                    'timestamp': timestamp103                })104 105latency_df = pd.DataFrame(latency_data)106latency_df.to_csv('data/regional_latency.csv', index=False)107print(f"Generated {len(latency_df)} latency measurements")108print(latency_df.head())109 110# ==================== PART 3: Generate Traffic Patterns ====================111print("\nGenerating Traffic Patterns...")112 113traffic_data = []114start_date = datetime(2024, 1, 1)115 116for days in range(90):  # 3 months117    for hour in range(24):118        timestamp = start_date + timedelta(days=days, hours=hour)119        120        # Peak hours are 9-17 (business hours)121        hour_of_day = timestamp.hour122        if 9 <= hour_of_day <= 17:123            traffic_multiplier = random.uniform(1.5, 2.5)124        elif 22 <= hour_of_day or hour_of_day <= 6:125            traffic_multiplier = random.uniform(0.2, 0.5)  # low traffic at night126        else:127            traffic_multiplier = random.uniform(0.8, 1.2)128        129        # Weekend traffic is lower130        if timestamp.weekday() >= 5:  # Saturday = 5, Sunday = 6131            traffic_multiplier *= 0.7132        133        for service_id, service_row in services_df.iterrows():134            base_traffic = service_row['traffic_volume_rps']135            136            for region in regions:137                # Different regions have different traffic volumes138                region_factor = {139                    'us-east-1': 1.0,140                    'us-west-2': 0.8,141                    'eu-west-1': 0.6,142                    'ap-southeast-1': 0.5,143                    'ap-northeast-1': 0.4,144                }[region]145                146                requests = int(base_traffic * traffic_multiplier * region_factor)147                148                traffic_data.append({149                    'service_id': service_id + 1,150                    'region': region,151                    'hour': hour,152                    'requests': requests,153                    'timestamp': timestamp154                })155 156traffic_df = pd.DataFrame(traffic_data)157traffic_df.to_csv('data/traffic_patterns.csv', index=False)158print(f"Generated {len(traffic_df)} traffic records")159print(traffic_df.head())160 161# ==================== PART 4: Generate Placement History ====================162print("\nGenerating Service Placement History...")163 164placement_data = []165start_date = datetime(2024, 1, 1)166 167for days in range(90):168    timestamp = start_date + timedelta(days=days)169    170    for service_id in range(1, len(service_names) + 1):171        service = services_df[services_df['service_id'] == service_id].iloc[0]172        173        # Latency critical services are usually in fewer regions174        if service['latency_critical']:175            num_regions = random.choice([1, 2])176        else:177            num_regions = random.choice([2, 3, 4])178        179        placement_regions = random.sample(regions, num_regions)180        181        for region in placement_regions:182            placement_data.append({183                'service_id': service_id,184                'region': region,185                'timestamp': timestamp,186                'instances': random.randint(1, 5),187                'avg_latency_ms': random.uniform(5, 100),188                'error_rate': random.uniform(0, 0.05)189            })190 191placement_df = pd.DataFrame(placement_data)192placement_df.to_csv('data/service_placement.csv', index=False)193print(f"Generated {len(placement_df)} placement records")194print(placement_df.head())195 196# ==================== Summary ====================197print("\n" + "="*50)198print("ALL DATA GENERATED SUCCESSFULLY!")199print("="*50)200print(f"\nFiles created in 'data/' folder:")201print(f"   • services.csv ({len(services_df)} rows)")202print(f"   • regional_latency.csv ({len(latency_df)} rows)")203print(f"   • traffic_patterns.csv ({len(traffic_df)} rows)")204print(f"   • service_placement.csv ({len(placement_df)} rows)")205print(f"\nTotal records generated: {len(services_df) + len(latency_df) + len(traffic_df) + len(placement_df):,}")206