LetsTryGPT/agent4-implementation

1/**2 * LLM Response Cache with TTL (Time To Live)3 * Reduces API costs and improves response time for repeated prompts4 */5 6import { logger } from './logger';7import crypto from 'crypto';8 9export interface CacheEntry<T> {10  value: T;11  timestamp: number;12  ttl: number;13  hits: number;14}15 16export interface CacheStats {17  hits: number;18  misses: number;19  size: number;20  hitRate: number;21}22 23export class LLMCache<T = string> {24  private cache: Map<string, CacheEntry<T>> = new Map();25  private defaultTTL: number;26  private maxSize: number;27  private hits: number = 0;28  private misses: number = 0;29  private cleanupInterval: ReturnType<typeof setInterval> | null = null;30 31  constructor(options: { defaultTTL?: number; maxSize?: number; cleanupIntervalMs?: number } = {}) {32    this.defaultTTL = options.defaultTTL || 5 * 60 * 1000; // 5 minutes default33    this.maxSize = options.maxSize || 1000; // Maximum 1000 cache entries34    const cleanupIntervalMs = options.cleanupIntervalMs || 60 * 1000; // Cleanup every minute35 36    // Start automatic cleanup of expired entries37    // Use unref() to allow process to exit gracefully even if interval is active38    this.cleanupInterval = setInterval(() => {39      this.cleanup();40    }, cleanupIntervalMs);41    this.cleanupInterval.unref();42 43    logger.info('LLM Cache initialized', {44      defaultTTL: this.defaultTTL,45      maxSize: this.maxSize,46      cleanupIntervalMs,47    });48  }49 50  /**51   * Generate cache key from prompt and options using SHA-256 hash52   *53   * @param prompt - The prompt text54   * @param options - Optional generation options (affects cache key)55   * @returns SHA-256 hash as cache key56   *57   * @remarks58   * Identical prompts with different options generate different keys.59   * Prompts are normalized (trimmed and lowercased) before hashing.60   */61  private generateKey(prompt: string, options?: Record<string, unknown>): string {62    const normalized = {63      prompt: prompt.trim().toLowerCase(),64      options: options || {},65    };66    const hash = crypto.createHash('sha256').update(JSON.stringify(normalized)).digest('hex');67    return hash;68  }69 70  /**71   * Get value from cache72   */73  get(prompt: string, options?: Record<string, unknown>): T | null {74    const key = this.generateKey(prompt, options);75    const entry = this.cache.get(key);76 77    if (!entry) {78      this.misses++;79      logger.debug('Cache miss', { prompt: prompt.substring(0, 50) });80      return null;81    }82 83    // Check if entry has expired84    const now = Date.now();85    if (now - entry.timestamp > entry.ttl) {86      this.cache.delete(key);87      this.misses++;88      logger.debug('Cache expired', {89        prompt: prompt.substring(0, 50),90        age: now - entry.timestamp,91      });92      return null;93    }94 95    // Update hit counter96    entry.hits++;97    this.hits++;98    logger.debug('Cache hit', {99      prompt: prompt.substring(0, 50),100      hits: entry.hits,101      age: now - entry.timestamp,102    });103 104    return entry.value;105  }106 107  /**108   * Set value in cache109   */110  set(prompt: string, value: T, options?: Record<string, unknown>, ttl?: number): void {111    // Check if cache is full112    if (this.cache.size >= this.maxSize) {113      this.evictOldest();114    }115 116    const key = this.generateKey(prompt, options);117    const entry: CacheEntry<T> = {118      value,119      timestamp: Date.now(),120      ttl: ttl || this.defaultTTL,121      hits: 0,122    };123 124    this.cache.set(key, entry);125    logger.debug('Cache set', {126      prompt: prompt.substring(0, 50),127      ttl: entry.ttl,128      size: this.cache.size,129    });130  }131 132  /**133   * Check if key exists and is valid134   */135  has(prompt: string, options?: Record<string, unknown>): boolean {136    return this.get(prompt, options) !== null;137  }138 139  /**140   * Delete entry from cache141   */142  delete(prompt: string, options?: Record<string, unknown>): boolean {143    const key = this.generateKey(prompt, options);144    const deleted = this.cache.delete(key);145    if (deleted) {146      logger.debug('Cache delete', { prompt: prompt.substring(0, 50) });147    }148    return deleted;149  }150 151  /**152   * Clear all cache entries153   */154  clear(): void {155    const size = this.cache.size;156    this.cache.clear();157    this.hits = 0;158    this.misses = 0;159    logger.info('Cache cleared', { entriesRemoved: size });160  }161 162  /**163   * Remove expired entries164   */165  private cleanup(): void {166    const now = Date.now();167    let removed = 0;168 169    for (const [key, entry] of this.cache.entries()) {170      if (now - entry.timestamp > entry.ttl) {171        this.cache.delete(key);172        removed++;173      }174    }175 176    if (removed > 0) {177      logger.debug('Cache cleanup', { entriesRemoved: removed, remainingSize: this.cache.size });178    }179  }180 181  /**182   * Evict oldest entry (LRU-like behavior based on timestamp)183   */184  private evictOldest(): void {185    let oldestKey: string | null = null;186    let oldestTime = Infinity;187 188    for (const [key, entry] of this.cache.entries()) {189      if (entry.timestamp < oldestTime) {190        oldestTime = entry.timestamp;191        oldestKey = key;192      }193    }194 195    if (oldestKey) {196      this.cache.delete(oldestKey);197      logger.debug('Cache evicted oldest entry', { size: this.cache.size });198    }199  }200 201  /**202   * Get cache statistics203   *204   * @returns Cache statistics including hits, misses, size, and hit rate205   *206   * @example207   * ```ts208   * const stats = cache.getStats();209   * console.log(`Hit rate: ${(stats.hitRate * 100).toFixed(2)}%`);210   * ```211   */212  getStats(): CacheStats {213    const total = this.hits + this.misses;214    return {215      hits: this.hits,216      misses: this.misses,217      size: this.cache.size,218      hitRate: total > 0 ? this.hits / total : 0,219    };220  }221 222  /**223   * Get all cache entries (for debugging)224   */225  getEntries(): Array<{ key: string; entry: CacheEntry<T> }> {226    return Array.from(this.cache.entries()).map(([key, entry]) => ({227      key,228      entry,229    }));230  }231 232  /**233   * Destroy cache and cleanup resources234   */235  destroy(): void {236    if (this.cleanupInterval) {237      clearInterval(this.cleanupInterval);238      this.cleanupInterval = null;239    }240    this.clear();241    logger.info('Cache destroyed');242  }243}244 245/**246 * Singleton cache instance for LLM responses247 */248export const llmCache = new LLMCache<string>({249  defaultTTL: 10 * 60 * 1000, // 10 minutes250  maxSize: 500, // Store up to 500 responses251  cleanupIntervalMs: 2 * 60 * 1000, // Cleanup every 2 minutes252});253