Yash030/claude-code-proxy
2
1"""FastAPI route handlers."""2 3from fastapi import APIRouter, Depends, HTTPException, Request, Response4from fastapi.responses import HTMLResponse5from loguru import logger6from starlette.templating import Jinja2Templates7 8from config.settings import Settings9from core.anthropic import get_token_count10from providers.nvidia_nim import metrics as nvidia_nim_metrics11from providers.registry import ProviderRegistry12 13from . import dependencies14from .dependencies import get_settings, require_api_key15from .gateway_model_ids import gateway_model_id, no_thinking_gateway_model_id16from .models.anthropic import MessagesRequest, TokenCountRequest17from .models.responses import ModelResponse, ModelsListResponse18from .services import ClaudeProxyService19 20router = APIRouter()21 22templates = Jinja2Templates(directory="templates")23 24DISCOVERED_MODEL_CREATED_AT = "1970-01-01T00:00:00Z"25 26 27# The proxy advertises a curated set of provider-backed models. Replace28# the previous hardcoded Claude model list with the requested NVIDIA-29# compatible models so clients only see those options.30REQUESTED_PROVIDER_MODELS = [31 # Zen/OpenCode free models32 "zen/minimax-m2.5-free",33 "zen/big-pickle",34 "zen/ring-2.6-1t-free",35 "zen/nemotron-3-super-free",36 # NVIDIA NIM models (top 5)37 "nvidia_nim/stepfun-ai/step-3.5-flash",38 "nvidia_nim/qwen/qwen3-coder-480b-a35b-instruct",39 "nvidia_nim/mistralai/mistral-large-3-675b-instruct-2512",40 "nvidia_nim/z-ai/glm4.7",41 "nvidia_nim/minimaxai/minimax-m2.7",42 # Cerebras models (key only has access to llama3.1-8b currently)43 # qwen-3-235b-a22b-instruct-2507 exists but is rate-limited44 # zai-glm-4.7 and gpt-oss-120b are not accessible with current key45 "cerebras/llama3.1-8b",46 # Silicon Flow models (top 5 for free tier)47 # DeepSeek-V3 - strong MoE model48 "silicon/deepseek-ai/DeepSeek-V3",49 # Qwen3-Coder-30B-A3B - coding specialized50 "silicon/Qwen/Qwen3-Coder-30B-A3B-Instruct",51 # Qwen3.6-35B-A3B - multimodal, 262K context52 "silicon/Qwen/Qwen3.6-35B-A3B",53 # Qwen2.5-72B - strong general purpose, 128K context54 "silicon/Qwen/Qwen2.5-72B-Instruct",55 # Qwen3-32B - reasoning model56 "silicon/Qwen/Qwen3-32B",57 # Groq models (ultra fast inference)58 "groq/llama-3.3-70b-versatile",59 "groq/llama-3.1-8b-instant",60 "groq/qwen3-32b",61]62 63 64def get_proxy_service(65 request: Request,66 settings: Settings = Depends(get_settings),67) -> ClaudeProxyService:68 """Build the request service for route handlers."""69 return ClaudeProxyService(70 settings,71 provider_getter=lambda provider_type: dependencies.resolve_provider(72 provider_type, app=request.app, settings=settings73 ),74 token_counter=get_token_count,75 )76 77 78def _probe_response(allow: str) -> Response:79 """Return an empty success response for compatibility probes."""80 return Response(status_code=204, headers={"Allow": allow})81 82 83def _discovered_model_response(model_id: str, *, display_name: str) -> ModelResponse:84 return ModelResponse(85 id=model_id,86 display_name=display_name,87 created_at=DISCOVERED_MODEL_CREATED_AT,88 )89 90 91def _append_unique_model(92 models: list[ModelResponse], seen: set[str], model: ModelResponse93) -> None:94 if model.id in seen:95 return96 seen.add(model.id)97 models.append(model)98 99 100def _append_provider_model_variants(101 models: list[ModelResponse],102 seen: set[str],103 provider_model_ref: str,104 *,105 supports_thinking: bool | None = None,106) -> None:107 if supports_thinking is not False:108 _append_unique_model(109 models,110 seen,111 _discovered_model_response(112 gateway_model_id(provider_model_ref),113 display_name=provider_model_ref,114 ),115 )116 _append_unique_model(117 models,118 seen,119 _discovered_model_response(120 no_thinking_gateway_model_id(provider_model_ref),121 display_name=f"{provider_model_ref} (no thinking)",122 ),123 )124 125 126def _build_models_list_response(127 settings: Settings, provider_registry: ProviderRegistry | None128) -> ModelsListResponse:129 models: list[ModelResponse] = []130 seen: set[str] = set()131 132 # Advertise only the requested provider models (no Claude models, no registry auto-discovery).133 # Each ref is added with both thinking and no-thinking variants.134 for provider_ref in REQUESTED_PROVIDER_MODELS:135 # If the ref already contains a provider prefix, use it as-is;136 # otherwise assume it belongs to the NVIDIA NIM provider.137 ref = provider_ref if "/" in provider_ref else f"nvidia_nim/{provider_ref}"138 supports_thinking = None139 if provider_registry is not None:140 # model_id for registry lookups should be provider-prefixed141 provider, model_id = (142 ref.split("/", 1) if "/" in ref else ("nvidia_nim", ref)143 )144 supports_thinking = provider_registry.cached_model_supports_thinking(145 provider, model_id146 )147 _append_provider_model_variants(148 models, seen, ref, supports_thinking=supports_thinking149 )150 151 # Add a virtual `auto` model that maps to the configured MODEL and enables152 # automatic fallback behavior when used by clients.153 _append_unique_model(154 models,155 seen,156 ModelResponse(157 id=gateway_model_id("auto"),158 display_name="auto (use configured fallbacks)",159 created_at=DISCOVERED_MODEL_CREATED_AT,160 ),161 )162 163 # Filter out any residual Claude-branded models so the proxy advertises164 # only the provider-backed models requested by the user.165 filtered = [166 m167 for m in models168 if "claude" not in (m.id or "").lower()169 and "claude" not in (m.display_name or "").lower()170 ]171 # Ensure `auto` model remains available even if filtering removed others.172 if not any(m.id == gateway_model_id("auto") for m in filtered):173 filtered.append(174 ModelResponse(175 id=gateway_model_id("auto"),176 display_name="auto (use configured fallbacks)",177 created_at=DISCOVERED_MODEL_CREATED_AT,178 )179 )180 181 return ModelsListResponse(182 data=filtered,183 first_id=filtered[0].id if filtered else None,184 has_more=False,185 last_id=filtered[-1].id if filtered else None,186 )187 188 189# =============================================================================190# Routes191# =============================================================================192@router.post("/v1/messages")193async def create_message(194 request: Request,195 request_data: MessagesRequest,196 service: ClaudeProxyService = Depends(get_proxy_service),197 _auth=Depends(require_api_key),198):199 """Create a message (always streaming)."""200 return service.create_message(request, request_data)201 202 203@router.api_route("/v1/messages", methods=["HEAD", "OPTIONS"])204async def probe_messages(_auth=Depends(require_api_key)):205 """Respond to Claude compatibility probes for the messages endpoint."""206 return _probe_response("POST, HEAD, OPTIONS")207 208 209@router.post("/v1/messages/count_tokens")210async def count_tokens(211 request_data: TokenCountRequest,212 service: ClaudeProxyService = Depends(get_proxy_service),213 _auth=Depends(require_api_key),214):215 """Count tokens for a request."""216 return service.count_tokens(request_data)217 218 219@router.api_route("/v1/messages/count_tokens", methods=["HEAD", "OPTIONS"])220async def probe_count_tokens(_auth=Depends(require_api_key)):221 """Respond to Claude compatibility probes for the token count endpoint."""222 return _probe_response("POST, HEAD, OPTIONS")223 224 225@router.get("/", response_class=HTMLResponse)226async def root(request: Request, _auth=Depends(require_api_key)):227 """Root endpoint - displays admin dashboard."""228 from .admin import _get_admin_data229 230 data = _get_admin_data()231 232 return templates.TemplateResponse("admin.html", {"request": request, **data})233 234 235@router.api_route("/", methods=["HEAD", "OPTIONS"])236async def probe_root(_auth=Depends(require_api_key)):237 """Respond to compatibility probes for the root endpoint."""238 return _probe_response("GET, HEAD, OPTIONS")239 240 241@router.get("/health")242async def health():243 """Health check endpoint."""244 return {"status": "healthy"}245 246 247@router.api_route("/health", methods=["HEAD", "OPTIONS"])248async def probe_health():249 """Respond to compatibility probes for the health endpoint."""250 return _probe_response("GET, HEAD, OPTIONS")251 252 253@router.get("/v1/models", response_model=ModelsListResponse)254async def list_models(255 request: Request,256 settings: Settings = Depends(get_settings),257 _auth=Depends(require_api_key),258):259 """List the model ids this proxy advertises to Claude-compatible clients."""260 registry = getattr(request.app.state, "provider_registry", None)261 provider_registry = registry if isinstance(registry, ProviderRegistry) else None262 return _build_models_list_response(settings, provider_registry)263 264 265@router.post("/stop")266async def stop_cli(request: Request, _auth=Depends(require_api_key)):267 """Stop all CLI sessions and pending tasks."""268 handler = getattr(request.app.state, "message_handler", None)269 if not handler:270 # Fallback if messaging not initialized271 cli_manager = getattr(request.app.state, "cli_manager", None)272 if cli_manager:273 await cli_manager.stop_all()274 logger.info("STOP_CLI: source=cli_manager cancelled_count=N/A")275 return {"status": "stopped", "source": "cli_manager"}276 raise HTTPException(status_code=503, detail="Messaging system not initialized")277 278 count = await handler.stop_all_tasks()279 logger.info("STOP_CLI: source=handler cancelled_count={}", count)280 return {"status": "stopped", "cancelled_count": count}281 282 283@router.get("/admin/fallbacks")284async def admin_fallbacks(_auth=Depends(require_api_key)):285 """Admin endpoint exposing NVIDIA NIM fallback metrics.286 287 Protected by the same API key as other endpoints.288 """289 try:290 data = nvidia_nim_metrics.snapshot()291 except Exception as e:292 logger.warning("ADMIN_FALLBACKS: failed to read metrics: {}", e)293 raise HTTPException(status_code=500, detail="failed to read metrics")294 return {"provider": "nvidia_nim", "fallbacks": data}295