Felipe97/llama-cpp-compiled
01.2k
1#!/usr/bin/env python32"""3Tests for MCP server integration via the /tools endpoint.4 5Invariants verified:61. MCP tools appear in /tools listing when configured72. MCP tools use <server>_<tool> naming83. MCP tools can be invoked and return correct results94. Misconfigured MCP servers do not crash the server105. Multiple MCP servers can be configured simultaneously116. Warmup populates the tool list at startup12"""13import json14import os15import sys16import tempfile17import time18 19import pytest20 21from utils import *22 23# Path to the test MCP server fixture24FIXTURES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "fixtures")25MCP_ECHO_SERVER = os.path.join(FIXTURES_DIR, "mcp_echo_server.py")26 27server: ServerProcess28 29 30def _mcp_config_json(servers: dict) -> str:31 """Create a JSON config string for --mcp-servers-json."""32 return json.dumps({"mcpServers": servers})33 34 35def _start_server_with_mcp(mcp_json: str, **kwargs) -> ServerProcess:36 """Helper to start a router server with MCP config."""37 srv = ServerPreset.router()38 srv.server_tools = "all"39 srv.no_ui = True40 srv.mcp_servers_json = mcp_json41 for k, v in kwargs.items():42 setattr(srv, k, v)43 srv.start()44 return srv45 46 47def test_mcp_tools_listed_in_tools_endpoint():48 """MCP tools should appear in GET /tools with server:tool naming."""49 global server50 mcp_json = _mcp_config_json({51 "echo": {52 "command": sys.executable,53 "args": [MCP_ECHO_SERVER],54 }55 })56 server = _start_server_with_mcp(mcp_json)57 58 try:59 res = server.make_request("GET", "/tools")60 assert res.status_code == 200, res.body61 62 tools = res.body63 assert isinstance(tools, list), f"Expected list, got {type(tools)}"64 65 # Find MCP tools - name is in "tool" field or definition.function.name66 def get_tool_name(t):67 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")68 69 mcp_tools = [t for t in tools if get_tool_name(t).startswith("echo_")]70 assert len(mcp_tools) >= 2, f"Expected at least 2 echo_ tools, got {len(mcp_tools)}: {mcp_tools}"71 72 tool_names = {get_tool_name(t) for t in mcp_tools}73 assert "echo_echo" in tool_names74 assert "echo_add" in tool_names75 76 # Verify tool structure77 echo_tool = next(t for t in mcp_tools if get_tool_name(t) == "echo_echo")78 assert "description" in echo_tool or "definition" in echo_tool79 finally:80 server.stop()81 82 83def test_mcp_tool_invocation():84 """MCP tools should be callable via POST /tools and return correct results."""85 global server86 mcp_json = _mcp_config_json({87 "echo": {88 "command": sys.executable,89 "args": [MCP_ECHO_SERVER],90 }91 })92 server = _start_server_with_mcp(mcp_json)93 94 try:95 # Call echo_echo96 res = server.make_request("POST", "/tools", data={97 "tool": "echo_echo",98 "params": {"message": "hello world"}99 })100 assert res.status_code == 200, res.body101 body = res.body102 assert "error" not in body, body103 # The result format depends on the tool implementation104 # For MCP tools, it should contain the tool result105 assert "plain_text_response" in body or "result" in body or "content" in body, body106 107 # Call echo_add108 res = server.make_request("POST", "/tools", data={109 "tool": "echo_add",110 "params": {"a": 3, "b": 5}111 })112 assert res.status_code == 200, res.body113 body = res.body114 assert "error" not in body, body115 finally:116 server.stop()117 118 119def test_mcp_bad_command_does_not_crash():120 """A misconfigured MCP server should not crash the llama-server."""121 global server122 mcp_json = _mcp_config_json({123 "nonexistent": {124 "command": "this_executable_does_not_exist_12345",125 "args": [],126 }127 })128 server = _start_server_with_mcp(mcp_json)129 130 try:131 # Server should still be healthy132 res = server.make_request("GET", "/health")133 assert res.status_code == 200, res.body134 135 # Builtin tools should still work136 res = server.make_request("GET", "/tools")137 assert res.status_code == 200, res.body138 tools = res.body139 # Should have builtin tools but no MCP tools from the bad server140 mcp_tools = [t for t in tools if t.get("name", "").startswith("nonexistent_")]141 assert len(mcp_tools) == 0, f"Expected no nonexistent_ tools, got {mcp_tools}"142 finally:143 server.stop()144 145 146def test_mcp_multiple_servers():147 """Multiple MCP servers can be configured simultaneously."""148 global server149 mcp_json = _mcp_config_json({150 "echo": {151 "command": sys.executable,152 "args": [MCP_ECHO_SERVER],153 },154 "echo2": {155 "command": sys.executable,156 "args": [MCP_ECHO_SERVER],157 }158 })159 server = _start_server_with_mcp(mcp_json)160 161 try:162 res = server.make_request("GET", "/tools")163 assert res.status_code == 200, res.body164 165 tools = res.body166 167 def get_tool_name(t):168 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")169 170 echo_tools = [t for t in tools if get_tool_name(t).startswith("echo_")]171 echo2_tools = [t for t in tools if get_tool_name(t).startswith("echo2_")]172 173 assert len(echo_tools) >= 2, f"Expected echo_ tools, got {echo_tools}"174 assert len(echo2_tools) >= 2, f"Expected echo2_ tools, got {echo2_tools}"175 finally:176 server.stop()177 178 179def test_mcp_tools_not_listed_when_not_configured():180 """Without MCP config, no MCP tools should appear."""181 global server182 server = ServerPreset.router()183 server.server_tools = "all"184 server.no_ui = True185 server.start()186 187 try:188 res = server.make_request("GET", "/tools")189 assert res.status_code == 200, res.body190 191 tools = res.body192 193 def get_tool_name(t):194 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")195 196 # Should only have builtin tools, no server: prefixed tools197 mcp_tools = [t for t in tools if ":" in get_tool_name(t)]198 assert len(mcp_tools) == 0, f"Expected no MCP tools, got {mcp_tools}"199 finally:200 server.stop()201 202 203def test_mcp_fail_once_tool_eventual_success():204 """Test that a tool that fails once eventually succeeds (tests instance respawn)."""205 global server206 mcp_json = _mcp_config_json({207 "echo": {208 "command": sys.executable,209 "args": [MCP_ECHO_SERVER],210 }211 })212 server = _start_server_with_mcp(mcp_json)213 214 try:215 # First call should succeed (warmup already spawned and shut down the instance,216 # but the first actual tool call will spawn a fresh instance)217 res = server.make_request("POST", "/tools", data={218 "tool": "echo_fail_once",219 "params": {}220 })221 # It might fail on first call if the warmup instance was shut down222 # and a new instance is spawned. The fail_once state is per-process,223 # so a fresh process will fail once then succeed.224 # Actually, warmup spawns, lists, then shuts down. So the first tool call225 # spawns a new process which will fail once.226 assert res.status_code in (200, 500), res.body227 finally:228 server.stop()229 230 231def test_mcp_tools_via_json_config_file():232 """Test that --mcp-servers-config (file) works as well as --mcp-servers-json."""233 global server234 config = {235 "mcpServers": {236 "echo": {237 "command": sys.executable,238 "args": [MCP_ECHO_SERVER],239 }240 }241 }242 243 with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:244 json.dump(config, f)245 config_path = f.name246 247 try:248 server = ServerPreset.router()249 server.server_tools = "all"250 server.no_ui = True251 server.mcp_servers_config = config_path252 server.start()253 254 res = server.make_request("GET", "/tools")255 assert res.status_code == 200, res.body256 257 tools = res.body258 259 def get_tool_name(t):260 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")261 262 mcp_tools = [t for t in tools if get_tool_name(t).startswith("echo_")]263 assert len(mcp_tools) >= 2, f"Expected echo_ tools, got {mcp_tools}"264 finally:265 os.unlink(config_path)266 server.stop()267 268 269def test_mcp_tools_slot_independent():270 """MCP tools should work without any slot concept; /tools is slot-independent."""271 global server272 mcp_json = _mcp_config_json({273 "echo": {274 "command": sys.executable,275 "args": [MCP_ECHO_SERVER],276 }277 })278 server = _start_server_with_mcp(mcp_json)279 280 try:281 # Call /tools without any slot binding - should succeed282 res = server.make_request("POST", "/tools", data={283 "tool": "echo_echo",284 "params": {"message": "hello"}285 })286 assert res.status_code == 200, res.body287 body = res.body288 assert "error" not in body, body289 finally:290 server.stop()291 292 293def test_mcp_concurrent_tool_calls():294 """Concurrent POST /tools to same MCP server should all succeed."""295 global server296 mcp_json = _mcp_config_json({297 "echo": {298 "command": sys.executable,299 "args": [MCP_ECHO_SERVER],300 }301 })302 server = _start_server_with_mcp(mcp_json)303 304 try:305 def call_tool():306 return server.make_request("POST", "/tools", data={307 "tool": "echo_echo",308 "params": {"message": "hi"}309 })310 311 with ThreadPoolExecutor(max_workers=10) as executor:312 futures = [executor.submit(call_tool) for _ in range(10)]313 results = [f.result() for f in futures]314 315 for res in results:316 assert res.status_code == 200, res.body317 assert "error" not in res.body, res.body318 finally:319 server.stop()320 321 322def test_mcp_tool_timeout():323 """Tool call should timeout if MCP server is too slow."""324 global server325 MCP_SLOW_SERVER = os.path.join(FIXTURES_DIR, "mcp_slow_server.py")326 mcp_json = _mcp_config_json({327 "slow": {328 "command": sys.executable,329 "args": [MCP_SLOW_SERVER, "--delay", "5"],330 "timeout_ms": 500331 }332 })333 server = _start_server_with_mcp(mcp_json)334 335 try:336 res = server.make_request("POST", "/tools", data={337 "tool": "slow_sleep",338 "params": {"seconds": 5}339 })340 assert res.status_code == 200, res.body341 body = res.body342 assert "error" in body, body343 finally:344 server.stop()345 346 347def test_mcp_warmup_partial_failure():348 """Good server's tools should appear even if bad server fails warmup."""349 global server350 mcp_json = _mcp_config_json({351 "good": {352 "command": sys.executable,353 "args": [MCP_ECHO_SERVER],354 },355 "bad": {356 "command": "nonexistent",357 "args": []358 }359 })360 server = _start_server_with_mcp(mcp_json)361 362 try:363 res = server.make_request("GET", "/tools")364 assert res.status_code == 200, res.body365 tools = res.body366 367 def get_tool_name(t):368 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")369 370 # good server tools should be present371 assert any("good_" in get_tool_name(t) for t in tools), f"Expected good: tools in {tools}"372 finally:373 server.stop()374 375 376def test_mcp_notification_during_request():377 """Notification during request should not be returned as response."""378 global server379 MCP_MALFORMED_SERVER = os.path.join(FIXTURES_DIR, "mcp_malformed_server.py")380 mcp_json = _mcp_config_json({381 "notifying": {382 "command": sys.executable,383 "args": [MCP_MALFORMED_SERVER],384 }385 })386 server = _start_server_with_mcp(mcp_json)387 388 try:389 res = server.make_request("POST", "/tools", data={390 "tool": "notifying_echo",391 "params": {"message": "hi"}392 })393 assert res.status_code == 200, res.body394 body = res.body395 assert "error" not in body, body396 finally:397 server.stop()398 399 400def test_mcp_instance_respawn_after_crash():401 """Tool call after process crash should respawn and succeed."""402 global server403 MCP_CRASH_SERVER = os.path.join(FIXTURES_DIR, "mcp_crash_server.py")404 mcp_json = _mcp_config_json({405 "crash": {406 "command": sys.executable,407 "args": [MCP_CRASH_SERVER],408 }409 })410 server = _start_server_with_mcp(mcp_json)411 412 try:413 # First call succeeds414 res1 = server.make_request("POST", "/tools", data={415 "tool": "crash_echo",416 "params": {"message": "hi"}417 })418 assert res1.status_code == 200, res1.body419 assert "error" not in res1.body, res1.body420 421 # Second call should also succeed (respawned instance)422 res2 = server.make_request("POST", "/tools", data={423 "tool": "crash_echo",424 "params": {"message": "hi2"}425 })426 assert res2.status_code == 200, res2.body427 assert "error" not in res2.body, res2.body428 finally:429 server.stop()430 431 432 433 434def test_mcp_fail_once_eventual_success_verified():435 """Verify that fail_once tool eventually succeeds after respawn."""436 global server437 mcp_json = _mcp_config_json({438 "echo": {439 "command": sys.executable,440 "args": [MCP_ECHO_SERVER],441 }442 })443 server = _start_server_with_mcp(mcp_json)444 445 try:446 # First call may fail (fresh process)447 res1 = server.make_request("POST", "/tools", data={448 "tool": "echo_fail_once",449 "params": {}450 })451 # Second call should succeed452 res2 = server.make_request("POST", "/tools", data={453 "tool": "echo_fail_once",454 "params": {}455 })456 assert res2.status_code == 200, res2.body457 assert "error" not in res2.body, res2.body458 finally:459 server.stop()460 461 462def test_mcp_config_file_errors():463 """Invalid JSON config and missing file should cause server to fail to start."""464 # Invalid JSON - server should fail to start465 server = ServerPreset.router()466 server.server_tools = "all"467 server.no_ui = True468 server.mcp_servers_json = "not valid json"469 try:470 server.start()471 assert False, "Server should not have started with invalid MCP JSON config"472 except RuntimeError:473 pass # Expected: server process dies due to bad config474 475 # Missing file - server should fail to start476 server = ServerPreset.router()477 server.server_tools = "all"478 server.no_ui = True479 server.mcp_servers_config = "/nonexistent/path.json"480 try:481 server.start()482 assert False, "Server should not have started with missing config file"483 except RuntimeError:484 pass # Expected: server process dies due to missing config485 486 487def test_mcp_empty_tool_list():488 """MCP server reporting zero tools should result in empty tool list."""489 global server490 # Create a minimal server that returns empty tools list491 empty_server = os.path.join(FIXTURES_DIR, "_empty_mcp_server.py")492 with open(empty_server, "w") as f:493 f.write('''#!/usr/bin/env python3494import json, sys, os495def main():496 sys.stdout = os.fdopen(sys.stdout.fileno(), "w", buffering=1)497 for line in sys.stdin:498 line = line.strip()499 if not line: continue500 try: request = json.loads(line)501 except: continue502 method = request.get("method")503 req_id = request.get("id")504 if method == "initialize":505 resp = {"jsonrpc": "2.0", "id": req_id, "result": {"protocolVersion": "2024-11-05", "capabilities": {"tools": {}}, "serverInfo": {"name": "empty", "version": "1.0"}}}506 elif method == "tools/list":507 resp = {"jsonrpc": "2.0", "id": req_id, "result": {"tools": []}}508 else:509 resp = {"jsonrpc": "2.0", "id": req_id, "error": {"code": -32601, "message": "Method not found"}}510 sys.stdout.write(json.dumps(resp) + "\\n")511 sys.stdout.flush()512if __name__ == "__main__":513 main()514''')515 try:516 mcp_json = _mcp_config_json({517 "empty": {518 "command": sys.executable,519 "args": [empty_server],520 }521 })522 server = _start_server_with_mcp(mcp_json)523 res = server.make_request("GET", "/tools")524 assert res.status_code == 200, res.body525 tools = res.body526 def get_tool_name(t):527 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")528 mcp_tools = [t for t in tools if get_tool_name(t).startswith("empty:")]529 assert len(mcp_tools) == 0, f"Expected no empty: tools, got {mcp_tools}"530 finally:531 os.unlink(empty_server)532 server.stop()533 534 535def test_mcp_rapid_succession_calls():536 """Many rapid calls should increment next_id correctly and correlate responses."""537 global server538 mcp_json = _mcp_config_json({539 "echo": {540 "command": sys.executable,541 "args": [MCP_ECHO_SERVER],542 }543 })544 server = _start_server_with_mcp(mcp_json)545 546 try:547 for i in range(20):548 res = server.make_request("POST", "/tools", data={549 "tool": "echo_echo",550 "params": {"message": f"msg{i}"}551 })552 assert res.status_code == 200, res.body553 assert "error" not in res.body, res.body554 finally:555 server.stop()556 557 558def test_mcp_notification_burst():559 """Notification + response in a single write() with no flush should not strand the response."""560 global server561 MCP_BURST_SERVER = os.path.join(FIXTURES_DIR, "mcp_burst_server.py")562 mcp_json = _mcp_config_json({563 "burst": {564 "command": sys.executable,565 "args": [MCP_BURST_SERVER],566 }567 })568 server = _start_server_with_mcp(mcp_json)569 570 try:571 res = server.make_request("POST", "/tools", data={572 "tool": "burst_echo",573 "params": {"message": "burst test"}574 })575 assert res.status_code == 200, res.body576 body = res.body577 assert "error" not in body, body578 finally:579 server.stop()580 581 582def test_mcp_tool_definition_shape_via_chat_completions():583 """MCP tool definitions returned by GET /tools should have the correct shape for chat/completions."""584 global server585 mcp_json = _mcp_config_json({586 "echo": {587 "command": sys.executable,588 "args": [MCP_ECHO_SERVER],589 }590 })591 server = _start_server_with_mcp(mcp_json)592 593 try:594 # Get MCP tool definitions595 res = server.make_request("GET", "/tools")596 assert res.status_code == 200, res.body597 tools = res.body598 599 def get_tool_name(t):600 return t.get("tool", "") or t.get("definition", {}).get("function", {}).get("name", "")601 602 echo_tools = [t for t in tools if get_tool_name(t).startswith("echo_")]603 assert len(echo_tools) >= 2, f"Expected echo_ tools, got {echo_tools}"604 605 echo_tool = next(t for t in echo_tools if get_tool_name(t) == "echo_echo")606 definition = echo_tool.get("definition", echo_tool)607 608 # Verify the definition has the standard function-calling shape609 assert definition.get("type") == "function", f"Expected type=function, got {definition.get('type')}"610 func = definition.get("function", {})611 assert "name" in func, "Missing function.name"612 assert "description" in func, "Missing function.description"613 assert "parameters" in func, f"Missing function.parameters, got keys: {list(func.keys())}"614 params = func["parameters"]615 assert params.get("type") == "object", f"Expected parameters.type=object, got {params.get('type')}"616 assert "properties" in params, "Missing parameters.properties"617 finally:618 server.stop()619 620 621def test_mcp_slow_tool_call_slot_release():622 """A slow tool call should not stall server shutdown for the full I/O timeout."""623 global server624 MCP_SLOW_SERVER = os.path.join(FIXTURES_DIR, "mcp_slow_server.py")625 mcp_json = _mcp_config_json({626 "slow": {627 "command": sys.executable,628 "args": [MCP_SLOW_SERVER, "--delay", "10"],629 "timeout_ms": 30000630 }631 })632 server = _start_server_with_mcp(mcp_json)633 634 try:635 # Start a slow tool call in a background thread636 def slow_call():637 return server.make_request("POST", "/tools", data={638 "tool": "slow_sleep",639 "params": {"seconds": 10}640 })641 642 with ThreadPoolExecutor(max_workers=1) as executor:643 future = executor.submit(slow_call)644 645 # Wait a moment for the call to start646 time.sleep(2)647 648 # Stop the server while the tool call is in progress.649 # With global MCP instances, close_all() is called explicitly at shutdown650 # (not from slot release), so shutdown should complete promptly.651 start_time = time.time()652 server.stop()653 elapsed = time.time() - start_time654 655 # The server should stop quickly, not wait for the full 30s I/O timeout.656 # With the terminating flag, send_rpc() bails out within one select()657 # slice (~50ms). This threshold MUST stay below the 5s force-kill658 # fallback in ServerProcess.stop(): without the flag, shutdown stalls659 # on the instance mutex and only completes when stop() sends SIGKILL660 # at ~5s -- which any threshold above 5 would still accept.661 assert elapsed < 3, f"Server stop took {elapsed:.1f}s, expected < 3s"662 663 # Wait for the future to complete (it will get an error response or timeout)664 try:665 res = future.result(timeout=5)666 # If we got a response, it should be an error since the server stopped667 if hasattr(res, 'status_code'):668 assert res.status_code in (200, 500, 502, 503, 504), f"Unexpected status: {res.status_code}"669 except Exception:670 # Thread may have raised due to connection error - that's acceptable671 pass672 finally:673 server.stop()674 675 676def test_mcp_grandchild_holding_pipes_does_not_deadlock():677 """An MCP server that leaves a grandchild inheriting its stdout/stderr must not deadlock678 teardown.679 680 subprocess_terminate() only SIGKILLs the direct MCP child, so the inherited pipe write ends681 stay open and a blocking read on them would never see EOF. That hung both warmup shutdown682 (the server would never reach "ready") and process shutdown. The polled, running-aware reader683 must exit regardless, so the server both starts and stops promptly here.684 """685 global server686 MCP_GRANDCHILD_SERVER = os.path.join(FIXTURES_DIR, "mcp_grandchild_server.py")687 mcp_json = _mcp_config_json({688 "gc": {689 "command": sys.executable,690 "args": [MCP_GRANDCHILD_SERVER],691 }692 })693 694 # If warmup teardown deadlocked, the server would never become ready and start() would time out.695 server = _start_server_with_mcp(mcp_json)696 697 try:698 # invoking the tool spawns a live transport whose reader thread holds the inherited pipe699 res = server.make_request("POST", "/tools", data={700 "tool": "gc_echo",701 "params": {"message": "hello"}702 })703 assert res.status_code == 200, res.body704 assert "error" not in res.body, res.body705 706 # shutdown must be prompt: a deadlocked reader-join would stall until the 5s SIGKILL707 # fallback in ServerProcess.stop(), so the threshold has to stay below that708 start = time.time()709 server.stop()710 elapsed = time.time() - start711 assert elapsed < 3, f"server shutdown took {elapsed:.1f}s (expected < 3s) — teardown likely deadlocked"712 finally:713 server.stop()714