Felipe97/llama-cpp-compiled
01.2k
1#!/usr/bin/env python32"""3Test parallel tool-calling capability via chat completions endpoint.4 5Only run this against models that actually support parallel tool calls — this6script does not attempt to toggle that setting on the server. Each scenario is7explicitly worded so that a capable model SHOULD emit multiple tool calls in a8single assistant turn (either the same tool N times, or several different9tools at once).10 11Each test case contains:12 - tools: list of tool definitions (OpenAI-compatible)13 - messages: initial conversation messages14 - mock_tool_responses: dict mapping tool_name -> callable(arguments) -> str (JSON)15 - expected_parallel: dict describing what constitutes a successful parallel turn16 {"min_parallel": int, # minimum tool_calls in one turn17 "require_same_tool": Optional[str], # all parallel calls must be this tool18 "require_distinct_tools": Optional[int], # >= N distinct tool names in one turn19 "min_distinct_args_key": Optional[str]} # parallel calls must span this20 # many distinct values of this arg key21 - validate: callable(turns, all_tool_calls, final_content) -> (passed, reason)22"""23 24import argparse25import json26import requests27import sys28 29# ---------------------------------------------------------------------------30# Color / formatting helpers31# ---------------------------------------------------------------------------32 33RESET = "\x1b[0m"34BOLD = "\x1b[1m"35DIM = "\x1b[2m"36CYAN = "\x1b[36m"37YELLOW = "\x1b[33m"38GREEN = "\x1b[32m"39RED = "\x1b[31m"40BLUE = "\x1b[34m"41WHITE = "\x1b[97m"42MAGENTA = "\x1b[35m"43 44 45def _print(text="", end="\n"):46 sys.stdout.write(text + end)47 sys.stdout.flush()48 49 50def print_header(title):51 bar = "─" * 6052 _print(f"\n{BOLD}{CYAN}┌{bar}┐{RESET}")53 _print(54 f"{BOLD}{CYAN}│ {WHITE}{title}{CYAN}{' ' * max(0, 58 - len(title))}│{RESET}"55 )56 _print(f"{BOLD}{CYAN}└{bar}┘{RESET}")57 58 59def print_turn_banner(turn_idx, n_calls):60 color = MAGENTA if n_calls >= 2 else DIM61 _print(f"\n {BOLD}{color}▶ turn {turn_idx} — {n_calls} tool call(s){RESET}")62 63 64def print_tool_call(name, args):65 args_str = json.dumps(args)66 _print(67 f" {BOLD}{YELLOW}⚙ {name}{RESET}{DIM}({args_str}){RESET}"68 )69 70 71def print_tool_result(result):72 preview = result[:140] + ("…" if len(result) > 140 else "")73 _print(f" {DIM}{BLUE}↳ {preview}{RESET}")74 75 76def print_model_output(text):77 sys.stdout.write(text)78 sys.stdout.flush()79 80 81def print_pass(reason):82 _print(f"\n{BOLD}{GREEN}✔ PASS{RESET} {reason}")83 84 85def print_fail(reason):86 _print(f"\n{BOLD}{RED}✘ FAIL{RESET} {reason}")87 88 89def print_info(msg):90 _print(f"{DIM}{msg}{RESET}")91 92 93def print_warn(msg):94 _print(f"{BOLD}{YELLOW}⚠ {msg}{RESET}")95 96 97# ---------------------------------------------------------------------------98# HTTP helpers99# ---------------------------------------------------------------------------100 101 102def chat_completion(url, messages, tools=None, stream=False):103 payload = {104 "messages": messages,105 "stream": stream,106 "max_tokens": 4096,107 }108 if tools:109 payload["tools"] = tools110 payload["tool_choice"] = "auto"111 112 try:113 response = requests.post(url, json=payload, stream=stream)114 response.raise_for_status()115 except requests.exceptions.RequestException as e:116 body = e.response.content if (e.response is not None) else b""117 print_fail(f"Request error: {e} | body: {body}")118 return None119 120 full_content = ""121 reasoning_content = ""122 tool_calls: list[dict] = []123 124 if stream:125 for line in response.iter_lines():126 if not line:127 continue128 decoded = line.decode("utf-8")129 if not decoded.startswith("data: "):130 continue131 data_str = decoded[6:]132 if data_str == "[DONE]":133 break134 try:135 data = json.loads(data_str)136 except json.JSONDecodeError:137 continue138 choices = data.get("choices", [])139 if not choices:140 continue141 delta = choices[0].get("delta", {})142 if delta.get("reasoning_content"):143 reasoning_content += delta["reasoning_content"]144 if delta.get("content"):145 full_content += delta["content"]146 print_model_output(delta["content"])147 for tc in delta.get("tool_calls", []):148 idx = tc.get("index", 0)149 while len(tool_calls) <= idx:150 tool_calls.append(151 {152 "id": "",153 "type": "function",154 "function": {"name": "", "arguments": ""},155 }156 )157 if "id" in tc:158 tool_calls[idx]["id"] += tc["id"]159 if "function" in tc:160 if "name" in tc["function"]:161 tool_calls[idx]["function"]["name"] += tc["function"]["name"]162 if "arguments" in tc["function"]:163 tool_calls[idx]["function"]["arguments"] += tc["function"][164 "arguments"165 ]166 else:167 data = response.json()168 choices = data.get("choices", [])169 if choices:170 msg = choices[0].get("message", {})171 full_content = msg.get("content") or ""172 reasoning_content = msg.get("reasoning_content") or ""173 tool_calls = msg.get("tool_calls") or []174 if full_content:175 print_model_output(full_content)176 177 result = {"content": full_content, "tool_calls": tool_calls}178 if reasoning_content:179 result["reasoning_content"] = reasoning_content180 return result181 182 183def run_agentic_loop(url, messages, tools, mock_tool_responses, stream, max_turns=6):184 """185 Drive the multi-turn tool-call loop, but record each turn's tool calls186 separately so parallelism can be validated.187 188 Returns (turns, all_tool_calls, final_content) where `turns` is a list189 of dicts: {"index": int, "tool_calls": [...], "content": str}.190 """191 msgs = list(messages)192 turns: list[dict] = []193 all_tool_calls: list[dict] = []194 195 for turn_idx in range(max_turns):196 result = chat_completion(url, msgs, tools=tools, stream=stream)197 if result is None:198 return turns, all_tool_calls, None199 200 tcs = result.get("tool_calls") or []201 content = result.get("content") or ""202 203 turns.append(204 {"index": turn_idx, "tool_calls": list(tcs), "content": content}205 )206 207 if not tcs:208 if content:209 _print(f"\n{DIM}{'·' * 60}{RESET}")210 _print(f"{DIM} model response:{RESET}\n")211 return turns, all_tool_calls, content212 213 print_turn_banner(turn_idx, len(tcs))214 all_tool_calls.extend(tcs)215 216 assistant_msg: dict = {217 "role": "assistant",218 "content": content,219 "tool_calls": tcs,220 }221 reasoning = result.get("reasoning_content")222 if reasoning:223 assistant_msg["reasoning_content"] = reasoning224 msgs.append(assistant_msg)225 226 for tc in tcs:227 tool_name = tc["function"]["name"]228 try:229 args = json.loads(tc["function"]["arguments"])230 except json.JSONDecodeError:231 args = {}232 233 print_tool_call(tool_name, args)234 235 mock_fn = mock_tool_responses.get(tool_name)236 if mock_fn:237 tool_result = mock_fn(args)238 else:239 tool_result = json.dumps({"error": f"Unknown tool: {tool_name}"})240 241 print_tool_result(tool_result)242 243 msgs.append(244 {245 "role": "tool",246 "tool_call_id": tc.get("id", ""),247 "content": tool_result,248 }249 )250 251 return turns, all_tool_calls, None252 253 254# ---------------------------------------------------------------------------255# Parallelism helpers256# ---------------------------------------------------------------------------257 258 259def _best_parallel_turn(turns):260 """Return the turn (dict) with the most tool calls, or None if no tools."""261 tool_turns = [t for t in turns if t["tool_calls"]]262 if not tool_turns:263 return None264 return max(tool_turns, key=lambda t: len(t["tool_calls"]))265 266 267def _distinct_tool_names(turn):268 return {tc["function"]["name"] for tc in turn["tool_calls"]}269 270 271def _distinct_arg_values(turn, key):272 values = set()273 for tc in turn["tool_calls"]:274 try:275 args = json.loads(tc["function"]["arguments"])276 except json.JSONDecodeError:277 continue278 v = args.get(key)279 if v is not None:280 if isinstance(v, str):281 values.add(v.strip().lower())282 else:283 values.add(v)284 return values285 286 287def _check_parallel(turns, expected):288 """289 Check that at least one turn satisfies the parallel-call expectations.290 Returns (ok, reason).291 """292 best = _best_parallel_turn(turns)293 if best is None:294 return False, "No tool calls were made at all"295 296 min_parallel = expected.get("min_parallel", 2)297 if len(best["tool_calls"]) < min_parallel:298 by_turn = [len(t["tool_calls"]) for t in turns]299 return False, (300 f"No turn had >= {min_parallel} parallel tool calls "301 f"(per-turn counts: {by_turn})"302 )303 304 require_same = expected.get("require_same_tool")305 if require_same is not None:306 names = [tc["function"]["name"] for tc in best["tool_calls"]]307 if any(n != require_same for n in names):308 return False, (309 f"Parallel turn mixed tools; expected all {require_same!r}, got {names}"310 )311 312 require_distinct = expected.get("require_distinct_tools")313 if require_distinct is not None:314 distinct = _distinct_tool_names(best)315 if len(distinct) < require_distinct:316 return False, (317 f"Parallel turn had only {len(distinct)} distinct tool names "318 f"({distinct}); need >= {require_distinct}"319 )320 321 distinct_key = expected.get("min_distinct_args_key")322 distinct_count = expected.get("min_distinct_args_count", min_parallel)323 if distinct_key is not None:324 values = _distinct_arg_values(best, distinct_key)325 if len(values) < distinct_count:326 return False, (327 f"Parallel turn had only {len(values)} distinct {distinct_key!r} "328 f"values ({values}); need >= {distinct_count}"329 )330 331 return True, (332 f"Parallel turn had {len(best['tool_calls'])} calls across "333 f"{len(_distinct_tool_names(best))} distinct tool(s)"334 )335 336 337# ---------------------------------------------------------------------------338# Test case runner339# ---------------------------------------------------------------------------340 341 342def run_test(url, test_case, stream):343 name = test_case["name"]344 mode = f"{'stream' if stream else 'non-stream'}"345 print_header(f"{name} [{mode}]")346 347 turns, all_tool_calls, final_content = run_agentic_loop(348 url,349 messages=test_case["messages"],350 tools=test_case["tools"],351 mock_tool_responses=test_case["mock_tool_responses"],352 stream=stream,353 )354 355 if not turns:356 print_fail("No response from server.")357 return False358 359 parallel_ok, parallel_reason = _check_parallel(turns, test_case["expected_parallel"])360 if not parallel_ok:361 print_fail(parallel_reason)362 return False363 364 passed, reason = test_case["validate"](turns, all_tool_calls, final_content)365 if passed:366 print_pass(f"{parallel_reason}; {reason}")367 else:368 print_fail(reason)369 return passed370 371 372# ---------------------------------------------------------------------------373# Test case definitions374# ---------------------------------------------------------------------------375 376# ---- Test 1: Multi-file read (same tool, multiple distinct paths) ----377 378_FILE_TOOLS = [379 {380 "type": "function",381 "function": {382 "name": "read_file",383 "description": (384 "Read the full contents of a file from the local filesystem. "385 "Call this tool in parallel when asked to read several files — "386 "each path needs its own call."387 ),388 "parameters": {389 "type": "object",390 "properties": {391 "path": {392 "type": "string",393 "description": "Absolute or repo-relative path to a file",394 },395 },396 "required": ["path"],397 },398 },399 },400]401 402_FILE_CONTENTS = {403 "config/database.yml": "host: db.internal\nport: 5432\nuser: svc_app\n",404 "config/redis.yml": "host: cache.internal\nport: 6379\ndb: 0\n",405 "config/queue.yml": "broker: rabbitmq.internal\nport: 5672\nvhost: prod\n",406 "config/auth.yml": "provider: oidc\nissuer: https://auth.internal\n",407}408 409 410def _read_file_mock(args):411 path = args.get("path", "")412 norm = path.lstrip("./").lstrip("/")413 content = _FILE_CONTENTS.get(norm)414 if content is None:415 for k, v in _FILE_CONTENTS.items():416 if path.endswith(k):417 content = v418 break419 if content is None:420 return json.dumps({"path": path, "error": "not found"})421 return json.dumps({"path": path, "content": content})422 423 424MULTIFILE_READ_TEST = {425 "name": "Parallel multi-file read (same tool, 4 distinct paths)",426 "tools": _FILE_TOOLS,427 "messages": [428 {429 "role": "user",430 "content": (431 "Please read all four of these config files so I can review them "432 "together: config/database.yml, config/redis.yml, config/queue.yml, "433 "and config/auth.yml. Call read_file for every path in parallel in "434 "a single batch — do NOT read them one by one sequentially across "435 "turns. After you have all four, give me a one-line summary of each."436 ),437 }438 ],439 "mock_tool_responses": {"read_file": _read_file_mock},440 "expected_parallel": {441 "min_parallel": 4,442 "require_same_tool": "read_file",443 "min_distinct_args_key": "path",444 "min_distinct_args_count": 4,445 },446 "validate": lambda turns, tcs, content: _validate_multifile(turns, tcs, content),447}448 449 450def _validate_multifile(turns, tcs, content):451 del turns452 if not content:453 return False, "No final summary produced"454 return True, f"{len(tcs)} total read_file calls; content length={len(content)}"455 456 457# ---- Test 2: Batch TODO marking (same tool, N calls in one turn) ----458 459_TODO_TOOLS = [460 {461 "type": "function",462 "function": {463 "name": "mark_todo_complete",464 "description": (465 "Mark a single TODO item as complete by ID. When the user wants "466 "several items marked at once, call this tool in parallel — "467 "one call per item — rather than sequentially across turns."468 ),469 "parameters": {470 "type": "object",471 "properties": {472 "todo_id": {473 "type": "string",474 "description": "Identifier of the TODO item",475 },476 "note": {477 "type": "string",478 "description": "Optional completion note",479 },480 },481 "required": ["todo_id"],482 },483 },484 },485]486 487_TODO_DB = {488 "T-101": "Draft onboarding doc",489 "T-102": "Update dependency lockfile",490 "T-103": "Fix flaky login test",491 "T-104": "Rotate service credentials",492 "T-105": "Archive Q4 reports",493}494 495 496def _mark_todo_mock(args):497 tid = args.get("todo_id", "")498 if tid in _TODO_DB:499 return json.dumps({"todo_id": tid, "title": _TODO_DB[tid], "status": "done"})500 return json.dumps({"todo_id": tid, "error": "unknown id"})501 502 503TODO_BATCH_TEST = {504 "name": "Batch TODO completion (same tool, 5 IDs in one turn)",505 "tools": _TODO_TOOLS,506 "messages": [507 {508 "role": "user",509 "content": (510 "I finished every item on today's list. Please mark all of the "511 "following TODOs as complete, in one parallel batch: T-101, T-102, "512 "T-103, T-104, T-105. Don't mark them one at a time across separate "513 "turns — issue all five mark_todo_complete calls at once. Afterwards "514 "confirm which ones succeeded."515 ),516 }517 ],518 "mock_tool_responses": {"mark_todo_complete": _mark_todo_mock},519 "expected_parallel": {520 "min_parallel": 5,521 "require_same_tool": "mark_todo_complete",522 "min_distinct_args_key": "todo_id",523 "min_distinct_args_count": 5,524 },525 "validate": lambda turns, tcs, content: _validate_todo(turns, tcs, content),526}527 528 529def _validate_todo(turns, tcs, content):530 del turns531 if not content:532 return False, "No confirmation summary produced"533 return True, f"{len(tcs)} total mark_todo_complete calls"534 535 536# ---- Test 3: Multi-city weather (same tool, N parallel locations) ----537 538_WEATHER_TOOLS = [539 {540 "type": "function",541 "function": {542 "name": "get_weather",543 "description": (544 "Fetch current weather for ONE city. When the user asks about "545 "several cities, call this tool in parallel — one call per city — "546 "instead of sequentially."547 ),548 "parameters": {549 "type": "object",550 "properties": {551 "city": {"type": "string", "description": "City name"},552 "units": {553 "type": "string",554 "enum": ["metric", "imperial"],555 "default": "metric",556 },557 },558 "required": ["city"],559 },560 },561 },562]563 564_WEATHER_DB = {565 "tokyo": {"city": "Tokyo", "temp_c": 18.4, "condition": "partly cloudy", "humidity": 64},566 "london": {"city": "London", "temp_c": 9.1, "condition": "overcast", "humidity": 81},567 "new york": {"city": "New York", "temp_c": 12.7, "condition": "clear", "humidity": 55},568 "paris": {"city": "Paris", "temp_c": 11.3, "condition": "light rain", "humidity": 78},569}570 571 572def _weather_mock(args):573 city = args.get("city", "").strip().lower()574 if city.startswith("new york"):575 city = "new york"576 if city in _WEATHER_DB:577 return json.dumps(_WEATHER_DB[city])578 return json.dumps({"city": args.get("city", ""), "error": "unknown city"})579 580 581MULTI_WEATHER_TEST = {582 "name": "Parallel multi-city weather (same tool, 4 cities)",583 "tools": _WEATHER_TOOLS,584 "messages": [585 {586 "role": "user",587 "content": (588 "I'm comparing today's weather across four cities for a travel "589 "decision: Tokyo, London, New York, and Paris. Please call "590 "get_weather for all four in parallel in a single turn — don't "591 "fetch them one at a time. Then rank them from warmest to coolest."592 ),593 }594 ],595 "mock_tool_responses": {"get_weather": _weather_mock},596 "expected_parallel": {597 "min_parallel": 4,598 "require_same_tool": "get_weather",599 "min_distinct_args_key": "city",600 "min_distinct_args_count": 4,601 },602 "validate": lambda turns, tcs, content: _validate_weather(turns, tcs, content),603}604 605 606def _validate_weather(turns, tcs, content):607 del turns608 if not content or not any(609 kw in content.lower() for kw in ("warmest", "rank", "hot", "cool")610 ):611 return False, f"Final content missing a ranking: {content!r}"612 return True, f"{len(tcs)} total get_weather calls; ranking produced"613 614 615# ---- Test 4: Trip planning (different tools, parallel in one turn) ----616 617_TRIP_TOOLS = [618 {619 "type": "function",620 "function": {621 "name": "search_flights",622 "description": "Search one-way flights between two airports on a given date.",623 "parameters": {624 "type": "object",625 "properties": {626 "from_airport": {"type": "string", "description": "IATA code, e.g. SFO"},627 "to_airport": {"type": "string", "description": "IATA code, e.g. JFK"},628 "date": {"type": "string", "description": "YYYY-MM-DD"},629 },630 "required": ["from_airport", "to_airport", "date"],631 },632 },633 },634 {635 "type": "function",636 "function": {637 "name": "search_hotels",638 "description": "Search hotels in a city for a date range.",639 "parameters": {640 "type": "object",641 "properties": {642 "city": {"type": "string"},643 "check_in": {"type": "string", "description": "YYYY-MM-DD"},644 "check_out": {"type": "string", "description": "YYYY-MM-DD"},645 "max_price": {"type": "integer"},646 },647 "required": ["city", "check_in", "check_out"],648 },649 },650 },651 {652 "type": "function",653 "function": {654 "name": "search_restaurants",655 "description": "Search restaurants in a city by cuisine.",656 "parameters": {657 "type": "object",658 "properties": {659 "city": {"type": "string"},660 "cuisine": {"type": "string"},661 },662 "required": ["city"],663 },664 },665 },666]667 668_FLIGHTS_RESULT = {669 "results": [670 {"flight": "UA 1552", "depart": "08:15", "arrive": "16:45", "price": 389},671 {"flight": "AA 20", "depart": "10:00", "arrive": "18:35", "price": 412},672 ]673}674_HOTELS_RESULT = {675 "results": [676 {"name": "Midtown Grand", "nightly_rate": 245, "rating": 4.3},677 {"name": "Harbour Boutique", "nightly_rate": 312, "rating": 4.6},678 ]679}680_RESTAURANTS_RESULT = {681 "results": [682 {"name": "Trattoria Nona", "cuisine": "italian", "rating": 4.5},683 {"name": "Osteria Blu", "cuisine": "italian", "rating": 4.4},684 ]685}686 687TRIP_PLAN_TEST = {688 "name": "Trip planning (3 different tools in parallel)",689 "tools": _TRIP_TOOLS,690 "messages": [691 {692 "role": "user",693 "content": (694 "I'm flying from SFO to JFK on 2026-06-12 and staying four nights "695 "(check out 2026-06-16). I'd also like some Italian restaurant "696 "suggestions in New York. Please call search_flights, search_hotels, "697 "and search_restaurants in parallel — all three in a single turn, "698 "since they don't depend on each other. Then give me a concise "699 "travel summary."700 ),701 }702 ],703 "mock_tool_responses": {704 "search_flights": lambda _: json.dumps(_FLIGHTS_RESULT),705 "search_hotels": lambda _: json.dumps(_HOTELS_RESULT),706 "search_restaurants": lambda _: json.dumps(_RESTAURANTS_RESULT),707 },708 "expected_parallel": {709 "min_parallel": 3,710 "require_distinct_tools": 3,711 },712 "validate": lambda turns, tcs, content: _validate_trip(turns, tcs, content),713}714 715 716def _validate_trip(turns, tcs, content):717 del turns718 names = {tc["function"]["name"] for tc in tcs}719 required = {"search_flights", "search_hotels", "search_restaurants"}720 missing = required - names721 if missing:722 return False, f"Missing tool calls: {missing}"723 if not content:724 return False, "No travel summary produced"725 return True, f"All three tools called; summary length={len(content)}"726 727 728# ---- Test 5: Portfolio check (same tool, parallel tickers) ----729 730_STOCK_TOOLS = [731 {732 "type": "function",733 "function": {734 "name": "get_stock_quote",735 "description": (736 "Get the latest quote for ONE ticker. When the user asks about "737 "multiple tickers, call this tool in parallel — one per symbol — "738 "rather than sequentially."739 ),740 "parameters": {741 "type": "object",742 "properties": {743 "symbol": {"type": "string", "description": "Ticker symbol"},744 },745 "required": ["symbol"],746 },747 },748 },749]750 751_STOCK_DB = {752 "AAPL": {"symbol": "AAPL", "price": 218.45, "change_pct": "+0.8%"},753 "MSFT": {"symbol": "MSFT", "price": 421.10, "change_pct": "+1.2%"},754 "GOOGL":{"symbol": "GOOGL","price": 175.22, "change_pct": "-0.3%"},755 "AMZN": {"symbol": "AMZN", "price": 189.76, "change_pct": "+0.5%"},756 "NVDA": {"symbol": "NVDA", "price": 140.88, "change_pct": "+2.4%"},757}758 759 760def _stock_mock(args):761 sym = args.get("symbol", "").strip().upper()762 if sym in _STOCK_DB:763 return json.dumps(_STOCK_DB[sym])764 return json.dumps({"symbol": sym, "error": "unknown ticker"})765 766 767PORTFOLIO_TEST = {768 "name": "Portfolio check (same tool, 5 tickers in parallel)",769 "tools": _STOCK_TOOLS,770 "messages": [771 {772 "role": "user",773 "content": (774 "Pull the latest quote for every ticker in my portfolio — AAPL, "775 "MSFT, GOOGL, AMZN, and NVDA — in a single parallel batch. These "776 "lookups are independent, so please don't chain them across turns. "777 "Once you have all five, tell me which ticker had the biggest "778 "percentage change today."779 ),780 }781 ],782 "mock_tool_responses": {"get_stock_quote": _stock_mock},783 "expected_parallel": {784 "min_parallel": 5,785 "require_same_tool": "get_stock_quote",786 "min_distinct_args_key": "symbol",787 "min_distinct_args_count": 5,788 },789 "validate": lambda turns, tcs, content: _validate_portfolio(turns, tcs, content),790}791 792 793def _validate_portfolio(turns, tcs, content):794 del turns795 if not content or ("nvda" not in content.lower() and "NVDA" not in content):796 return False, f"Expected NVDA to be identified as the biggest mover: {content!r}"797 return True, f"{len(tcs)} total quotes pulled"798 799 800# ---- Test 6: Mixed — translate + dictionary in parallel for the same word ----801 802_LANG_TOOLS = [803 {804 "type": "function",805 "function": {806 "name": "translate_text",807 "description": "Translate a short text into a target language.",808 "parameters": {809 "type": "object",810 "properties": {811 "text": {"type": "string"},812 "target_language": {"type": "string",813 "description": "ISO 639-1 language code, e.g. 'es'"},814 },815 "required": ["text", "target_language"],816 },817 },818 },819 {820 "type": "function",821 "function": {822 "name": "get_definition",823 "description": "Get the English dictionary definition of a word.",824 "parameters": {825 "type": "object",826 "properties": {827 "word": {"type": "string"},828 },829 "required": ["word"],830 },831 },832 },833 {834 "type": "function",835 "function": {836 "name": "get_synonyms",837 "description": "Get English synonyms for a word.",838 "parameters": {839 "type": "object",840 "properties": {841 "word": {"type": "string"},842 },843 "required": ["word"],844 },845 },846 },847]848 849 850def _translate_mock(args):851 t = args.get("text", "")852 lang = args.get("target_language", "")853 return json.dumps({"source": t, "target_language": lang, "translation": f"[{lang}] {t}"})854 855 856def _definition_mock(args):857 w = args.get("word", "")858 return json.dumps({859 "word": w,860 "definition": f"A standard dictionary definition of {w!r}.",861 })862 863 864def _synonyms_mock(args):865 w = args.get("word", "")866 return json.dumps({867 "word": w,868 "synonyms": ["synonym_a", "synonym_b", "synonym_c"],869 })870 871 872LANG_TOOLKIT_TEST = {873 "name": "Language toolkit (translate + definition + synonyms in parallel)",874 "tools": _LANG_TOOLS,875 "messages": [876 {877 "role": "user",878 "content": (879 "For the English word 'resilient', I need three independent "880 "look-ups at once: (a) translate it into Spanish, (b) fetch its "881 "dictionary definition, and (c) list its synonyms. These three "882 "calls don't depend on each other — please issue them in parallel "883 "in a single turn. Then present the combined results as a short "884 "language note."885 ),886 }887 ],888 "mock_tool_responses": {889 "translate_text": _translate_mock,890 "get_definition": _definition_mock,891 "get_synonyms": _synonyms_mock,892 },893 "expected_parallel": {894 "min_parallel": 3,895 "require_distinct_tools": 3,896 },897 "validate": lambda turns, tcs, content: _validate_lang(turns, tcs, content),898}899 900 901def _validate_lang(turns, tcs, content):902 del turns903 names = {tc["function"]["name"] for tc in tcs}904 required = {"translate_text", "get_definition", "get_synonyms"}905 missing = required - names906 if missing:907 return False, f"Missing tool calls: {missing}"908 if not content:909 return False, "No language note produced"910 return True, f"All three lookup tools called; note length={len(content)}"911 912 913# ---------------------------------------------------------------------------914# All test cases915# ---------------------------------------------------------------------------916 917ALL_TEST_CASES = [918 MULTIFILE_READ_TEST,919 TODO_BATCH_TEST,920 MULTI_WEATHER_TEST,921 TRIP_PLAN_TEST,922 PORTFOLIO_TEST,923 LANG_TOOLKIT_TEST,924]925 926 927# ---------------------------------------------------------------------------928# Entry point929# ---------------------------------------------------------------------------930 931 932def main():933 parser = argparse.ArgumentParser(934 description=(935 "Test llama-server parallel tool-calling capability. Run this only "936 "against models configured for parallel tool calls — this script "937 "does not configure that itself."938 )939 )940 parser.add_argument("--host", default="localhost")941 parser.add_argument("--port", default=8080, type=int)942 parser.add_argument(943 "--no-stream", action="store_true", help="Disable streaming mode tests"944 )945 parser.add_argument(946 "--stream-only", action="store_true", help="Only run streaming mode tests"947 )948 parser.add_argument(949 "--test",950 help="Run only the test whose name contains this substring (case-insensitive)",951 )952 args = parser.parse_args()953 954 url = f"http://{args.host}:{args.port}/v1/chat/completions"955 print_info(f"Testing server at {url}")956 print_warn(957 "This script expects the target model to emit multiple tool calls in a "958 "single assistant turn. Run it only against parallel-tool-capable models."959 )960 961 modes: list[bool] = []962 if not args.stream_only:963 modes.append(False)964 if not args.no_stream:965 modes.append(True)966 967 cases: list[dict] = ALL_TEST_CASES968 if args.test:969 name_filter = args.test.lower()970 cases = [c for c in cases if name_filter in str(c["name"]).lower()]971 if not cases:972 print_fail(f"No test cases matched '{args.test}'")973 sys.exit(1)974 975 total = 0976 passed = 0977 for stream in modes:978 for case in cases:979 total += 1980 if run_test(url, case, stream=stream):981 passed += 1982 983 color = GREEN if passed == total else RED984 _print(f"\n{BOLD}{color}{'─' * 60}{RESET}")985 _print(f"{BOLD}{color} Results: {passed}/{total} passed{RESET}")986 _print(f"{BOLD}{color}{'─' * 60}{RESET}\n")987 sys.exit(0 if passed == total else 1)988 989 990if __name__ == "__main__":991 main()992