Brunobkr/llama.cpp_AlgMor24_github

1#!/usr/bin/env python32"""3Test parallel tool-calling capability via chat completions endpoint.4 5Only run this against models that actually support parallel tool calls — this6script does not attempt to toggle that setting on the server. Each scenario is7explicitly worded so that a capable model SHOULD emit multiple tool calls in a8single assistant turn (either the same tool N times, or several different9tools at once).10 11Each test case contains:12  - tools: list of tool definitions (OpenAI-compatible)13  - messages: initial conversation messages14  - mock_tool_responses: dict mapping tool_name -> callable(arguments) -> str (JSON)15  - expected_parallel: dict describing what constitutes a successful parallel turn16        {"min_parallel": int,                # minimum tool_calls in one turn17         "require_same_tool": Optional[str], # all parallel calls must be this tool18         "require_distinct_tools": Optional[int], # >= N distinct tool names in one turn19         "min_distinct_args_key": Optional[str]}  # parallel calls must span this20                                                   # many distinct values of this arg key21  - validate: callable(turns, all_tool_calls, final_content) -> (passed, reason)22"""23 24import argparse25import json26import requests27import sys28 29# ---------------------------------------------------------------------------30# Color / formatting helpers31# ---------------------------------------------------------------------------32 33RESET = "\x1b[0m"34BOLD = "\x1b[1m"35DIM = "\x1b[2m"36CYAN = "\x1b[36m"37YELLOW = "\x1b[33m"38GREEN = "\x1b[32m"39RED = "\x1b[31m"40BLUE = "\x1b[34m"41WHITE = "\x1b[97m"42MAGENTA = "\x1b[35m"43 44 45def _print(text="", end="\n"):46    sys.stdout.write(text + end)47    sys.stdout.flush()48 49 50def print_header(title):51    bar = "─" * 6052    _print(f"\n{BOLD}{CYAN}┌{bar}┐{RESET}")53    _print(54        f"{BOLD}{CYAN}│  {WHITE}{title}{CYAN}{' ' * max(0, 58 - len(title))}│{RESET}"55    )56    _print(f"{BOLD}{CYAN}└{bar}┘{RESET}")57 58 59def print_turn_banner(turn_idx, n_calls):60    color = MAGENTA if n_calls >= 2 else DIM61    _print(f"\n  {BOLD}{color}▶ turn {turn_idx} — {n_calls} tool call(s){RESET}")62 63 64def print_tool_call(name, args):65    args_str = json.dumps(args)66    _print(67        f"    {BOLD}{YELLOW}⚙ {name}{RESET}{DIM}({args_str}){RESET}"68    )69 70 71def print_tool_result(result):72    preview = result[:140] + ("…" if len(result) > 140 else "")73    _print(f"      {DIM}{BLUE}↳ {preview}{RESET}")74 75 76def print_model_output(text):77    sys.stdout.write(text)78    sys.stdout.flush()79 80 81def print_pass(reason):82    _print(f"\n{BOLD}{GREEN}✔ PASS{RESET}  {reason}")83 84 85def print_fail(reason):86    _print(f"\n{BOLD}{RED}✘ FAIL{RESET}  {reason}")87 88 89def print_info(msg):90    _print(f"{DIM}{msg}{RESET}")91 92 93def print_warn(msg):94    _print(f"{BOLD}{YELLOW}⚠ {msg}{RESET}")95 96 97# ---------------------------------------------------------------------------98# HTTP helpers99# ---------------------------------------------------------------------------100 101 102def chat_completion(url, messages, tools=None, stream=False):103    payload = {104        "messages": messages,105        "stream": stream,106        "max_tokens": 4096,107    }108    if tools:109        payload["tools"] = tools110        payload["tool_choice"] = "auto"111 112    try:113        response = requests.post(url, json=payload, stream=stream)114        response.raise_for_status()115    except requests.exceptions.RequestException as e:116        body = e.response.content if (e.response is not None) else b""117        print_fail(f"Request error: {e} | body: {body}")118        return None119 120    full_content = ""121    reasoning_content = ""122    tool_calls: list[dict] = []123 124    if stream:125        for line in response.iter_lines():126            if not line:127                continue128            decoded = line.decode("utf-8")129            if not decoded.startswith("data: "):130                continue131            data_str = decoded[6:]132            if data_str == "[DONE]":133                break134            try:135                data = json.loads(data_str)136            except json.JSONDecodeError:137                continue138            choices = data.get("choices", [])139            if not choices:140                continue141            delta = choices[0].get("delta", {})142            if delta.get("reasoning_content"):143                reasoning_content += delta["reasoning_content"]144            if delta.get("content"):145                full_content += delta["content"]146                print_model_output(delta["content"])147            for tc in delta.get("tool_calls", []):148                idx = tc.get("index", 0)149                while len(tool_calls) <= idx:150                    tool_calls.append(151                        {152                            "id": "",153                            "type": "function",154                            "function": {"name": "", "arguments": ""},155                        }156                    )157                if "id" in tc:158                    tool_calls[idx]["id"] += tc["id"]159                if "function" in tc:160                    if "name" in tc["function"]:161                        tool_calls[idx]["function"]["name"] += tc["function"]["name"]162                    if "arguments" in tc["function"]:163                        tool_calls[idx]["function"]["arguments"] += tc["function"][164                            "arguments"165                        ]166    else:167        data = response.json()168        choices = data.get("choices", [])169        if choices:170            msg = choices[0].get("message", {})171            full_content = msg.get("content") or ""172            reasoning_content = msg.get("reasoning_content") or ""173            tool_calls = msg.get("tool_calls") or []174            if full_content:175                print_model_output(full_content)176 177    result = {"content": full_content, "tool_calls": tool_calls}178    if reasoning_content:179        result["reasoning_content"] = reasoning_content180    return result181 182 183def run_agentic_loop(url, messages, tools, mock_tool_responses, stream, max_turns=6):184    """185    Drive the multi-turn tool-call loop, but record each turn's tool calls186    separately so parallelism can be validated.187 188    Returns (turns, all_tool_calls, final_content) where `turns` is a list189    of dicts: {"index": int, "tool_calls": [...], "content": str}.190    """191    msgs = list(messages)192    turns: list[dict] = []193    all_tool_calls: list[dict] = []194 195    for turn_idx in range(max_turns):196        result = chat_completion(url, msgs, tools=tools, stream=stream)197        if result is None:198            return turns, all_tool_calls, None199 200        tcs = result.get("tool_calls") or []201        content = result.get("content") or ""202 203        turns.append(204            {"index": turn_idx, "tool_calls": list(tcs), "content": content}205        )206 207        if not tcs:208            if content:209                _print(f"\n{DIM}{'·' * 60}{RESET}")210                _print(f"{DIM}  model response:{RESET}\n")211            return turns, all_tool_calls, content212 213        print_turn_banner(turn_idx, len(tcs))214        all_tool_calls.extend(tcs)215 216        assistant_msg: dict = {217            "role": "assistant",218            "content": content,219            "tool_calls": tcs,220        }221        reasoning = result.get("reasoning_content")222        if reasoning:223            assistant_msg["reasoning_content"] = reasoning224        msgs.append(assistant_msg)225 226        for tc in tcs:227            tool_name = tc["function"]["name"]228            try:229                args = json.loads(tc["function"]["arguments"])230            except json.JSONDecodeError:231                args = {}232 233            print_tool_call(tool_name, args)234 235            mock_fn = mock_tool_responses.get(tool_name)236            if mock_fn:237                tool_result = mock_fn(args)238            else:239                tool_result = json.dumps({"error": f"Unknown tool: {tool_name}"})240 241            print_tool_result(tool_result)242 243            msgs.append(244                {245                    "role": "tool",246                    "tool_call_id": tc.get("id", ""),247                    "content": tool_result,248                }249            )250 251    return turns, all_tool_calls, None252 253 254# ---------------------------------------------------------------------------255# Parallelism helpers256# ---------------------------------------------------------------------------257 258 259def _best_parallel_turn(turns):260    """Return the turn (dict) with the most tool calls, or None if no tools."""261    tool_turns = [t for t in turns if t["tool_calls"]]262    if not tool_turns:263        return None264    return max(tool_turns, key=lambda t: len(t["tool_calls"]))265 266 267def _distinct_tool_names(turn):268    return {tc["function"]["name"] for tc in turn["tool_calls"]}269 270 271def _distinct_arg_values(turn, key):272    values = set()273    for tc in turn["tool_calls"]:274        try:275            args = json.loads(tc["function"]["arguments"])276        except json.JSONDecodeError:277            continue278        v = args.get(key)279        if v is not None:280            if isinstance(v, str):281                values.add(v.strip().lower())282            else:283                values.add(v)284    return values285 286 287def _check_parallel(turns, expected):288    """289    Check that at least one turn satisfies the parallel-call expectations.290    Returns (ok, reason).291    """292    best = _best_parallel_turn(turns)293    if best is None:294        return False, "No tool calls were made at all"295 296    min_parallel = expected.get("min_parallel", 2)297    if len(best["tool_calls"]) < min_parallel:298        by_turn = [len(t["tool_calls"]) for t in turns]299        return False, (300            f"No turn had >= {min_parallel} parallel tool calls "301            f"(per-turn counts: {by_turn})"302        )303 304    require_same = expected.get("require_same_tool")305    if require_same is not None:306        names = [tc["function"]["name"] for tc in best["tool_calls"]]307        if any(n != require_same for n in names):308            return False, (309                f"Parallel turn mixed tools; expected all {require_same!r}, got {names}"310            )311 312    require_distinct = expected.get("require_distinct_tools")313    if require_distinct is not None:314        distinct = _distinct_tool_names(best)315        if len(distinct) < require_distinct:316            return False, (317                f"Parallel turn had only {len(distinct)} distinct tool names "318                f"({distinct}); need >= {require_distinct}"319            )320 321    distinct_key = expected.get("min_distinct_args_key")322    distinct_count = expected.get("min_distinct_args_count", min_parallel)323    if distinct_key is not None:324        values = _distinct_arg_values(best, distinct_key)325        if len(values) < distinct_count:326            return False, (327                f"Parallel turn had only {len(values)} distinct {distinct_key!r} "328                f"values ({values}); need >= {distinct_count}"329            )330 331    return True, (332        f"Parallel turn had {len(best['tool_calls'])} calls across "333        f"{len(_distinct_tool_names(best))} distinct tool(s)"334    )335 336 337# ---------------------------------------------------------------------------338# Test case runner339# ---------------------------------------------------------------------------340 341 342def run_test(url, test_case, stream):343    name = test_case["name"]344    mode = f"{'stream' if stream else 'non-stream'}"345    print_header(f"{name}  [{mode}]")346 347    turns, all_tool_calls, final_content = run_agentic_loop(348        url,349        messages=test_case["messages"],350        tools=test_case["tools"],351        mock_tool_responses=test_case["mock_tool_responses"],352        stream=stream,353    )354 355    if not turns:356        print_fail("No response from server.")357        return False358 359    parallel_ok, parallel_reason = _check_parallel(turns, test_case["expected_parallel"])360    if not parallel_ok:361        print_fail(parallel_reason)362        return False363 364    passed, reason = test_case["validate"](turns, all_tool_calls, final_content)365    if passed:366        print_pass(f"{parallel_reason}; {reason}")367    else:368        print_fail(reason)369    return passed370 371 372# ---------------------------------------------------------------------------373# Test case definitions374# ---------------------------------------------------------------------------375 376# ---- Test 1: Multi-file read (same tool, multiple distinct paths) ----377 378_FILE_TOOLS = [379    {380        "type": "function",381        "function": {382            "name": "read_file",383            "description": (384                "Read the full contents of a file from the local filesystem. "385                "Call this tool in parallel when asked to read several files — "386                "each path needs its own call."387            ),388            "parameters": {389                "type": "object",390                "properties": {391                    "path": {392                        "type": "string",393                        "description": "Absolute or repo-relative path to a file",394                    },395                },396                "required": ["path"],397            },398        },399    },400]401 402_FILE_CONTENTS = {403    "config/database.yml": "host: db.internal\nport: 5432\nuser: svc_app\n",404    "config/redis.yml":    "host: cache.internal\nport: 6379\ndb: 0\n",405    "config/queue.yml":    "broker: rabbitmq.internal\nport: 5672\nvhost: prod\n",406    "config/auth.yml":     "provider: oidc\nissuer: https://auth.internal\n",407}408 409 410def _read_file_mock(args):411    path = args.get("path", "")412    norm = path.lstrip("./").lstrip("/")413    content = _FILE_CONTENTS.get(norm)414    if content is None:415        for k, v in _FILE_CONTENTS.items():416            if path.endswith(k):417                content = v418                break419    if content is None:420        return json.dumps({"path": path, "error": "not found"})421    return json.dumps({"path": path, "content": content})422 423 424MULTIFILE_READ_TEST = {425    "name": "Parallel multi-file read (same tool, 4 distinct paths)",426    "tools": _FILE_TOOLS,427    "messages": [428        {429            "role": "user",430            "content": (431                "Please read all four of these config files so I can review them "432                "together: config/database.yml, config/redis.yml, config/queue.yml, "433                "and config/auth.yml. Call read_file for every path in parallel in "434                "a single batch — do NOT read them one by one sequentially across "435                "turns. After you have all four, give me a one-line summary of each."436            ),437        }438    ],439    "mock_tool_responses": {"read_file": _read_file_mock},440    "expected_parallel": {441        "min_parallel": 4,442        "require_same_tool": "read_file",443        "min_distinct_args_key": "path",444        "min_distinct_args_count": 4,445    },446    "validate": lambda turns, tcs, content: _validate_multifile(turns, tcs, content),447}448 449 450def _validate_multifile(turns, tcs, content):451    del turns452    if not content:453        return False, "No final summary produced"454    return True, f"{len(tcs)} total read_file calls; content length={len(content)}"455 456 457# ---- Test 2: Batch TODO marking (same tool, N calls in one turn) ----458 459_TODO_TOOLS = [460    {461        "type": "function",462        "function": {463            "name": "mark_todo_complete",464            "description": (465                "Mark a single TODO item as complete by ID. When the user wants "466                "several items marked at once, call this tool in parallel — "467                "one call per item — rather than sequentially across turns."468            ),469            "parameters": {470                "type": "object",471                "properties": {472                    "todo_id": {473                        "type": "string",474                        "description": "Identifier of the TODO item",475                    },476                    "note": {477                        "type": "string",478                        "description": "Optional completion note",479                    },480                },481                "required": ["todo_id"],482            },483        },484    },485]486 487_TODO_DB = {488    "T-101": "Draft onboarding doc",489    "T-102": "Update dependency lockfile",490    "T-103": "Fix flaky login test",491    "T-104": "Rotate service credentials",492    "T-105": "Archive Q4 reports",493}494 495 496def _mark_todo_mock(args):497    tid = args.get("todo_id", "")498    if tid in _TODO_DB:499        return json.dumps({"todo_id": tid, "title": _TODO_DB[tid], "status": "done"})500    return json.dumps({"todo_id": tid, "error": "unknown id"})501 502 503TODO_BATCH_TEST = {504    "name": "Batch TODO completion (same tool, 5 IDs in one turn)",505    "tools": _TODO_TOOLS,506    "messages": [507        {508            "role": "user",509            "content": (510                "I finished every item on today's list. Please mark all of the "511                "following TODOs as complete, in one parallel batch: T-101, T-102, "512                "T-103, T-104, T-105. Don't mark them one at a time across separate "513                "turns — issue all five mark_todo_complete calls at once. Afterwards "514                "confirm which ones succeeded."515            ),516        }517    ],518    "mock_tool_responses": {"mark_todo_complete": _mark_todo_mock},519    "expected_parallel": {520        "min_parallel": 5,521        "require_same_tool": "mark_todo_complete",522        "min_distinct_args_key": "todo_id",523        "min_distinct_args_count": 5,524    },525    "validate": lambda turns, tcs, content: _validate_todo(turns, tcs, content),526}527 528 529def _validate_todo(turns, tcs, content):530    del turns531    if not content:532        return False, "No confirmation summary produced"533    return True, f"{len(tcs)} total mark_todo_complete calls"534 535 536# ---- Test 3: Multi-city weather (same tool, N parallel locations) ----537 538_WEATHER_TOOLS = [539    {540        "type": "function",541        "function": {542            "name": "get_weather",543            "description": (544                "Fetch current weather for ONE city. When the user asks about "545                "several cities, call this tool in parallel — one call per city — "546                "instead of sequentially."547            ),548            "parameters": {549                "type": "object",550                "properties": {551                    "city":  {"type": "string", "description": "City name"},552                    "units": {553                        "type": "string",554                        "enum": ["metric", "imperial"],555                        "default": "metric",556                    },557                },558                "required": ["city"],559            },560        },561    },562]563 564_WEATHER_DB = {565    "tokyo":  {"city": "Tokyo",  "temp_c": 18.4, "condition": "partly cloudy", "humidity": 64},566    "london": {"city": "London", "temp_c":  9.1, "condition": "overcast",       "humidity": 81},567    "new york": {"city": "New York", "temp_c": 12.7, "condition": "clear",      "humidity": 55},568    "paris":  {"city": "Paris",  "temp_c": 11.3, "condition": "light rain",     "humidity": 78},569}570 571 572def _weather_mock(args):573    city = args.get("city", "").strip().lower()574    if city.startswith("new york"):575        city = "new york"576    if city in _WEATHER_DB:577        return json.dumps(_WEATHER_DB[city])578    return json.dumps({"city": args.get("city", ""), "error": "unknown city"})579 580 581MULTI_WEATHER_TEST = {582    "name": "Parallel multi-city weather (same tool, 4 cities)",583    "tools": _WEATHER_TOOLS,584    "messages": [585        {586            "role": "user",587            "content": (588                "I'm comparing today's weather across four cities for a travel "589                "decision: Tokyo, London, New York, and Paris. Please call "590                "get_weather for all four in parallel in a single turn — don't "591                "fetch them one at a time. Then rank them from warmest to coolest."592            ),593        }594    ],595    "mock_tool_responses": {"get_weather": _weather_mock},596    "expected_parallel": {597        "min_parallel": 4,598        "require_same_tool": "get_weather",599        "min_distinct_args_key": "city",600        "min_distinct_args_count": 4,601    },602    "validate": lambda turns, tcs, content: _validate_weather(turns, tcs, content),603}604 605 606def _validate_weather(turns, tcs, content):607    del turns608    if not content or not any(609        kw in content.lower() for kw in ("warmest", "rank", "hot", "cool")610    ):611        return False, f"Final content missing a ranking: {content!r}"612    return True, f"{len(tcs)} total get_weather calls; ranking produced"613 614 615# ---- Test 4: Trip planning (different tools, parallel in one turn) ----616 617_TRIP_TOOLS = [618    {619        "type": "function",620        "function": {621            "name": "search_flights",622            "description": "Search one-way flights between two airports on a given date.",623            "parameters": {624                "type": "object",625                "properties": {626                    "from_airport": {"type": "string", "description": "IATA code, e.g. SFO"},627                    "to_airport":   {"type": "string", "description": "IATA code, e.g. JFK"},628                    "date":         {"type": "string", "description": "YYYY-MM-DD"},629                },630                "required": ["from_airport", "to_airport", "date"],631            },632        },633    },634    {635        "type": "function",636        "function": {637            "name": "search_hotels",638            "description": "Search hotels in a city for a date range.",639            "parameters": {640                "type": "object",641                "properties": {642                    "city":       {"type": "string"},643                    "check_in":   {"type": "string", "description": "YYYY-MM-DD"},644                    "check_out":  {"type": "string", "description": "YYYY-MM-DD"},645                    "max_price":  {"type": "integer"},646                },647                "required": ["city", "check_in", "check_out"],648            },649        },650    },651    {652        "type": "function",653        "function": {654            "name": "search_restaurants",655            "description": "Search restaurants in a city by cuisine.",656            "parameters": {657                "type": "object",658                "properties": {659                    "city":    {"type": "string"},660                    "cuisine": {"type": "string"},661                },662                "required": ["city"],663            },664        },665    },666]667 668_FLIGHTS_RESULT = {669    "results": [670        {"flight": "UA 1552", "depart": "08:15", "arrive": "16:45", "price": 389},671        {"flight": "AA  20",  "depart": "10:00", "arrive": "18:35", "price": 412},672    ]673}674_HOTELS_RESULT = {675    "results": [676        {"name": "Midtown Grand",    "nightly_rate": 245, "rating": 4.3},677        {"name": "Harbour Boutique", "nightly_rate": 312, "rating": 4.6},678    ]679}680_RESTAURANTS_RESULT = {681    "results": [682        {"name": "Trattoria Nona", "cuisine": "italian", "rating": 4.5},683        {"name": "Osteria Blu",    "cuisine": "italian", "rating": 4.4},684    ]685}686 687TRIP_PLAN_TEST = {688    "name": "Trip planning (3 different tools in parallel)",689    "tools": _TRIP_TOOLS,690    "messages": [691        {692            "role": "user",693            "content": (694                "I'm flying from SFO to JFK on 2026-06-12 and staying four nights "695                "(check out 2026-06-16). I'd also like some Italian restaurant "696                "suggestions in New York. Please call search_flights, search_hotels, "697                "and search_restaurants in parallel — all three in a single turn, "698                "since they don't depend on each other. Then give me a concise "699                "travel summary."700            ),701        }702    ],703    "mock_tool_responses": {704        "search_flights": lambda _: json.dumps(_FLIGHTS_RESULT),705        "search_hotels": lambda _: json.dumps(_HOTELS_RESULT),706        "search_restaurants": lambda _: json.dumps(_RESTAURANTS_RESULT),707    },708    "expected_parallel": {709        "min_parallel": 3,710        "require_distinct_tools": 3,711    },712    "validate": lambda turns, tcs, content: _validate_trip(turns, tcs, content),713}714 715 716def _validate_trip(turns, tcs, content):717    del turns718    names = {tc["function"]["name"] for tc in tcs}719    required = {"search_flights", "search_hotels", "search_restaurants"}720    missing = required - names721    if missing:722        return False, f"Missing tool calls: {missing}"723    if not content:724        return False, "No travel summary produced"725    return True, f"All three tools called; summary length={len(content)}"726 727 728# ---- Test 5: Portfolio check (same tool, parallel tickers) ----729 730_STOCK_TOOLS = [731    {732        "type": "function",733        "function": {734            "name": "get_stock_quote",735            "description": (736                "Get the latest quote for ONE ticker. When the user asks about "737                "multiple tickers, call this tool in parallel — one per symbol — "738                "rather than sequentially."739            ),740            "parameters": {741                "type": "object",742                "properties": {743                    "symbol": {"type": "string", "description": "Ticker symbol"},744                },745                "required": ["symbol"],746            },747        },748    },749]750 751_STOCK_DB = {752    "AAPL": {"symbol": "AAPL", "price": 218.45, "change_pct": "+0.8%"},753    "MSFT": {"symbol": "MSFT", "price": 421.10, "change_pct": "+1.2%"},754    "GOOGL":{"symbol": "GOOGL","price": 175.22, "change_pct": "-0.3%"},755    "AMZN": {"symbol": "AMZN", "price": 189.76, "change_pct": "+0.5%"},756    "NVDA": {"symbol": "NVDA", "price": 140.88, "change_pct": "+2.4%"},757}758 759 760def _stock_mock(args):761    sym = args.get("symbol", "").strip().upper()762    if sym in _STOCK_DB:763        return json.dumps(_STOCK_DB[sym])764    return json.dumps({"symbol": sym, "error": "unknown ticker"})765 766 767PORTFOLIO_TEST = {768    "name": "Portfolio check (same tool, 5 tickers in parallel)",769    "tools": _STOCK_TOOLS,770    "messages": [771        {772            "role": "user",773            "content": (774                "Pull the latest quote for every ticker in my portfolio — AAPL, "775                "MSFT, GOOGL, AMZN, and NVDA — in a single parallel batch. These "776                "lookups are independent, so please don't chain them across turns. "777                "Once you have all five, tell me which ticker had the biggest "778                "percentage change today."779            ),780        }781    ],782    "mock_tool_responses": {"get_stock_quote": _stock_mock},783    "expected_parallel": {784        "min_parallel": 5,785        "require_same_tool": "get_stock_quote",786        "min_distinct_args_key": "symbol",787        "min_distinct_args_count": 5,788    },789    "validate": lambda turns, tcs, content: _validate_portfolio(turns, tcs, content),790}791 792 793def _validate_portfolio(turns, tcs, content):794    del turns795    if not content or ("nvda" not in content.lower() and "NVDA" not in content):796        return False, f"Expected NVDA to be identified as the biggest mover: {content!r}"797    return True, f"{len(tcs)} total quotes pulled"798 799 800# ---- Test 6: Mixed — translate + dictionary in parallel for the same word ----801 802_LANG_TOOLS = [803    {804        "type": "function",805        "function": {806            "name": "translate_text",807            "description": "Translate a short text into a target language.",808            "parameters": {809                "type": "object",810                "properties": {811                    "text":            {"type": "string"},812                    "target_language": {"type": "string",813                                        "description": "ISO 639-1 language code, e.g. 'es'"},814                },815                "required": ["text", "target_language"],816            },817        },818    },819    {820        "type": "function",821        "function": {822            "name": "get_definition",823            "description": "Get the English dictionary definition of a word.",824            "parameters": {825                "type": "object",826                "properties": {827                    "word": {"type": "string"},828                },829                "required": ["word"],830            },831        },832    },833    {834        "type": "function",835        "function": {836            "name": "get_synonyms",837            "description": "Get English synonyms for a word.",838            "parameters": {839                "type": "object",840                "properties": {841                    "word": {"type": "string"},842                },843                "required": ["word"],844            },845        },846    },847]848 849 850def _translate_mock(args):851    t = args.get("text", "")852    lang = args.get("target_language", "")853    return json.dumps({"source": t, "target_language": lang, "translation": f"[{lang}] {t}"})854 855 856def _definition_mock(args):857    w = args.get("word", "")858    return json.dumps({859        "word": w,860        "definition": f"A standard dictionary definition of {w!r}.",861    })862 863 864def _synonyms_mock(args):865    w = args.get("word", "")866    return json.dumps({867        "word": w,868        "synonyms": ["synonym_a", "synonym_b", "synonym_c"],869    })870 871 872LANG_TOOLKIT_TEST = {873    "name": "Language toolkit (translate + definition + synonyms in parallel)",874    "tools": _LANG_TOOLS,875    "messages": [876        {877            "role": "user",878            "content": (879                "For the English word 'resilient', I need three independent "880                "look-ups at once: (a) translate it into Spanish, (b) fetch its "881                "dictionary definition, and (c) list its synonyms. These three "882                "calls don't depend on each other — please issue them in parallel "883                "in a single turn. Then present the combined results as a short "884                "language note."885            ),886        }887    ],888    "mock_tool_responses": {889        "translate_text":  _translate_mock,890        "get_definition":  _definition_mock,891        "get_synonyms":    _synonyms_mock,892    },893    "expected_parallel": {894        "min_parallel": 3,895        "require_distinct_tools": 3,896    },897    "validate": lambda turns, tcs, content: _validate_lang(turns, tcs, content),898}899 900 901def _validate_lang(turns, tcs, content):902    del turns903    names = {tc["function"]["name"] for tc in tcs}904    required = {"translate_text", "get_definition", "get_synonyms"}905    missing = required - names906    if missing:907        return False, f"Missing tool calls: {missing}"908    if not content:909        return False, "No language note produced"910    return True, f"All three lookup tools called; note length={len(content)}"911 912 913# ---------------------------------------------------------------------------914# All test cases915# ---------------------------------------------------------------------------916 917ALL_TEST_CASES = [918    MULTIFILE_READ_TEST,919    TODO_BATCH_TEST,920    MULTI_WEATHER_TEST,921    TRIP_PLAN_TEST,922    PORTFOLIO_TEST,923    LANG_TOOLKIT_TEST,924]925 926 927# ---------------------------------------------------------------------------928# Entry point929# ---------------------------------------------------------------------------930 931 932def main():933    parser = argparse.ArgumentParser(934        description=(935            "Test llama-server parallel tool-calling capability. Run this only "936            "against models configured for parallel tool calls — this script "937            "does not configure that itself."938        )939    )940    parser.add_argument("--host", default="localhost")941    parser.add_argument("--port", default=8080, type=int)942    parser.add_argument(943        "--no-stream", action="store_true", help="Disable streaming mode tests"944    )945    parser.add_argument(946        "--stream-only", action="store_true", help="Only run streaming mode tests"947    )948    parser.add_argument(949        "--test",950        help="Run only the test whose name contains this substring (case-insensitive)",951    )952    args = parser.parse_args()953 954    url = f"http://{args.host}:{args.port}/v1/chat/completions"955    print_info(f"Testing server at {url}")956    print_warn(957        "This script expects the target model to emit multiple tool calls in a "958        "single assistant turn. Run it only against parallel-tool-capable models."959    )960 961    modes: list[bool] = []962    if not args.stream_only:963        modes.append(False)964    if not args.no_stream:965        modes.append(True)966 967    cases: list[dict] = ALL_TEST_CASES968    if args.test:969        name_filter = args.test.lower()970        cases = [c for c in cases if name_filter in str(c["name"]).lower()]971        if not cases:972            print_fail(f"No test cases matched '{args.test}'")973            sys.exit(1)974 975    total = 0976    passed = 0977    for stream in modes:978        for case in cases:979            total += 1980            if run_test(url, case, stream=stream):981                passed += 1982 983    color = GREEN if passed == total else RED984    _print(f"\n{BOLD}{color}{'─' * 60}{RESET}")985    _print(f"{BOLD}{color}  Results: {passed}/{total} passed{RESET}")986    _print(f"{BOLD}{color}{'─' * 60}{RESET}\n")987    sys.exit(0 if passed == total else 1)988 989 990if __name__ == "__main__":991    main()992