Felipe97/llama-cpp-compiled
01.2k
1#!/usr/bin/env python2import pytest3 4# ensure grandparent path is in sys.path5from pathlib import Path6import sys7path = Path(__file__).resolve().parents[1]8sys.path.insert(0, str(path))9 10from utils import *11from enum import Enum12from typing import TypedDict13 14server: ServerProcess15 16TIMEOUT_START_SLOW = 15 * 60 # this is needed for real model tests17TIMEOUT_HTTP_REQUEST = 6018 19@pytest.fixture(autouse=True)20def create_server():21 global server22 server = ServerPreset.tinyllama2()23 server.model_alias = "tinyllama-2-tool-call"24 server.n_slots = 125 server.n_ctx = 819226 server.n_batch = 204827 28class CompletionMode(Enum):29 NORMAL = "normal"30 STREAMED = "streamed"31 32class ToolParameters(TypedDict):33 type: str34 properties: dict[str, dict]35 required: list[str]36 37class ToolFunction(TypedDict):38 name: str39 description: str40 parameters: ToolParameters41 42class ToolDefinition(TypedDict):43 type: str44 function: ToolFunction45 46TEST_TOOL = ToolDefinition(47 type = "function",48 function = ToolFunction(49 name = "test",50 description = "",51 parameters = ToolParameters(52 type = "object",53 properties = {54 "success": {55 "type": "boolean",56 "const": True,57 },58 },59 required = ["success"],60 ),61 ),62)63 64PYTHON_TOOL = ToolDefinition(65 type = "function",66 function = ToolFunction(67 name = "python",68 description = "Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.",69 parameters = ToolParameters(70 type = "object",71 properties = {72 "code": {73 "type": "string",74 "description": "The code to run in the ipython interpreter.",75 },76 },77 required = ["code"],78 ),79 ),80)81 82WEATHER_TOOL = ToolDefinition(83 type = "function",84 function = ToolFunction(85 name = "get_current_weather",86 description = "Get the current weather in a given location",87 parameters = ToolParameters(88 type = "object",89 properties = {90 "location": {91 "type": "string",92 "description": "The city and country/state, e.g. 'San Francisco, CA', or 'Paris, France'",93 },94 },95 required = ["location"],96 ),97 ),98)99 100def do_test_completion_with_required_tool_tiny(server: ServerProcess, tool: dict, argument_key: str | None, n_predict, **kwargs):101 body = server.make_any_request("POST", "/v1/chat/completions", data={102 "max_tokens": n_predict,103 "messages": [104 {"role": "system", "content": "You are a coding assistant."},105 {"role": "user", "content": "Write an example"},106 ],107 "tool_choice": "required",108 "tools": [tool],109 "parallel_tool_calls": False,110 **kwargs,111 })112 # assert res.status_code == 200, f"Expected status code 200, got {res.status_code}"113 choice = body["choices"][0]114 tool_calls = choice["message"].get("tool_calls")115 assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'116 tool_call = tool_calls[0]117 assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'118 # assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}'119 expected_function_name = "python" if tool["type"] == "code_interpreter" else tool["function"]["name"]120 assert expected_function_name == tool_call["function"]["name"], f'Expected tool name to be {tool_call["function"]["name"]} in {choice["message"]}'121 actual_arguments = tool_call["function"]["arguments"]122 assert isinstance(actual_arguments, dict) or isinstance(actual_arguments, str), f'Expected arguments to be a dict or str, got: {actual_arguments}'123 if argument_key is not None:124 if (isinstance(actual_arguments, str)):125 actual_arguments = json.loads(actual_arguments)126 assert argument_key in actual_arguments, f"tool arguments: {actual_arguments}, expected: {argument_key}"127 128# PR #22654: commented out since we're now allowing content before tool calls in tool_call: required, so we can't force this129# in the tiny model just by using the grammar130#131# @pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])132# @pytest.mark.parametrize("template_name,tool,argument_key", [133# ("Qwen3-Coder", TEST_TOOL, "success"),134# ("Qwen3-Coder", TEST_TOOL, "success"),135# ("meta-llama-Llama-3.3-70B-Instruct", TEST_TOOL, "success"),136# ("meta-llama-Llama-3.3-70B-Instruct", TEST_TOOL, "success"),137# ("meta-llama-Llama-3.3-70B-Instruct", PYTHON_TOOL, "code"),138# ("meta-llama-Llama-3.3-70B-Instruct", PYTHON_TOOL, "code"),139# ])140# def test_completion_with_required_tool_tiny_fast(template_name: str, tool: dict, argument_key: str | None, stream: CompletionMode):141# global server142# n_predict = 1024143# # server = ServerPreset.stories15m_moe()144# server.jinja = True145# server.n_predict = n_predict146# server.chat_template_file = f'../../../models/templates/{template_name}.jinja'147# server.start()148# do_test_completion_with_required_tool_tiny(server, tool, argument_key, n_predict, stream=stream == CompletionMode.STREAMED, temperature=0.0, top_k=1, top_p=1.0)149 150# @pytest.mark.slow151# @pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])152# @pytest.mark.parametrize("template_name,tool,argument_key", [153# ("meta-llama-Llama-3.1-8B-Instruct", TEST_TOOL, "success"),154# ("meta-llama-Llama-3.1-8B-Instruct", PYTHON_TOOL, "code"),155 156# ("meetkai-functionary-medium-v3.1", TEST_TOOL, "success"),157# ("meetkai-functionary-medium-v3.1", PYTHON_TOOL, "code"),158 159# ("meetkai-functionary-medium-v3.2", TEST_TOOL, "success"),160# # Functionary v3.2 format supports raw python content, which w/ a dummy stories model will never end on its own.161# # ("meetkai-functionary-medium-v3.2", PYTHON_TOOL, "code"),162 163# ("NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use", TEST_TOOL, "success"),164# ("NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use", PYTHON_TOOL, "code"),165 166# ("meta-llama-Llama-3.2-3B-Instruct", TEST_TOOL, "success"),167# ("meta-llama-Llama-3.2-3B-Instruct", PYTHON_TOOL, "code"),168 169# ("mistralai-Mistral-Nemo-Instruct-2407", TEST_TOOL, "success"),170# ("mistralai-Mistral-Nemo-Instruct-2407", PYTHON_TOOL, "code"),171 172# ("NousResearch-Hermes-3-Llama-3.1-8B-tool_use", TEST_TOOL, "success"),173# ("NousResearch-Hermes-3-Llama-3.1-8B-tool_use", PYTHON_TOOL, "code"),174 175# ("deepseek-ai-DeepSeek-R1-Distill-Llama-8B", TEST_TOOL, "success"),176# ("deepseek-ai-DeepSeek-R1-Distill-Llama-8B", PYTHON_TOOL, "code"),177 178# ("fireworks-ai-llama-3-firefunction-v2", TEST_TOOL, "success"),179# # ("fireworks-ai-llama-3-firefunction-v2", PYTHON_TOOL, "codeFalse), True),180# # ("fireworks-ai-llama-3-firefunction-v2", PYTHON_TOOL, "code"),181 182# ])183# def test_completion_with_required_tool_tiny_slow(template_name: str, tool: dict, argument_key: str | None, stream: CompletionMode):184# global server185# n_predict = 512186# # server = ServerPreset.stories15m_moe()187# server.jinja = True188# server.n_predict = n_predict189# server.chat_template_file = f'../../../models/templates/{template_name}.jinja'190# server.start(timeout_seconds=TIMEOUT_START_SLOW)191# do_test_completion_with_required_tool_tiny(server, tool, argument_key, n_predict, stream=stream == CompletionMode.STREAMED)192 193 194@pytest.mark.slow195@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])196@pytest.mark.parametrize("tool,argument_key,hf_repo,template_override", [197 (TEST_TOOL, "success", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),198 (PYTHON_TOOL, "code", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),199 (PYTHON_TOOL, "code", "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),200 201 (TEST_TOOL, "success", "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),202 (PYTHON_TOOL, "code", "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),203 (PYTHON_TOOL, "code", "bartowski/gemma-2-2b-it-GGUF:Q4_K_M", "chatml"),204 205 (TEST_TOOL, "success", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),206 (PYTHON_TOOL, "code", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),207 (PYTHON_TOOL, "code", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),208 209 (TEST_TOOL, "success", "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", None),210 (PYTHON_TOOL, "code", "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", None),211 (PYTHON_TOOL, "code", "bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", "chatml"),212 213 (TEST_TOOL, "success", "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", None),214 (PYTHON_TOOL, "code", "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", None),215 (PYTHON_TOOL, "code", "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", "chatml"),216 217 (TEST_TOOL, "success", "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),218 (PYTHON_TOOL, "code", "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),219 (PYTHON_TOOL, "code", "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),220 221 (TEST_TOOL, "success", "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),222 (PYTHON_TOOL, "code", "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),223 (PYTHON_TOOL, "code", "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),224 225 (TEST_TOOL, "success", "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),226 (PYTHON_TOOL, "code", "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),227 (PYTHON_TOOL, "code", "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),228 229 # (TEST_TOOL, "success", "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),230 # (PYTHON_TOOL, "code", "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),231 # (PYTHON_TOOL, "code", "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),232 233 (TEST_TOOL, "success", "bartowski/functionary-small-v3.2-GGUF:Q4_K_M", ("meetkai/functionary-medium-v3.2", None)),234 (PYTHON_TOOL, "code", "bartowski/functionary-small-v3.2-GGUF:Q4_K_M", ("meetkai/functionary-medium-v3.2", None)),235 (PYTHON_TOOL, "code", "bartowski/functionary-small-v3.2-GGUF:Q4_K_M", "chatml"),236 237 (TEST_TOOL, "success", "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),238 (PYTHON_TOOL, "code", "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),239 (PYTHON_TOOL, "code", "bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),240 241 (TEST_TOOL, "success", "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),242 (PYTHON_TOOL, "code", "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),243 (PYTHON_TOOL, "code", "bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", "chatml"),244 245 (TEST_TOOL, "success", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),246 (PYTHON_TOOL, "code", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),247])248def test_completion_with_required_tool_real_model(tool: dict, argument_key: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):249 global server250 n_predict = 512251 server.jinja = True252 server.n_ctx = 8192253 server.n_predict = n_predict254 server.model_hf_repo = hf_repo255 server.model_hf_file = None256 if isinstance(template_override, tuple):257 (template_hf_repo, template_variant) = template_override258 server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"259 assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."260 elif isinstance(template_override, str):261 server.chat_template = template_override262 server.start(timeout_seconds=TIMEOUT_START_SLOW)263 body = server.make_any_request("POST", "/v1/chat/completions", data={264 "max_tokens": n_predict,265 "messages": [266 {"role": "system", "content": "You are a coding assistant."},267 {"role": "user", "content": "Write an example"},268 ],269 "tool_choice": "required",270 "tools": [tool],271 "parallel_tool_calls": False,272 "stream": stream == CompletionMode.STREAMED,273 "temperature": 0.0,274 "top_k": 1,275 "top_p": 1.0,276 }, timeout=TIMEOUT_HTTP_REQUEST)277 choice = body["choices"][0]278 tool_calls = choice["message"].get("tool_calls")279 assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'280 tool_call = tool_calls[0]281 # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'282 expected_function_name = "python" if tool["type"] == "code_interpreter" else tool["function"]["name"]283 assert expected_function_name == tool_call["function"]["name"]284 actual_arguments = tool_call["function"]["arguments"]285 assert isinstance(actual_arguments, str)286 if argument_key is not None:287 actual_arguments = json.loads(actual_arguments)288 assert argument_key in actual_arguments, f"tool arguments: {json.dumps(actual_arguments)}, expected: {argument_key}"289 290 291def do_test_completion_without_tool_call(server: ServerProcess, n_predict: int, tools: list[dict], tool_choice: str | None, **kwargs):292 body = server.make_any_request("POST", "/v1/chat/completions", data={293 "max_tokens": n_predict,294 "messages": [295 {"role": "system", "content": "You are a coding assistant."},296 {"role": "user", "content": "say hello world with python"},297 ],298 "tools": tools if tools else None,299 "tool_choice": tool_choice,300 **kwargs,301 }, timeout=TIMEOUT_HTTP_REQUEST)302 choice = body["choices"][0]303 assert choice["message"].get("tool_calls") is None, f'Expected no tool call in {choice["message"]}'304 305 306@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])307@pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [308 ("meta-llama-Llama-3.3-70B-Instruct", 128, [], None),309 ("meta-llama-Llama-3.3-70B-Instruct", 128, [TEST_TOOL], None),310 ("meta-llama-Llama-3.3-70B-Instruct", 128, [PYTHON_TOOL], 'none'),311])312def test_completion_without_tool_call_fast(template_name: str, n_predict: int, tools: list[dict], tool_choice: str | None, stream: CompletionMode):313 global server314 server.n_predict = n_predict315 server.jinja = True316 server.chat_template_file = f'../../../models/templates/{template_name}.jinja'317 server.start()318 do_test_completion_without_tool_call(server, n_predict, tools, tool_choice, stream=stream == CompletionMode.STREAMED)319 320 321@pytest.mark.slow322@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])323@pytest.mark.parametrize("template_name,n_predict,tools,tool_choice", [324 ("meetkai-functionary-medium-v3.2", 256, [], None),325 ("meetkai-functionary-medium-v3.2", 256, [TEST_TOOL], None),326 ("meetkai-functionary-medium-v3.2", 256, [PYTHON_TOOL], 'none'),327 ("meetkai-functionary-medium-v3.1", 256, [], None),328 ("meetkai-functionary-medium-v3.1", 256, [TEST_TOOL], None),329 ("meetkai-functionary-medium-v3.1", 256, [PYTHON_TOOL], 'none'),330 ("meta-llama-Llama-3.2-3B-Instruct", 256, [], None),331 ("meta-llama-Llama-3.2-3B-Instruct", 256, [TEST_TOOL], None),332 ("meta-llama-Llama-3.2-3B-Instruct", 256, [PYTHON_TOOL], 'none'),333])334def test_completion_without_tool_call_slow(template_name: str, n_predict: int, tools: list[dict], tool_choice: str | None, stream: CompletionMode):335 global server336 server.n_predict = n_predict337 server.jinja = True338 server.chat_template_file = f'../../../models/templates/{template_name}.jinja'339 server.start(timeout_seconds=TIMEOUT_START_SLOW)340 do_test_completion_without_tool_call(server, n_predict, tools, tool_choice, stream=stream == CompletionMode.STREAMED)341 342 343@pytest.mark.slow344@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])345@pytest.mark.parametrize("hf_repo,template_override", [346 ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),347 ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),348 349 ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),350 ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),351 352 ("bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", None),353 ("bartowski/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M", "chatml"),354 355 ("bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", None),356 ("bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", "chatml"),357 358 ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),359 ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),360 361 ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),362 ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),363 364 ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),365 ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),366 367 # ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),368 # ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),369 370 # ("bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),371 # ("bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),372 373 ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),374 ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", "chatml"),375 376 ("bartowski/c4ai-command-r7b-12-2024-GGUF:Q6_K_L", ("CohereForAI/c4ai-command-r7b-12-2024", "tool_use")),377 378 ("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),379 380 # Note: gemma-2-2b-it knows itself as "model", not "assistant", so we don't test the ill-suited chatml on it.381 ("bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),382 383 # ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama/Llama-3.2-3B-Instruct", None)),384])385def test_weather(hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):386 global server387 n_predict = 512388 server.jinja = True389 server.n_ctx = 8192390 server.n_predict = n_predict391 server.model_hf_repo = hf_repo392 server.model_hf_file = None393 if isinstance(template_override, tuple):394 (template_hf_repo, template_variant) = template_override395 server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"396 assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."397 elif isinstance(template_override, str):398 server.chat_template = template_override399 server.start()400 do_test_weather(server, stream=stream == CompletionMode.STREAMED, max_tokens=n_predict)401 402 403def do_test_weather(server: ServerProcess, **kwargs):404 body = server.make_any_request("POST", "/v1/chat/completions", data={405 "messages": [406 {"role": "system", "content": "You are a chatbot that uses tools/functions. Dont overthink things."},407 {"role": "user", "content": "What is the weather in Istanbul?"},408 ],409 "tools": [WEATHER_TOOL],410 **kwargs,411 }, timeout=TIMEOUT_HTTP_REQUEST)412 choice = body["choices"][0]413 tool_calls = choice["message"].get("tool_calls")414 assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'415 tool_call = tool_calls[0]416 # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'417 assert tool_call["function"]["name"] == WEATHER_TOOL["function"]["name"], f'Expected weather tool call, got {tool_call["function"]["name"]}'418 # assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}'419 actual_arguments = json.loads(tool_call["function"]["arguments"])420 assert 'location' in actual_arguments, f"location not found in {json.dumps(actual_arguments)}"421 location = actual_arguments["location"]422 assert isinstance(location, str), f"Expected location to be a string, got {type(location)}: {json.dumps(location)}"423 assert re.match('^Istanbul(( |, ?)(TR|Turkey|Tรผrkiye))?$', location), f'Expected Istanbul for location, got {location}'424 425 426@pytest.mark.slow427@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])428@pytest.mark.parametrize("result_override,n_predict,hf_repo,template_override", [429 (None, 128, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),430 (None, 128, "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", None),431 (None, 128, "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF:Q4_K_M", "chatml"),432 (None, 128, "bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),433 (None, 128, "bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),434 (None, 128, "bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-3-Llama-3.1-8B", "tool_use")),435 (None, 128, "bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai/functionary-medium-v3.2", None)),436 (None, 128, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),437 (None, 128, "bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),438 (None, 128, "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),439 ("[\\s\\S]*?\\*\\*\\s*0.5($|\\*\\*)", 8192, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)),440 441 # TODO: fix these (wrong results, either didn't respect decimal instruction or got wrong value)442 # (None, 128, "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),443 # ("[\\s\\S]*?\\*\\*\\s*0.5($|\\*\\*)", 8192, "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),444])445def test_calc_result(result_override: str | None, n_predict: int, hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):446 global server447 server.jinja = True448 server.n_ctx = 8192 * 2449 server.n_predict = n_predict450 server.model_hf_repo = hf_repo451 server.model_hf_file = None452 if isinstance(template_override, tuple):453 (template_hf_repo, template_variant) = template_override454 server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"455 assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."456 elif isinstance(template_override, str):457 server.chat_template = template_override458 server.start(timeout_seconds=TIMEOUT_START_SLOW)459 do_test_calc_result(server, result_override, n_predict, stream=stream == CompletionMode.STREAMED)460 461 462def do_test_calc_result(server: ServerProcess, result_override: str | None, n_predict: int, **kwargs):463 body = server.make_any_request("POST", "/v1/chat/completions", data={464 "max_tokens": n_predict,465 "messages": [466 {"role": "system", "content": "You are a tools-calling assistant. You express numerical values with at most two decimals."},467 {"role": "user", "content": "What's the y coordinate of a point on the unit sphere at angle 30 degrees?"},468 {469 "role": "assistant",470 "content": None,471 "tool_calls": [472 {473 "id": "call_6789",474 "type": "function",475 "function": {476 "name": "calculate",477 "arguments": "{\"expression\":\"sin(30 * pi / 180)\"}"478 }479 }480 ]481 },482 {483 "role": "tool",484 "name": "calculate",485 "content": "0.55644242476",486 "tool_call_id": "call_6789"487 }488 ],489 "tools": [490 {491 "type":"function",492 "function":{493 "name":"calculate",494 "description":"A calculator function that computes values of arithmetic expressions in the Python syntax",495 "parameters":{496 "type":"object",497 "properties":{498 "expression":{499 "type":"string",500 "description":"An arithmetic expression to compute the value of (Python syntad, assuming all floats)"501 }502 },503 "required":["expression"]504 }505 }506 }507 ],508 **kwargs,509 }, timeout=TIMEOUT_HTTP_REQUEST)510 choice = body["choices"][0]511 tool_calls = choice["message"].get("tool_calls")512 assert tool_calls is None, f'Expected no tool call in {choice["message"]}'513 content = choice["message"].get("content")514 assert content is not None, f'Expected content in {choice["message"]}'515 if result_override is not None:516 assert re.match(result_override, content), f'Expected {result_override}, got {content}'517 else:518 assert re.match('^[\\s\\S]*?((That\'s|\\bis) (approximately )?)?\\b0\\.(5\\b|56\\b|556)', content), \519 f'Expected something like "The y coordinate is 0.56.", got {content}'520 521 522@pytest.mark.slow523@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])524@pytest.mark.parametrize("n_predict,reasoning_format,expect_reasoning_content,expect_content,hf_repo,template_override", [525 (128, 'deepseek', None, "^The sum of 102 and 7 is 109[\\s\\S]*", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),526 (128, None, None, "^The sum of 102 and 7 is 109[\\s\\S]*", "bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),527 (1024, 'deepseek', "I need to calculate the sum of 102 and 7[\\s\\S]*", "To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),528 (1024, 'deepseek', "First, I [\\s\\S]*", "To find the sum of[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", ("llama-cpp-deepseek-r1", None)),529 # (1024, 'none', CompletionMode.NORMAL, None, "^(<think>\\s*)?I need[\\s\\S]*?</think>\\s*To find[\\s\\S]*", "bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),530 # (128, 'deepseek', None, "^Okay, let me figure out the sum of 102 and 7[\\s\\S]*", "bartowski/Qwen_QwQ-32B-GGUF:Q4_K_M", None),531])532def test_thoughts(n_predict: int, reasoning_format: Literal['deepseek', 'none'] | None, expect_content: str | None, expect_reasoning_content: str | None, hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):533 global server534 server.reasoning_format = reasoning_format535 server.jinja = True536 server.n_ctx = 8192 * 2537 server.n_predict = n_predict538 server.model_hf_repo = hf_repo539 server.model_hf_file = None540 if isinstance(template_override, tuple):541 (template_hf_repo, template_variant) = template_override542 server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"543 assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."544 elif isinstance(template_override, str):545 server.chat_template = template_override546 server.start()547 body = server.make_any_request("POST", "/v1/chat/completions", data={548 "max_tokens": n_predict,549 "messages": [550 {"role": "user", "content": "What's the sum of 102 and 7?"},551 ],552 "stream": stream == CompletionMode.STREAMED,553 }, timeout=TIMEOUT_HTTP_REQUEST)554 choice = body["choices"][0]555 assert choice["message"].get("tool_calls") is None, f'Expected no tool call in {choice["message"]}'556 557 content = choice["message"].get("content")558 if expect_content is None:559 assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'560 else:561 assert re.match(expect_content, content), f'Expected {expect_content}, got {content}'562 563 reasoning_content = choice["message"].get("reasoning_content")564 if expect_reasoning_content is None:565 assert reasoning_content is None, f'Expected no reasoning content in {choice["message"]}'566 else:567 assert re.match(expect_reasoning_content, reasoning_content), f'Expected {expect_reasoning_content}, got {reasoning_content}'568 569 570@pytest.mark.slow571@pytest.mark.parametrize("stream", [CompletionMode.NORMAL, CompletionMode.STREAMED])572@pytest.mark.parametrize("hf_repo,template_override", [573 ("bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF:Q4_K_M", None),574 575 ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", None),576 ("bartowski/Phi-3.5-mini-instruct-GGUF:Q4_K_M", "chatml"),577 578 ("bartowski/functionary-small-v3.2-GGUF:Q8_0", ("meetkai-functionary-medium-v3.2", None)),579 ("bartowski/functionary-small-v3.2-GGUF:Q8_0", "chatml"),580 581 # ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", None),582 ("bartowski/Meta-Llama-3.1-8B-Instruct-GGUF:Q4_K_M", "chatml"),583 584 ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),585 ("bartowski/Llama-3.2-1B-Instruct-GGUF:Q4_K_M", None),586 587 ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", ("meta-llama-Llama-3.2-3B-Instruct", None)),588 ("bartowski/Llama-3.2-3B-Instruct-GGUF:Q4_K_M", None),589 590 ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", None),591 ("bartowski/Qwen2.5-7B-Instruct-GGUF:Q4_K_M", "chatml"),592 593 ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", ("NousResearch/Hermes-2-Pro-Llama-3-8B", "tool_use")),594 ("bartowski/Hermes-2-Pro-Llama-3-8B-GGUF:Q4_K_M", "chatml"),595 596 ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", ("NousResearch-Hermes-3-Llama-3.1-8B", "tool_use")),597 ("bartowski/Hermes-3-Llama-3.1-8B-GGUF:Q4_K_M", "chatml"),598 599 ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", None),600 ("bartowski/Mistral-Nemo-Instruct-2407-GGUF:Q4_K_M", "chatml"),601 602 ("bartowski/gemma-2-2b-it-GGUF:Q4_K_M", None),603 ("bartowski/gemma-2-2b-it-GGUF:Q4_K_M", "chatml"),604])605def test_hello_world(hf_repo: str, template_override: str | Tuple[str, str | None] | None, stream: CompletionMode):606 global server607 n_predict = 512 # High because of DeepSeek R1608 server.jinja = True609 server.n_ctx = 8192610 server.n_predict = n_predict611 server.model_hf_repo = hf_repo612 server.model_hf_file = None613 if isinstance(template_override, tuple):614 (template_hf_repo, template_variant) = template_override615 server.chat_template_file = f"../../../models/templates/{template_hf_repo.replace('/', '-') + ('-' + template_variant if template_variant else '')}.jinja"616 assert os.path.exists(server.chat_template_file), f"Template file {server.chat_template_file} does not exist. Run `python scripts/get_chat_template.py {template_hf_repo} {template_variant} > {server.chat_template_file}` to download the template."617 elif isinstance(template_override, str):618 server.chat_template = template_override619 server.start(timeout_seconds=TIMEOUT_START_SLOW)620 621 do_test_hello_world(server, stream=stream == CompletionMode.STREAMED, max_tokens=n_predict)622 623 624def do_test_hello_world(server: ServerProcess, **kwargs):625 body = server.make_any_request("POST", "/v1/chat/completions", data={626 "messages": [627 {"role": "system", "content": "You are a tool-calling agent."},628 {"role": "user", "content": "say hello world with python"},629 ],630 "tools": [PYTHON_TOOL],631 **kwargs,632 }, timeout=TIMEOUT_HTTP_REQUEST)633 choice = body["choices"][0]634 tool_calls = choice["message"].get("tool_calls")635 assert tool_calls and len(tool_calls) == 1, f'Expected 1 tool call in {choice["message"]}'636 tool_call = tool_calls[0]637 # assert choice["message"].get("content") in (None, ""), f'Expected no content in {choice["message"]}'638 assert tool_call["function"]["name"] == PYTHON_TOOL["function"]["name"]639 # assert len(tool_call.get("id", "")) > 0, f'Expected non empty tool call id in {tool_call}'640 actual_arguments = json.loads(tool_call["function"]["arguments"])641 assert 'code' in actual_arguments, f"code not found in {json.dumps(actual_arguments)}"642 code = actual_arguments["code"]643 assert isinstance(code, str), f"Expected code to be a string, got {type(code)}: {json.dumps(code)}"644 assert re.match(r'''print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', re.sub(r'#.*\n?', '', code)), f'Expected hello world, got {code}'645