Felipe97/llama-cpp-compiled

1import pytest2from utils import *3 4server = ServerPreset.tinyllama2()5 6 7@pytest.fixture(autouse=True)8def create_server():9    global server10    server = ServerPreset.tinyllama2()11    server.server_metrics = True12 13 14def fetch_metrics(server: ServerProcess) -> str:15    """get /metrics as raw prometheus text"""16    res = server.make_request("GET", "/metrics")17    assert res.status_code == 20018    assert "Process-Start-Time-Unix" in res.headers19    assert isinstance(res.body, str)20    return res.body21 22 23def parse_metrics(text: str) -> dict:24    """parse the prometheus text format into {name: (type, value)}"""25    out = {}26    types = {}27    for line in text.splitlines():28        if line.startswith("# TYPE "):29            _, _, name, kind = line.split(" ", 3)30            types[name] = kind31        elif line.startswith("llamacpp:") and "{" not in line:32            name, value = line.split(" ", 1)33            assert name in types, f"{name} has no # TYPE line"34            out[name] = (types[name], float(value))35    return out36 37 38def test_metrics_disabled():39    global server40    server.server_metrics = False41    server.start()42    res = server.make_request("GET", "/metrics")43    assert res.status_code == 501  # ERROR_TYPE_NOT_SUPPORTED44 45 46def test_metrics_prometheus_format():47    global server48    server.start()49    server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})50 51    text = fetch_metrics(server)52    metrics = parse_metrics(text)53 54    expected_counters = [55        "llamacpp:prompt_tokens_total",56        "llamacpp:prompt_tokens_cached_total",57        "llamacpp:prompt_seconds_total",58        "llamacpp:tokens_predicted_total",59        "llamacpp:tokens_predicted_seconds_total",60        "llamacpp:n_decode_total",61        "llamacpp:n_tokens_max",62        "llamacpp:spec_decode_num_draft_tokens_total",63        "llamacpp:spec_decode_num_accepted_tokens_total",64        "llamacpp:spec_decode_num_drafts_total",65    ]66    expected_gauges = [67        "llamacpp:prompt_tokens_seconds",68        "llamacpp:predicted_tokens_seconds",69        "llamacpp:requests_processing",70        "llamacpp:requests_deferred",71        "llamacpp:n_busy_slots_per_decode",72    ]73 74    for name in expected_counters:75        assert metrics[name][0] == "counter"76    for name in expected_gauges:77        assert metrics[name][0] == "gauge"78 79    # every metric must carry a help line80    for name in expected_counters + expected_gauges:81        assert f"# HELP {name} " in text82 83    assert metrics["llamacpp:n_decode_total"][1] > 084    assert metrics["llamacpp:requests_processing"][1] == 085 86 87def test_metrics_prompt_processed_and_cached():88    global server89    server.n_slots = 1  # keep the prompt cache on a single slot90    server.start()91 92    prompt = "the quick brown fox jumps over the lazy dog"93 94    n_processed = 095    n_cached = 096    for _ in range(2):97        res = server.make_request("POST", "/completion", data={"prompt": prompt, "n_predict": 4})98        assert res.status_code == 20099        n_processed += res.body["timings"]["prompt_n"]100        n_cached += res.body["timings"]["cache_n"]101 102    # the second request must reuse the prompt of the first one103    assert n_cached > 0104 105    metrics = parse_metrics(fetch_metrics(server))106 107    # cached tokens are counted apart, they cost no decode108    assert metrics["llamacpp:prompt_tokens_total"][1] == n_processed109    assert metrics["llamacpp:prompt_tokens_cached_total"][1] == n_cached110 111 112def test_metrics_predicted_total_matches_requests():113    global server114    server.start()115 116    n_predicted = 0117    for n_predict in [1, 4, 16]:118        res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})119        assert res.status_code == 200120        n_predicted += res.body["timings"]["predicted_n"]121 122    metrics = parse_metrics(fetch_metrics(server))123    assert metrics["llamacpp:tokens_predicted_total"][1] == n_predicted124 125 126def test_metrics_generation_rate_excludes_first_token():127    global server128    server.start()129 130    # the first token comes from the logits of the last prompt batch, so it costs no decode step131    res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 1})132    timings = res.body["timings"]133    assert timings["predicted_n"] == 1134    assert timings["predicted_per_second"] == 0.0135    assert timings["predicted_per_token_ms"] == 0.0136 137    res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 16})138    timings = res.body["timings"]139    assert timings["predicted_n"] == 16140    # the rate is over 15 decode steps, not 16 tokens141    expected = 1e3 / timings["predicted_ms"] * 15142    assert abs(timings["predicted_per_second"] - expected) < 1e-6143 144 145@pytest.mark.parametrize("n_predict", [1, 8])146def test_metrics_timings_are_finite(n_predict: int):147    global server148    server.start()149    res = server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": n_predict})150    timings = res.body["timings"]151 152    # a null here means the server produced inf or nan153    for key, value in timings.items():154        assert value is not None, f"{key} is null"155        assert value >= 0, f"{key} is negative"156 157    assert timings["prompt_ms"] > 0158    assert timings["prompt_per_token_ms"] > 0159 160 161def test_metrics_timings_on_prompt_progress():162    global server163    server.start()164 165    # a long prompt so that it is split over several batches (n_batch = 32)166    prompt = "the quick brown fox jumps over the lazy dog " * 8167    chunks = list(server.make_stream_request("POST", "/completion", data={168        "prompt": prompt,169        "n_predict": 4,170        "stream": True,171        "timings_per_token": True,172        "return_progress": True,173    }))174 175    progress = [c for c in chunks if "prompt_progress" in c]176    assert len(progress) > 1  # the prompt did not fit in a single batch177 178    # the very first update is sent before any prompt token is decoded179    first = progress[0]["timings"]180    assert first["prompt_n"] == 0181    assert first["prompt_ms"] == 0.0182    assert first["predicted_n"] == 0183    assert first["predicted_ms"] == 0.0184 185    # timings must never go backwards, nor report bogus values186    prompt_ms = 0.0187    for chunk in progress:188        timings = chunk["timings"]189        for key, value in timings.items():190            assert value is not None, f"{key} is null"191            assert value >= 0, f"{key} is negative"192        assert timings["prompt_ms"] >= prompt_ms193        prompt_ms = timings["prompt_ms"]194 195    assert prompt_ms > 0196 197 198def test_metrics_slots_idle_after_completion():199    global server200    server.server_slots = True201    server.start()202    server.make_request("POST", "/completion", data={"prompt": "I believe", "n_predict": 8})203 204    res = server.make_request("GET", "/slots")205    assert res.status_code == 200206    for slot in res.body:207        assert slot["is_processing"] is False208        if "next_token" in slot:209            # the budget of the finished task must not leak into the idle slot210            assert slot["next_token"][0]["n_remain"] == -1211            assert slot["next_token"][0]["n_decoded"] == 0212 213 214def test_metrics_embedding_prompt_is_counted():215    global server216    server = ServerPreset.bert_bge_small()217    server.server_metrics = True218    server.start()219 220    res = server.make_request("POST", "/v1/embeddings", data={"input": ["hello world", "goodbye world"]})221    assert res.status_code == 200222 223    # embedding tasks never sample a token, but their prompt still costs a decode224    metrics = parse_metrics(fetch_metrics(server))225    assert metrics["llamacpp:prompt_tokens_total"][1] > 0226    assert metrics["llamacpp:n_decode_total"][1] > 0227    assert metrics["llamacpp:tokens_predicted_total"][1] == 0228