matthoffner/llama-cpp-server

1from llama_cpp.server.app import create_app, Settings2from fastapi.responses import HTMLResponse3from fastapi.middleware.cors import CORSMiddleware4from fastapi.responses import RedirectResponse5import os6 7model_path = "/home/user/model/gguf-model.gguf"8 9app = create_app(10    Settings(11        n_threads=4,12        model=model_path,13        embedding=True,14        n_gpu_layers=3315    )16)17app.add_middleware(18    CORSMiddleware,19    allow_origins=["*"],20    allow_credentials=True,21    allow_methods=["*"],22    allow_headers=["*"],23)24 25@app.get("/")26async def redirect_root_to_docs():27    return RedirectResponse("/docs")28 29if __name__ == "__main__":30    import uvicorn31    uvicorn.run(app, host="0.0.0.0", port=7860)32