Xenobd/whisper.cpp

1// Command line voice assisted chess2//3// Speak chess move commands to the microphone.4// The moves will translated to chessboard positions.5//6//7 8#include "WChess.h"9#include "common-sdl.h"10#include <iostream>11 12#include <memory>13#include <thread>14 15// command-line parameters16struct whisper_params {17    int32_t n_threads  = std::min(4, (int32_t) std::thread::hardware_concurrency());18    int32_t prompt_ms  = 5000;19    int32_t command_ms = 8000;20    int32_t capture_id = -1;21    int32_t max_tokens = 32;22    int32_t audio_ctx  = 0;23 24    float vad_thold  = 0.6f;25    float freq_thold = 100.0f;26 27    float grammar_penalty = 100.0f;28 29    bool translate     = false;30    bool print_special = false;31    bool print_energy  = false;32    bool no_timestamps = true;33    bool use_gpu       = true;34    bool flash_attn    = false;35 36    std::string language  = "en";37    std::string model     = "models/ggml-base.en.bin";38    std::string fname_out;39    std::string commands;40    std::string prompt;41    std::string context;42    std::string grammar;43};44 45void whisper_print_usage(int /*argc*/, char ** argv, const whisper_params & params) {46    fprintf(stderr, "\n");47    fprintf(stderr, "usage: %s [options]\n", argv[0]);48    fprintf(stderr, "\n");49    fprintf(stderr, "options:\n");50    fprintf(stderr, "  -h,         --help           [default] show this help message and exit\n");51    fprintf(stderr, "  -t N,       --threads N      [%-7d] number of threads to use during computation\n", params.n_threads);52    fprintf(stderr, "  -pms N,     --prompt-ms N    [%-7d] prompt duration in milliseconds\n",             params.prompt_ms);53    fprintf(stderr, "  -cms N,     --command-ms N   [%-7d] command duration in milliseconds\n",            params.command_ms);54    fprintf(stderr, "  -c ID,      --capture ID     [%-7d] capture device ID\n",                           params.capture_id);55    fprintf(stderr, "  -mt N,      --max-tokens N   [%-7d] maximum number of tokens per audio chunk\n",    params.max_tokens);56    fprintf(stderr, "  -ac N,      --audio-ctx N    [%-7d] audio context size (0 - all)\n",                params.audio_ctx);57    fprintf(stderr, "  -vth N,     --vad-thold N    [%-7.2f] voice activity detection threshold\n",        params.vad_thold);58    fprintf(stderr, "  -fth N,     --freq-thold N   [%-7.2f] high-pass frequency cutoff\n",                params.freq_thold);59    fprintf(stderr, "  -tr,        --translate      [%-7s] translate from source language to english\n",   params.translate ? "true" : "false");60    fprintf(stderr, "  -ps,        --print-special  [%-7s] print special tokens\n",                        params.print_special ? "true" : "false");61    fprintf(stderr, "  -pe,        --print-energy   [%-7s] print sound energy (for debugging)\n",          params.print_energy ? "true" : "false");62    fprintf(stderr, "  -ng,        --no-gpu         [%-7s] disable GPU\n",                                 params.use_gpu ? "false" : "true");63    fprintf(stderr, "  -fa,        --flash-attn     [%-7s] flash attention during decoding\n",             params.flash_attn ? "true" : "false");64    fprintf(stderr, "  -l LANG,    --language LANG  [%-7s] spoken language\n",                             params.language.c_str());65    fprintf(stderr, "  -m FNAME,   --model FNAME    [%-7s] model path\n",                                  params.model.c_str());66    fprintf(stderr, "  -f FNAME,   --file FNAME     [%-7s] text output file name\n",                       params.fname_out.c_str());67    fprintf(stderr, "  -cmd FNAME, --commands FNAME [%-7s] text file with allowed commands\n",             params.commands.c_str());68    fprintf(stderr, "  -p,         --prompt         [%-7s] the required activation prompt\n",              params.prompt.c_str());69    fprintf(stderr, "  -ctx,       --context        [%-7s] sample text to help the transcription\n",       params.context.c_str());70    fprintf(stderr, "  --grammar-penalty N          [%-7.1f] scales down logits of nongrammar tokens\n",   params.grammar_penalty);71    fprintf(stderr, "\n");72}73 74bool whisper_params_parse(int argc, char ** argv, whisper_params & params) {75    for (int i = 1; i < argc; i++) {76        std::string arg = argv[i];77 78        if (arg == "-h" || arg == "--help") {79            whisper_print_usage(argc, argv, params);80            exit(0);81        }82        else if (arg == "-t"   || arg == "--threads")       { params.n_threads     = std::stoi(argv[++i]); }83        else if (arg == "-pms" || arg == "--prompt-ms")     { params.prompt_ms     = std::stoi(argv[++i]); }84        else if (arg == "-cms" || arg == "--command-ms")    { params.command_ms    = std::stoi(argv[++i]); }85        else if (arg == "-c"   || arg == "--capture")       { params.capture_id    = std::stoi(argv[++i]); }86        else if (arg == "-mt"  || arg == "--max-tokens")    { params.max_tokens    = std::stoi(argv[++i]); }87        else if (arg == "-ac"  || arg == "--audio-ctx")     { params.audio_ctx     = std::stoi(argv[++i]); }88        else if (arg == "-vth" || arg == "--vad-thold")     { params.vad_thold     = std::stof(argv[++i]); }89        else if (arg == "-fth" || arg == "--freq-thold")    { params.freq_thold    = std::stof(argv[++i]); }90        else if (arg == "-tr"  || arg == "--translate")     { params.translate     = true; }91        else if (arg == "-ps"  || arg == "--print-special") { params.print_special = true; }92        else if (arg == "-pe"  || arg == "--print-energy")  { params.print_energy  = true; }93        else if (arg == "-ng"  || arg == "--no-gpu")        { params.use_gpu       = false; }94        else if (arg == "-fa"  || arg == "--flash-attn")    { params.flash_attn    = true; }95        else if (arg == "-l"   || arg == "--language")      { params.language      = argv[++i]; }96        else if (arg == "-m"   || arg == "--model")         { params.model         = argv[++i]; }97        else if (arg == "-f"   || arg == "--file")          { params.fname_out     = argv[++i]; }98        else if (arg == "-cmd" || arg == "--commands")      { params.commands      = argv[++i]; }99        else if (arg == "-p"   || arg == "--prompt")        { params.prompt        = argv[++i]; }100        else if (arg == "-ctx" || arg == "--context")       { params.context       = argv[++i]; }101        else if (                 arg == "--grammar-penalty") { params.grammar_penalty = std::stof(argv[++i]); }102        else {103            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());104            whisper_print_usage(argc, argv, params);105            exit(0);106        }107    }108 109    return true;110}111 112std::unique_ptr<WChess> g_wchess;113int g_moveCount = 0;114void set_move(const std::string & move, float) {115    if (!move.empty()) {116        g_moveCount++;117        fprintf(stdout, "Move: %s\n\n", move.c_str());118    }119    else fprintf(stdout, "Move rejected\n\n");120    fprintf(stdout, "%s\n", g_wchess->stringify_board().c_str());121    fprintf(stdout, "%s\n", g_moveCount ? "White's turn" : "Black's turn");122}123 124audio_async g_audio(30*1000);125bool g_listening = false;126std::vector<float> g_pcmf32;127 128bool read_input() {129    std::string input;130    while (true) {131        fprintf(stdout, "[(l)isten/(p)ause/(q)uit]: ");132        std::cin >> input;133        fprintf(stdout, "\n");134        if (input[0] == 'q') {135            fprintf(stdout, "Quitting\n");136            return false;137        }138        if (input[0] == 'l') {139            if (!g_listening) {140                fprintf(stdout, "Listening\n");141                g_listening = true;142                g_pcmf32.clear();143                g_audio.resume();144                g_audio.clear();145            }146            else fprintf(stdout, "Still listening\n");147            return true;148        }149        else {150            if (g_listening) {151                g_listening = false;152                g_audio.get(0, g_pcmf32);153                g_audio.pause();154                fprintf(stdout, "Processing\n");155            }156            else fprintf(stdout, "Not listening\n");157            return true;158        }159    }160    return true;161}162 163bool get_audio(std::vector<float> & pcmf32_cur) {164    if (!read_input()) return false;165    if (!g_pcmf32.empty()) pcmf32_cur = std::move(g_pcmf32);166    else pcmf32_cur.clear();167    return true;168}169 170int main(int argc, char ** argv) {171    ggml_backend_load_all();172 173    whisper_params params;174 175    if (whisper_params_parse(argc, argv, params) == false) {176        return 1;177    }178 179    if (whisper_lang_id(params.language.c_str()) == -1) {180        fprintf(stderr, "error: unknown language '%s'\n", params.language.c_str());181        whisper_print_usage(argc, argv, params);182        exit(0);183    }184 185    // whisper init186 187    struct whisper_context_params cparams = whisper_context_default_params();188 189    cparams.use_gpu    = params.use_gpu;190    cparams.flash_attn = params.flash_attn;191 192    struct whisper_context * ctx = whisper_init_from_file_with_params(params.model.c_str(), cparams);193    if (!ctx) {194        fprintf(stderr, "%s: whisper_init_from_file_with_params() failed!\n", __func__);195        return 1;196    }197 198    // init audio199 200    if (!g_audio.init(params.capture_id, WHISPER_SAMPLE_RATE)) {201        fprintf(stderr, "%s: audio.init() failed!\n", __func__);202        return 1;203    }204 205    struct whisper_full_params wparams = whisper_full_default_params(whisper_sampling_strategy::WHISPER_SAMPLING_GREEDY);206    wparams.offset_ms        = 0;207    wparams.translate        = false;208    wparams.no_context       = true;209    wparams.single_segment   = true;210    wparams.print_realtime   = false;211    wparams.print_progress   = false;212    wparams.print_timestamps = true;213    wparams.print_special    = false;214    wparams.no_timestamps    = true;215 216    wparams.max_tokens       = 32;217    wparams.audio_ctx        = 768; // partial encoder context for better performance218 219    wparams.temperature     = 0.0f;220    wparams.temperature_inc = 2.0f;221    wparams.greedy.best_of  = 1;222 223    wparams.beam_search.beam_size = 1;224 225    wparams.language         = "en";226 227    wparams.grammar_penalty = 100.0;228 229    wparams.initial_prompt = params.context.data();230 231    WChess::callbacks cb;232    cb.get_audio = get_audio;233    cb.set_move = set_move;234 235    WChess::settings s;236    s.vad_ms = 2000;237    s.prompt_ms = params.prompt_ms;238    s.command_ms = params.command_ms;239    s.vad_thold = params.vad_thold;240    s.freq_thold = params.freq_thold;241    s.print_energy = params.print_energy;242 243    g_wchess.reset(new WChess(ctx, wparams, cb, s));244    set_move("start", 0);245    g_wchess->run();246 247    whisper_print_timings(ctx);248    whisper_free(ctx);249 250    return 0;251}252