reyden009/speculative-decoding-lab
8
1#!/usr/bin/env bash2# Definitive final chain (uli decision): 25 configs + conditional q5-dflash.3# Replaces F1/F2/F3 (legacy INTACT in scripts/old/, experiments/runs/old/).4# Greedy T=0, non-thinking (--reasoning off), persistent llama-server via5# bench_accept.py, idempotent resume per config, "FINAL RUN DONE" marker.6#7# Usage:8# SMOKE=1 bash scripts/run_final_chain.sh # smoke: --max-prompts 2 per config9# bash scripts/run_final_chain.sh # full chain (~67-75 h ≈ 3 nights)10# DRY_RUN=1 bash scripts/run_final_chain.sh # prints the matrix (26) and exits11#12# Exit: 0 with no FAILED configs; 1 if any FAILED (check queue-final.log and errors.jsonl).13set -uo pipefail14REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"15cd "$REPO"16source scripts/spec-env.sh17set +e # spec-env.sh enables set -e; the chain handles errors via exit codes18 19LOG="experiments/runs/queue-final.log"20log() { echo "[$(date '+%F %T')] $*" | tee -a "$LOG"; }21 22SAMPLES="experiments/prompts/f1-sample.jsonl"23NTOK=25624Q="models/Qwen3-8B"25G="models/gemma-4-12b-it"26DK="models/drafts"27BENCH="python scripts/bench_accept.py"28 29# SMOKE=1 → only 2 prompts per config (smoke gate of the final-run-matrix spec).30MAXP=()31[ "${SMOKE:-0}" = "1" ] && MAXP=(--max-prompts 2)32 33OK_COUNT=034FAILED_COUNT=035SKIP_COUNT=036OOM_OK=137G8_CHECKED=038Q5_DFLASH_OK=039 40# Matrix of 25 configs (spec final-run-matrix): name|target|draft|spec_type|p_min|out41# (draft/spec_type/p_min empty in solo). out = experiments/runs/final-<name>.42MATRIX=(43 # Qwen3-8B Q4_K_M (8)44 "final-qwen-q4-solo|$Q-Q4_K_M.gguf||||experiments/runs/final-qwen-q4-solo"45 "final-qwen-q4-vanilla17b|$Q-Q4_K_M.gguf|$DK/Qwen3-1.7B-Q4_K_M.gguf|draft-simple||experiments/runs/final-qwen-q4-vanilla17b"46 "final-qwen-q4-eagle3|$Q-Q4_K_M.gguf|$DK/Qwen3-8B-speculator.eagle3-F16.gguf|draft-eagle3||experiments/runs/final-qwen-q4-eagle3"47 "final-qwen-q4-dflash|$Q-Q4_K_M.gguf|$DK/dflash_qwen3_8b_block7.gguf|draft-dflash||experiments/runs/final-qwen-q4-dflash"48 "final-qwen-q4-dspark-p0|$Q-Q4_K_M.gguf|$DK/dspark_qwen3_8b_block7.gguf|draft-dspark|0.0|experiments/runs/final-qwen-q4-dspark-p0"49 "final-qwen-q4-dspark-p2|$Q-Q4_K_M.gguf|$DK/dspark_qwen3_8b_block7.gguf|draft-dspark|0.2|experiments/runs/final-qwen-q4-dspark-p2"50 "final-qwen-q4-dspark-p4|$Q-Q4_K_M.gguf|$DK/dspark_qwen3_8b_block7.gguf|draft-dspark|0.4|experiments/runs/final-qwen-q4-dspark-p4"51 "final-qwen-q4-dspark-p6|$Q-Q4_K_M.gguf|$DK/dspark_qwen3_8b_block7.gguf|draft-dspark|0.6|experiments/runs/final-qwen-q4-dspark-p6"52 # Qwen3-8B Q5_K_M (3)53 "final-qwen-q5-solo|$Q-Q5_K_M.gguf||||experiments/runs/final-qwen-q5-solo"54 "final-qwen-q5-eagle3|$Q-Q5_K_M.gguf|$DK/Qwen3-8B-speculator.eagle3-F16.gguf|draft-eagle3||experiments/runs/final-qwen-q5-eagle3"55 "final-qwen-q5-dspark-p0|$Q-Q5_K_M.gguf|$DK/dspark_qwen3_8b_block7.gguf|draft-dspark|0.0|experiments/runs/final-qwen-q5-dspark-p0"56 # Qwen3-8B Q8_0 (3)57 "final-qwen-q8-solo|$Q-Q8_0.gguf||||experiments/runs/final-qwen-q8-solo"58 "final-qwen-q8-eagle3|$Q-Q8_0.gguf|$DK/Qwen3-8B-speculator.eagle3-F16.gguf|draft-eagle3||experiments/runs/final-qwen-q8-eagle3"59 "final-qwen-q8-dspark-p0|$Q-Q8_0.gguf|$DK/dspark_qwen3_8b_block7.gguf|draft-dspark|0.0|experiments/runs/final-qwen-q8-dspark-p0"60 # Gemma 4 12B Q4_K_M (5)61 "final-gemma-q4-solo|$G-Q4_K_M.gguf||||experiments/runs/final-gemma-q4-solo"62 "final-gemma-q4-mtp|$G-Q4_K_M.gguf|$DK/mtp-gemma-4-12b-it-Q8_0.gguf|draft-mtp||experiments/runs/final-gemma-q4-mtp"63 "final-gemma-q4-dflash-f16|$G-Q4_K_M.gguf|$DK/gemma-4-12B-it-DFlash-F16.gguf|draft-dflash||experiments/runs/final-gemma-q4-dflash-f16"64 "final-gemma-q4-dflash-q4|$G-Q4_K_M.gguf|$DK/gemma-4-12B-it-DFlash-Q4_K_M.gguf|draft-dflash||experiments/runs/final-gemma-q4-dflash-q4"65 "final-gemma-q4-dflash-q8|$G-Q4_K_M.gguf|$DK/gemma-4-12B-it-DFlash-Q8_0.gguf|draft-dflash||experiments/runs/final-gemma-q4-dflash-q8"66 # Gemma 4 12B Q5_K_M (3)67 "final-gemma-q5-solo|$G-Q5_K_M.gguf||||experiments/runs/final-gemma-q5-solo"68 "final-gemma-q5-mtp|$G-Q5_K_M.gguf|$DK/mtp-gemma-4-12b-it-Q8_0.gguf|draft-mtp||experiments/runs/final-gemma-q5-mtp"69 "final-gemma-q5-dflash-f16|$G-Q5_K_M.gguf|$DK/gemma-4-12B-it-DFlash-F16.gguf|draft-dflash||experiments/runs/final-gemma-q5-dflash-f16"70 # Gemma 4 12B Q8_0 (3) — block after OOM-CHECK71 "final-gemma-q8-solo|$G-Q8_0.gguf||||experiments/runs/final-gemma-q8-solo"72 "final-gemma-q8-mtp|$G-Q8_0.gguf|$DK/mtp-gemma-4-12b-it-Q8_0.gguf|draft-mtp||experiments/runs/final-gemma-q8-mtp"73 "final-gemma-q8-dflash-f16|$G-Q8_0.gguf|$DK/gemma-4-12B-it-DFlash-F16.gguf|draft-dflash||experiments/runs/final-gemma-q8-dflash-f16|-ub 512"74)75 76# Files present and ≥ 100 MB (the quantized DFlash drafts 422M/752M fit in;77# lesson F3 #4: the 1 GB threshold stalled downloads for 2 h).78have_file() {79 local f="$1"80 [ -f "$f" ] && [ "$(stat -c%s "$f" 2>/dev/null || echo 0)" -gt 100000000 ]81}82 83run_job() {84 local name="$1" out="$2"85 shift 286 local rc87 log "JOB START: $name"88 $BENCH --config-name "$name" --prompts "$SAMPLES" --n-tokens "$NTOK" \89 --out "$out" --resume "${MAXP[@]}" "$@"90 rc=$?91 if [ "$rc" -eq 3 ]; then92 log "JOB LOCK: $name (exit 3 = another runner on this --out) — wait 300s and retry once"93 sleep 30094 $BENCH --config-name "$name" --prompts "$SAMPLES" --n-tokens "$NTOK" \95 --out "$out" --resume "${MAXP[@]}" "$@"96 rc=$?97 if [ "$rc" -eq 3 ]; then98 log "CHAIN ABORT: $name still locked (another live chain) — aborting to avoid overwriting results"99 exit 3100 fi101 fi102 case "$rc" in103 0) log "JOB DONE: $name (exit 0)"; OK_COUNT=$((OK_COUNT + 1)) ;;104 2) log "JOB DONE WITH FAILURES: $name (exit 2) — prompts in errors.jsonl"; FAILED_COUNT=$((FAILED_COUNT + 1)) ;;105 *) log "JOB FAILED: $name (exit $rc)"; FAILED_COUNT=$((FAILED_COUNT + 1)) ;;106 esac107 return "$rc"108}109 110run_config() {111 local entry="$1"112 local name target draft stype pmin out extra113 IFS='|' read -r name target draft stype pmin out extra <<< "$entry"114 [ -n "$out" ] || out="experiments/runs/final-$name"115 if ! have_file "$target"; then116 log "SKIP: $name (target not available: $target)"117 SKIP_COUNT=$((SKIP_COUNT + 1))118 return119 fi120 if [ -n "$draft" ] && ! have_file "$draft"; then121 log "SKIP: $name (draft not available: $draft)"122 SKIP_COUNT=$((SKIP_COUNT + 1))123 return124 fi125 local args=()126 if [ -n "$draft" ]; then127 args+=(--draft "$draft" --spec-type "$stype")128 [ -n "$pmin" ] && args+=(--spec-draft-p-min "$pmin")129 fi130 [ -n "$extra" ] && args+=(--extra "$extra")131 run_job "$name" "$out" --model "$target" "${args[@]}"132}133 134# OOM-CHECK before the Gemma Q8 block (legacy pattern): llama-cli with the135# worst case g8+dflash-f16 (superset of g8-solo). Failure → SKIP the 3 g8 configs.136oom_check_g8() {137 log "OOM-CHECK: gemma q8 + dflash f16 (llama-cli -ub 512, 1 prompt n=8) — worst case of the g8 block"138 if "$LLAMA_CPP_BIN/llama-cli" -m "$G-Q8_0.gguf" -md "$DK/gemma-4-12B-it-DFlash-F16.gguf" \139 --spec-type draft-dflash -ngl 99 -ngld 99 -n 8 -p "Hello." -ub 512 \140 --no-conversation --single-turn --reasoning off -c 2048 -t 8 --seed 42 \141 < /dev/null > /tmp/opencode/oom-check-final.log 2>&1; then142 log "OOM-CHECK: OK (gemma q8 usable with -ub 512)"143 OOM_OK=1144 else145 log "OOM-CHECK: FAILED → SKIP the 3 g8 configs (see /tmp/opencode/oom-check-final.log; fallback: ctx 1024 or draft Q4/Q8)"146 OOM_OK=0147 fi148}149 150# Conditional q5-dflash (spec): decision with REAL sizes + KV ctx 2048 +151# overhead vs free VRAM; the decision (numbers + reason) stays in queue-final.log.152q5_dflash_decide() {153 local target="$Q-Q5_K_M.gguf" draft="$DK/dflash_qwen3_8b_block7.gguf"154 local tsize dsize free_mib need155 tsize=$(( $(stat -c%s "$target" 2>/dev/null || echo 0) / 1048576 ))156 dsize=$(( $(stat -c%s "$draft" 2>/dev/null || echo 0) / 1048576 ))157 if ! have_file "$target" || ! have_file "$draft"; then158 log "q5-dflash: excluded (missing files: $target / $draft)"159 return 1160 fi161 free_mib=$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits 2>/dev/null | head -n1 | tr -d ' ')162 local kv_mib=1024 overhead_mib=512163 need=$((tsize + dsize + kv_mib + overhead_mib))164 if [ -n "$free_mib" ] && [ "$free_mib" -ge "$need" ]; then165 log "q5-dflash: FITS — target ${tsize}MiB + draft ${dsize}MiB + KV ${kv_mib}MiB + overhead ${overhead_mib}MiB = ${need}MiB ≤ free VRAM ${free_mib}MiB"166 return 0167 fi168 log "q5-dflash: DOES NOT FIT — target ${tsize}MiB + draft ${dsize}MiB + KV ${kv_mib}MiB + overhead ${overhead_mib}MiB = ${need}MiB > free VRAM ${free_mib:-unknown}MiB → excluded"169 return 1170}171 172log "FINAL CHAIN START (SMOKE=${SMOKE:-0}) — ${#MATRIX[@]} configs + conditional q5-dflash"173 174# DRY_RUN: prints the matrix (26 configs) without touching the GPU and exits.175if [ "${DRY_RUN:-0}" = "1" ]; then176 echo "DRY-RUN: final-run matrix (${#MATRIX[@]} + 1 conditional):"177 for entry in "${MATRIX[@]}"; do178 IFS='|' read -r dname dtarget ddraft dstype dpmin dout <<< "$entry"179 echo " $dname | $dtarget | ${ddraft:-—} | ${dstype:-—} | ${dpmin:-—}"180 done181 echo " final-qwen-q5-dflash (conditional — decided by real VRAM at runtime)"182 echo "DRY-RUN: end ($((${#MATRIX[@]} + 1)) configs)"183 exit 0184fi185 186if q5_dflash_decide; then187 Q5_DFLASH_OK=1188fi189 190T0=$(date +%s)191for entry in "${MATRIX[@]}"; do192 name="${entry%%|*}"193 case "$name" in194 final-qwen-q5-dspark-p0)195 # Inserts the conditional config at the end of the Q5 block.196 if [ "$Q5_DFLASH_OK" = "1" ]; then197 run_config "final-qwen-q5-dflash|$Q-Q5_K_M.gguf|$DK/dflash_qwen3_8b_block7.gguf|draft-dflash||experiments/runs/final-qwen-q5-dflash"198 fi199 ;;200 final-gemma-q8-*)201 if [ "$G8_CHECKED" -eq 0 ]; then202 oom_check_g8203 G8_CHECKED=1204 fi205 if [ "$OOM_OK" != "1" ]; then206 log "SKIP: $name (OOM-CHECK g8 failed; fallback: ctx 1024 or draft Q4/Q8)"207 SKIP_COUNT=$((SKIP_COUNT + 1))208 continue209 fi210 ;;211 esac212 run_config "$entry"213done214 215DUR=$(( $(date +%s) - T0 ))216log "FINAL RUN DONE (duration ${DUR}s ≈ $((DUR / 3600))h $((DUR % 3600 / 60))m)"217log "FINAL SUMMARY: configs OK=$OK_COUNT FAILED=$FAILED_COUNT SKIP=$SKIP_COUNT"218if [ "$FAILED_COUNT" -eq 0 ] && [ "$SKIP_COUNT" -eq 0 ]; then219 log "FINAL RUN: exit 0 (no FAILED or SKIP)"220 exit 0221fi222log "FINAL RUN: exit 1 (FAILED=$FAILED_COUNT SKIP=$SKIP_COUNT — check queue-final.log and errors.jsonl)"223exit 1224