Felipe97/llama-cpp-compiled
01.2k
1#!/usr/bin/env python32#3# Run llama.cpp tools on Snapdragon devices (natively, via ADB, or SSH).4#5 6import sys7import os8import argparse9import subprocess10import platform11import shlex12import logging13 14logger = logging.getLogger("run")15 16 17MANAGED_ENV_NAMES = (18 "GGML_HEXAGON_DEVICES",19 "GGML_HEXAGON_VERBOSE",20 "GGML_HEXAGON_PROFILE",21 "GGML_HEXAGON_NHVX",22 "GGML_HEXAGON_NHMX",23 "GGML_HEXAGON_HOSTBUF",24 "GGML_HEXAGON_OPBATCH",25 "GGML_HEXAGON_OPQUEUE",26 "GGML_HEXAGON_OPPOLL",27 "GGML_HEXAGON_OPFILTER",28 "GGML_HEXAGON_OPFUSION",29 "GGML_HEXAGON_VMEM",30 "GGML_HEXAGON_MBUF",31 "GGML_HEXAGON_MM_SELECT",32 "GGML_HEXAGON_FA_SELECT",33 "GGML_HEXAGON_AR_SELECT",34 "GGML_HEXAGON_ETM",35 "GGML_HEXAGON_ARCH",36 "GGML_HEXAGON_OPTRACE",37 "GGML_OPENCL_PLATFORM",38 "GGML_OPENCL_DEVICE",39 "GGML_OPENCL_OPFILTER",40 "GGML_OPENCL_KERNEL_CACHE_DIR",41 "GGML_OPENCL_KERNEL_CACHE_DEBUG",42 "GGML_OPENCL_FA_TUNE",43 "GGML_OPENCL_DISABLE_FUSION",44 "GGML_OPENCL_ADRENO_XMEM_GEMM",45 "GGML_OPENCL_ADRENO_USE_LARGE_BUFFER",46 "GGML_SCHED_DEBUG",47 "MTMD_BACKEND_DEVICE",48 "D",49 "DEVICE",50)51 52 53def parse_target(target_str):54 if not target_str:55 return None, None56 if target_str.startswith("adb") or target_str.startswith("android"):57 parts = target_str.split(":", 1)58 serial = parts[1] if len(parts) > 1 else None59 return "android", serial60 elif target_str.startswith("lnx") or target_str.startswith("linux") or target_str.startswith("ubuntu"):61 parts = target_str.split(":", 1)62 host = parts[1] if len(parts) > 1 else None63 return "linux", host64 elif target_str in ("wos", "windows"):65 return "windows", None66 else:67 return None, None68 69 70def shlex_join(args_list):71 if hasattr(shlex, 'join'):72 return shlex.join(args_list)73 import pipes74 return " ".join(pipes.quote(x) for x in args_list)75 76 77def split_device_list(devices):78 parts = []79 curr = []80 bracket_depth = 081 82 for ch in devices:83 if ch == '[':84 bracket_depth += 185 curr.append(ch)86 elif ch == ']':87 if bracket_depth > 0:88 bracket_depth -= 189 curr.append(ch)90 elif ch == ',' and bracket_depth == 0:91 part = "".join(curr).strip()92 if part:93 parts.append(part)94 curr = []95 else:96 curr.append(ch)97 98 part = "".join(curr).strip()99 if part:100 parts.append(part)101 102 return parts103 104 105def device_arg_from_devices(devices):106 if devices.isdigit():107 n = int(devices)108 return ",".join(f"HTP{i}" for i in range(n))109 110 names = []111 for part in split_device_list(devices):112 if "[" in part:113 part = part.split("[", 1)[0].strip()114 if part:115 names.append(part)116 117 return ",".join(names)118 119 120def normalize_cmd_device_args(cmd_args):121 for i, arg in enumerate(cmd_args):122 if arg == "--device" and i + 1 < len(cmd_args):123 cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1])124 elif arg.startswith("--device="):125 cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1])126 127 128def main():129 logging.basicConfig(level=logging.INFO, format='%(message)s')130 # Split arguments at '--'131 if '--' in sys.argv:132 idx = sys.argv.index('--')133 run_args = sys.argv[1:idx]134 cmd_args = sys.argv[idx + 1:]135 else:136 run_args = sys.argv[1:]137 cmd_args = []138 139 parser = argparse.ArgumentParser(140 description="Unified runner for llama.cpp tools on Snapdragon (natively, via ADB, or via SSH)."141 )142 parser.add_argument("--target", help="Execution target (e.g. android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, windows/wos) (default: local run)")143 parser.add_argument("--target-dir", help="Target directory on the device (default: /data/local/tmp/llama.cpp for Android, ~/llama.cpp for Linux)")144 parser.add_argument("--install-dir", help="Install directory name (defaults to pkg-TARGET or pkg-TARGET-dbg prefix based on target)")145 parser.add_argument("--debug", action="store_true", help="Use debug build (defaults to pkg-TARGET-dbg folder)")146 parser.add_argument("--devices", "--device", "-d", help="Select execution devices (split into NPU and OpenCL GPUs automatically, default: HTP0)")147 parser.add_argument("--verbose", help="Verbose level (enables both Hexagon and OpenCL kernel cache debugging)")148 parser.add_argument("--profile", help="Profiling flag (enables Hexagon profiling and OpenCL autotuning)")149 parser.add_argument("--sched-debug", action="store_true", help="Enable GGML/llama.cpp scheduler debug output (GGML_SCHED_DEBUG=2)")150 parser.add_argument("--mtmd-device", help="Specify the backend device ID for Multi-Threaded Multi-Device setup (MTMD_BACKEND_DEVICE)")151 152 # Hexagon specific parameters153 parser.add_argument("--hex-verbose", help="Enable verbose logging (GGML_HEXAGON_VERBOSE)")154 parser.add_argument("--hex-profile", help="Enable NPU/Hexagon profiling and performance metrics print (GGML_HEXAGON_PROFILE)")155 parser.add_argument("--hex-nhvx", help="Number of HVX units to use (GGML_HEXAGON_NHVX)")156 parser.add_argument("--hex-nhmx", help="Number of HMX units to use. 0 disables HMX power-up (GGML_HEXAGON_NHMX)")157 parser.add_argument("--hex-hostbuf", help="Enable host buffers (GGML_HEXAGON_HOSTBUF)")158 parser.add_argument("--hex-opbatch", help="Maximum number of operations to batch into a single HTP execution (GGML_HEXAGON_OPBATCH)")159 parser.add_argument("--hex-opqueue", help="Size of the asynchronous NPU operation queue (GGML_HEXAGON_OPQUEUE)")160 parser.add_argument("--hex-oppoll", default="1", help="Enable (1) or Disable (0) polling for NPU opbatch completion (GGML_HEXAGON_OPPOLL) (default: 1)")161 parser.add_argument("--hex-opfilter", help="Regex pattern to filter/select which operators are offloaded to NPU (GGML_HEXAGON_OPFILTER)")162 parser.add_argument("--hex-opfusion", help="NPU graph node fusion optimization level (0: disabled, 1: enabled) (GGML_HEXAGON_OPFUSION)")163 parser.add_argument("--hex-vmem", help="Maximum NPU VMEM size limit in MB to allocate (GGML_HEXAGON_VMEM)")164 parser.add_argument("--hex-mbuf", help="Maximum host buffer size limit in MB to allocate (GGML_HEXAGON_MBUF)")165 parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 3:HMX,2:HVX-tiled,1:HVX-flat,0:disable")166 parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable")167 parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable")168 parser.add_argument("--hex-etm", help="Enable Embedded Trace Macrocell hardware tracing / trace logging (GGML_HEXAGON_ETM)")169 parser.add_argument("--hex-arch", help="Target Hexagon NPU architecture version override (v73, v75, v79, v81, etc.) (GGML_HEXAGON_ARCH)")170 parser.add_argument("--hex-optrace", help="Trace buffer size in number of records (GGML_HEXAGON_OPTRACE)")171 172 # OpenCL specific parameters173 parser.add_argument("--cl-platform", help="Select OpenCL platform name/regex (e.g. Qualified Qualcomm OpenCL platform) (GGML_OPENCL_PLATFORM)")174 parser.add_argument("--cl-device", help="Select OpenCL device name/regex (e.g. Adreno GPU) (GGML_OPENCL_DEVICE)")175 parser.add_argument("--cl-opfilter", help="Regex pattern to filter/select which operators are offloaded to OpenCL (GGML_OPENCL_OPFILTER)")176 parser.add_argument("--cl-disable-fusion", action="store_true", help="Disable OpenCL kernel fusion optimizations (GGML_OPENCL_DISABLE_FUSION)")177 parser.add_argument("--cl-cache-dir", help="Directory path to store compiled OpenCL program binaries (GGML_OPENCL_KERNEL_CACHE_DIR)")178 parser.add_argument("--cl-cache-debug", help="Enable verbose debugging logs for the kernel caching system (GGML_OPENCL_KERNEL_CACHE_DEBUG)")179 parser.add_argument("--cl-fa-tune", action="store_true", help="Enable automatic Flash Attention kernel autotuning (GGML_OPENCL_FA_TUNE)")180 parser.add_argument("--cl-adreno-xmem", action="store_true", help="Enforce matmul using texture/image (xmem) memory paths on Adreno GPUs (GGML_OPENCL_ADRENO_XMEM_GEMM)")181 parser.add_argument("--cl-adreno-large-buffer", action="store_true", help="Allow allocating larger buffer sizes on Adreno GPUs (GGML_OPENCL_ADRENO_USE_LARGE_BUFFER)")182 183 args = parser.parse_args(run_args)184 185 if not cmd_args:186 parser.print_help()187 logger.error("\nError: No command specified after '--'")188 sys.exit(1)189 190 target_type = None191 target_val = None192 target_prefix = None193 if args.target:194 target_type, target_val = parse_target(args.target)195 if not target_type:196 logger.error(f"Error: Invalid target format '{args.target}'. Must be android[:serial]/adb[:serial], linux:[user@]host/lnx:[user@]host/ubuntu:[user@]host, or windows/wos.")197 sys.exit(1)198 target_prefix = args.target.split(":", 1)[0]199 200 # Resolve install directory201 install_dir = args.install_dir202 if not install_dir:203 if target_prefix:204 suffix = "-dbg" if args.debug else ""205 install_dir = f"pkg-{target_prefix}{suffix}"206 else:207 # Smart branch folder detection for local run if default is not set208 prefixes = ("wos", "windows", "lnx", "linux", "ubuntu", "adb", "android")209 suffixes = ("-dbg", "") if args.debug else ("", "-dbg")210 found = False211 for suffix in suffixes:212 for prefix in prefixes:213 test_path = f"./pkg-{prefix}{suffix}/llama.cpp"214 if os.path.exists(test_path):215 install_dir = f"pkg-{prefix}{suffix}"216 found = True217 break218 if found:219 break220 if not install_dir:221 install_dir = "pkg-android" # Fallback default222 223 # Host side package path224 package_path = os.path.join(install_dir, "llama.cpp")225 226 # Environment variables to map227 env_vars = {}228 229 def set_env(env_name, opt_val):230 if opt_val is not None:231 env_vars[env_name] = str(opt_val)232 233 # Resolve and filter devices (HTP vs OpenCL)234 device_in_cmd = None235 for i, arg in enumerate(cmd_args):236 if arg == "--device" and i + 1 < len(cmd_args):237 device_in_cmd = cmd_args[i + 1]238 break239 elif arg.startswith("--device="):240 device_in_cmd = arg.split("=", 1)[1]241 break242 243 if args.devices is not None:244 devices_val = args.devices245 elif device_in_cmd is not None:246 devices_val = device_in_cmd247 else:248 devices_val = "HTP0"249 250 if devices_val.isdigit():251 hex_devices = devices_val252 cl_device = ""253 else:254 parts = split_device_list(devices_val)255 # Any device containing "htp" is Hexagon, rest is OpenCL256 hex_parts = [p for p in parts if "htp" in p.lower()]257 cl_parts = [258 p for p in parts259 if "htp" not in p.lower()260 and p.lower() not in ("none", "cpu")261 and not p.lower().startswith("gpuopencl")262 ]263 hex_devices = ",".join(hex_parts)264 cl_device = ",".join(cl_parts)265 266 # Set Hexagon devices267 if hex_devices:268 env_vars["GGML_HEXAGON_DEVICES"] = hex_devices269 270 normalize_cmd_device_args(cmd_args)271 272 # Set OpenCL device (unless overridden by --cl-device)273 final_cl_device = args.cl_device if args.cl_device is not None else cl_device274 if final_cl_device:275 env_vars["GGML_OPENCL_DEVICE"] = final_cl_device276 277 # Map shared & backend-specific parameters with correct overrides278 279 # Verbose logging mapping280 hex_verbose_val = args.hex_verbose if args.hex_verbose is not None else args.verbose281 set_env("GGML_HEXAGON_VERBOSE", hex_verbose_val)282 283 cl_cache_debug_val = args.cl_cache_debug if args.cl_cache_debug is not None else args.verbose284 set_env("GGML_OPENCL_KERNEL_CACHE_DEBUG", cl_cache_debug_val)285 286 # Profiling mapping287 hex_profile_val = args.hex_profile if args.hex_profile is not None else args.profile288 set_env("GGML_HEXAGON_PROFILE", hex_profile_val)289 290 if args.cl_fa_tune or args.profile is not None:291 env_vars["GGML_OPENCL_FA_TUNE"] = "1"292 293 # Other Hexagon environment variables294 set_env("GGML_HEXAGON_NHVX", args.hex_nhvx)295 set_env("GGML_HEXAGON_NHMX", args.hex_nhmx)296 set_env("GGML_HEXAGON_HOSTBUF", args.hex_hostbuf)297 set_env("GGML_HEXAGON_OPBATCH", args.hex_opbatch)298 set_env("GGML_HEXAGON_OPQUEUE", args.hex_opqueue)299 set_env("GGML_HEXAGON_OPPOLL", args.hex_oppoll)300 set_env("GGML_HEXAGON_OPFILTER", args.hex_opfilter)301 set_env("GGML_HEXAGON_OPFUSION", args.hex_opfusion)302 set_env("GGML_HEXAGON_VMEM", args.hex_vmem)303 set_env("GGML_HEXAGON_MBUF", args.hex_mbuf)304 set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select)305 set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select)306 set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select)307 set_env("GGML_HEXAGON_ETM", args.hex_etm)308 set_env("GGML_HEXAGON_ARCH", args.hex_arch)309 set_env("GGML_HEXAGON_OPTRACE", args.hex_optrace)310 set_env("MTMD_BACKEND_DEVICE", args.mtmd_device)311 312 # OpenCL environment variables313 set_env("GGML_OPENCL_PLATFORM", args.cl_platform)314 set_env("GGML_OPENCL_OPFILTER", args.cl_opfilter)315 set_env("GGML_OPENCL_KERNEL_CACHE_DIR", args.cl_cache_dir)316 317 if args.cl_disable_fusion:318 env_vars["GGML_OPENCL_DISABLE_FUSION"] = "1"319 320 if args.cl_adreno_xmem:321 env_vars["GGML_OPENCL_ADRENO_XMEM_GEMM"] = "1"322 323 if args.cl_adreno_large_buffer:324 env_vars["GGML_OPENCL_ADRENO_USE_LARGE_BUFFER"] = "1"325 326 if args.sched_debug:327 env_vars["GGML_SCHED_DEBUG"] = "2"328 329 # Resolve executable path330 executable = cmd_args[0]331 known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"]332 if executable in known_binaries:333 if target_type in ("android", "linux"):334 resolved_exec = f"./bin/{executable}"335 else:336 if platform.system() == "Windows":337 resolved_exec = os.path.normpath(os.path.join(package_path, "bin", f"{executable}.exe"))338 else:339 resolved_exec = os.path.normpath(os.path.join(package_path, "bin", executable))340 cmd_args[0] = resolved_exec341 342 # Infer device string to pass to the tool343 basename = os.path.basename(executable)344 if basename.endswith(".exe"):345 basename = basename[:-4]346 347 device_val = None348 if basename == "test-backend-ops":349 for i in range(len(cmd_args)):350 if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args):351 val = cmd_args[i + 1]352 new_val = ""353 for j, char in enumerate(val):354 if char in ('[', ']'):355 if j > 0 and val[j - 1] == '\\':356 new_val += char357 else:358 new_val += '\\' + char359 else:360 new_val += char361 cmd_args[i + 1] = new_val362 363 has_b = any(arg == "-b" for arg in cmd_args)364 if not has_b:365 if args.devices:366 device_val = device_arg_from_devices(args.devices)367 else:368 device_val = "HTP0"369 if device_val:370 cmd_args += ["-b", device_val]371 else:372 has_device = any(arg.startswith("--device") for arg in cmd_args)373 if not has_device:374 if args.devices:375 device_val = device_arg_from_devices(args.devices)376 else:377 device_val = "HTP0"378 379 if device_val:380 cmd_args += ["--device", device_val]381 382 # Automatically add -v to known llama tools if sched-debug, verbose, or profile are set383 verbose_trigger = (384 args.sched_debug385 or args.verbose is not None386 or args.profile is not None387 or args.hex_verbose is not None388 or args.hex_profile is not None389 or args.hex_optrace is not None390 )391 if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"):392 if "-v" not in cmd_args and "--verbose" not in cmd_args:393 cmd_args.append("-v")394 395 # Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user396 if basename in ("llama-cli", "llama-completion", "llama-server"):397 if "-ngl" not in cmd_args and "--n-gpu-layers" not in cmd_args:398 cmd_args += ["-ngl", "99"]399 if "-fa" not in cmd_args and "--flash-attn" not in cmd_args:400 cmd_args += ["-fa", "on"]401 402 # Use ubatch-size 1024 for hexagon backend (HTP devices)403 if hex_devices and basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):404 if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:405 cmd_args += ["--ubatch-size", "1024"]406 elif basename in ("llama-cli", "llama-completion", "llama-server"):407 if "--ubatch-size" not in cmd_args and "-ub" not in cmd_args:408 cmd_args += ["--ubatch-size", "1024"]409 410 if basename in ("llama-cli", "llama-completion", "llama-server", "llama-bench"):411 if "-t" not in cmd_args and "--threads" not in cmd_args:412 cmd_args += ["-t", "6"]413 414 # Resolve target directory on device415 target_dir = args.target_dir416 if not target_dir:417 target_dir = "/data/local/tmp/llama.cpp" if target_type == "android" else "~/llama.cpp"418 419 if target_type == "android":420 # Run via ADB421 adb_base = ["adb"]422 if target_val: # serial423 adb_base += ["-s", target_val]424 425 env_parts = [426 "LD_LIBRARY_PATH=./lib",427 "ADSP_LIBRARY_PATH=./lib"428 ]429 for k, v in env_vars.items():430 env_parts.append(f"{k}={v}")431 env_str = " ".join(env_parts)432 433 cmd_str = shlex_join(cmd_args)434 adb_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}"435 full_cmd = adb_base + ["shell", adb_shell_cmd]436 437 logger.info(f"+ {' '.join(full_cmd)}")438 res = subprocess.run(full_cmd)439 sys.exit(res.returncode)440 441 elif target_type == "linux":442 ssh_host = target_val443 if not ssh_host:444 logger.error("Error: SSH host not specified in target (e.g. use linux:user@host, lnx:user@host, or ubuntu:user@host). Cannot execute.")445 sys.exit(1)446 447 # Linux remote run via SSH448 env_parts = [449 "LD_LIBRARY_PATH=./lib",450 "ADSP_LIBRARY_PATH=./lib"451 ]452 for k, v in env_vars.items():453 env_parts.append(f"{k}={v}")454 env_str = " ".join(env_parts)455 456 cmd_str = shlex_join(cmd_args)457 ssh_shell_cmd = f"cd {target_dir} && ulimit -c unlimited && {env_str} {cmd_str}"458 full_cmd = ["ssh", ssh_host, ssh_shell_cmd]459 460 logger.info(f"+ {' '.join(full_cmd)}")461 res = subprocess.run(full_cmd)462 sys.exit(res.returncode)463 464 elif target_type == "windows":465 logger.info("Windows target execution is currently a stub.")466 sys.exit(0)467 468 else:469 # Run locally470 local_env = os.environ.copy()471 lib_dir = os.path.normpath(os.path.join(package_path, "lib"))472 local_env["ADSP_LIBRARY_PATH"] = lib_dir473 if platform.system() == "Windows":474 local_env["PATH"] = lib_dir + os.path.pathsep + local_env.get("PATH", "")475 else:476 local_env["LD_LIBRARY_PATH"] = lib_dir + os.path.pathsep + local_env.get("LD_LIBRARY_PATH", "")477 478 for k in MANAGED_ENV_NAMES:479 local_env.pop(k, None)480 for k, v in env_vars.items():481 local_env[k] = v482 483 logger.info(f"+ {shlex_join(cmd_args)}")484 res = subprocess.run(cmd_args, env=local_env)485 sys.exit(res.returncode)486 487 488if __name__ == "__main__":489 try:490 main()491 except KeyboardInterrupt:492 logger.info("\nInterrupted by user.")493 sys.exit(130)494