KBaba7/llama.cpp

1# The flake interface to llama.cpp's Nix expressions. The flake is used as a2# more discoverable entry-point, as well as a way to pin the dependencies and3# expose default outputs, including the outputs built by the CI.4 5# For more serious applications involving some kind of customization  you may6# want to consider consuming the overlay, or instantiating `llamaPackages`7# directly:8#9# ```nix10# pkgs.callPackage ${llama-cpp-root}/.devops/nix/scope.nix { }`11# ```12 13# Cf. https://jade.fyi/blog/flakes-arent-real/ for a more detailed exposition14# of the relation between Nix and the Nix Flakes.15{16  description = "Port of Facebook's LLaMA model in C/C++";17 18  inputs = {19    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";20    flake-parts.url = "github:hercules-ci/flake-parts";21  };22 23  # There's an optional binary cache available. The details are below, but they're commented out.24  #25  # Why? The terrible experience of being prompted to accept them on every single Nix command run.26  # Plus, there are warnings shown about not being a trusted user on a default Nix install27  # if you *do* say yes to the prompts.28  #29  # This experience makes having `nixConfig` in a flake a persistent UX problem.30  #31  # To make use of the binary cache, please add the relevant settings to your `nix.conf`.32  # It's located at `/etc/nix/nix.conf` on non-NixOS systems. On NixOS, adjust the `nix.settings`33  # option in your NixOS configuration to add `extra-substituters` and `extra-trusted-public-keys`,34  # as shown below.35  #36  # ```37  # nixConfig = {38  #   extra-substituters = [39  #     # Populated by the CI in ggerganov/llama.cpp40  #     "https://llama-cpp.cachix.org"41  #42  #     # A development cache for nixpkgs imported with `config.cudaSupport = true`.43  #     # Populated by https://hercules-ci.com/github/SomeoneSerge/nixpkgs-cuda-ci.44  #     # This lets one skip building e.g. the CUDA-enabled openmpi.45  #     # TODO: Replace once nix-community obtains an official one.46  #     "https://cuda-maintainers.cachix.org"47  #   ];48  #49  #   # Verify these are the same keys as published on50  #   # - https://app.cachix.org/cache/llama-cpp51  #   # - https://app.cachix.org/cache/cuda-maintainers52  #   extra-trusted-public-keys = [53  #     "llama-cpp.cachix.org-1:H75X+w83wUKTIPSO1KWy9ADUrzThyGs8P5tmAbkWhQc="54  #     "cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E="55  #   ];56  # };57  # ```58 59  # For inspection, use `nix flake show github:ggerganov/llama.cpp` or the nix repl:60  #61  # ```bash62  # ❯ nix repl63  # nix-repl> :lf github:ggerganov/llama.cpp64  # Added 13 variables.65  # nix-repl> outputs.apps.x86_64-linux.quantize66  # { program = "/nix/store/00000000000000000000000000000000-llama.cpp/bin/llama-quantize"; type = "app"; }67  # ```68  outputs =69    { self, flake-parts, ... }@inputs:70    let71      # We could include the git revisions in the package names but those would72      # needlessly trigger rebuilds:73      # llamaVersion = self.dirtyShortRev or self.shortRev;74 75      # Nix already uses cryptographic hashes for versioning, so we'll just fix76      # the fake semver for now:77      llamaVersion = "0.0.0";78    in79    flake-parts.lib.mkFlake { inherit inputs; }80 81      {82 83        imports = [84          .devops/nix/nixpkgs-instances.nix85          .devops/nix/apps.nix86          .devops/nix/devshells.nix87          .devops/nix/jetson-support.nix88        ];89 90        # An overlay can be used to have a more granular control over llama-cpp's91        # dependencies and configuration, than that offered by the `.override`92        # mechanism. Cf. https://nixos.org/manual/nixpkgs/stable/#chap-overlays.93        #94        # E.g. in a flake:95        # ```96        # { nixpkgs, llama-cpp, ... }:97        # let pkgs = import nixpkgs {98        #     overlays = [ (llama-cpp.overlays.default) ];99        #     system = "aarch64-linux";100        #     config.allowUnfree = true;101        #     config.cudaSupport = true;102        #     config.cudaCapabilities = [ "7.2" ];103        #     config.cudaEnableForwardCompat = false;104        # }; in {105        #     packages.aarch64-linux.llamaJetsonXavier = pkgs.llamaPackages.llama-cpp;106        # }107        # ```108        #109        # Cf. https://nixos.org/manual/nix/unstable/command-ref/new-cli/nix3-flake.html?highlight=flake#flake-format110        flake.overlays.default = (111          final: prev: {112            llamaPackages = final.callPackage .devops/nix/scope.nix { inherit llamaVersion; };113            inherit (final.llamaPackages) llama-cpp;114          }115        );116 117        systems = [118          "aarch64-darwin"119          "aarch64-linux"120          "x86_64-darwin" # x86_64-darwin isn't tested (and likely isn't relevant)121          "x86_64-linux"122        ];123 124        perSystem =125          {126            config,127            lib,128            system,129            pkgs,130            pkgsCuda,131            pkgsRocm,132            ...133          }:134          {135            # For standardised reproducible formatting with `nix fmt`136            formatter = pkgs.nixfmt-rfc-style;137 138            # Unlike `.#packages`, legacyPackages may contain values of139            # arbitrary types (including nested attrsets) and may even throw140            # exceptions. This attribute isn't recursed into by `nix flake141            # show` either.142            #143            # You can add arbitrary scripts to `.devops/nix/scope.nix` and144            # access them as `nix build .#llamaPackages.${scriptName}` using145            # the same path you would with an overlay.146            legacyPackages = {147              llamaPackages = pkgs.callPackage .devops/nix/scope.nix { inherit llamaVersion; };148              llamaPackagesWindows = pkgs.pkgsCross.mingwW64.callPackage .devops/nix/scope.nix {149                inherit llamaVersion;150              };151              llamaPackagesCuda = pkgsCuda.callPackage .devops/nix/scope.nix { inherit llamaVersion; };152              llamaPackagesRocm = pkgsRocm.callPackage .devops/nix/scope.nix { inherit llamaVersion; };153            };154 155            # We don't use the overlay here so as to avoid making too many instances of nixpkgs,156            # cf. https://zimbatm.com/notes/1000-instances-of-nixpkgs157            packages =158              {159                default = config.legacyPackages.llamaPackages.llama-cpp;160                vulkan = config.packages.default.override { useVulkan = true; };161                windows = config.legacyPackages.llamaPackagesWindows.llama-cpp;162                python-scripts = config.legacyPackages.llamaPackages.python-scripts;163              }164              // lib.optionalAttrs pkgs.stdenv.isLinux {165                cuda = config.legacyPackages.llamaPackagesCuda.llama-cpp;166 167                mpi-cpu = config.packages.default.override { useMpi = true; };168                mpi-cuda = config.packages.default.override { useMpi = true; };169              }170              // lib.optionalAttrs (system == "x86_64-linux") {171                rocm = config.legacyPackages.llamaPackagesRocm.llama-cpp;172              };173 174            # Packages exposed in `.#checks` will be built by the CI and by175            # `nix flake check`.176            #177            # We could test all outputs e.g. as `checks = confg.packages`.178            #179            # TODO: Build more once https://github.com/ggerganov/llama.cpp/issues/6346 has been addressed180            checks = {181              inherit (config.packages) default vulkan;182            };183          };184      };185}186