all repos — nixfiles @ 88b5327ab5091e79f8ac6511ed0c352d2f5a9e13

System and user configuration, managed by nix and home-manager

marvin/dev: set up llama.cpp (via llama-swap) Faster inference than ollama, at least with gpt-oss-20b (around 20%)

Alan Pearce
commit

88b5327ab5091e79f8ac6511ed0c352d2f5a9e13

parent

642a58d2d0a83b35cc451927b4be44d952a800a8

1 file changed, 42 insertions(+), 1 deletion(-)

changed files
M user/hosts/marvin.nix → user/hosts/marvin.nix
@@ -1,4 +1,17 @@
-{ config, pkgs, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib) removePrefix concatMapStringsSep isDerivation;
+
+  nixfiles = config.home.homeDirectory + "/projects/alin.ovh/nixfiles";
+  direct_symlink = path: config.lib.file.mkOutOfStoreSymlink "${nixfiles}/${removePrefix "/" path}";
+
+  makeDrvBinPath = ps: concatMapStringsSep ":" (p: if isDerivation p then "${p}/bin" else p) ps;
+in
 {
   imports = [
     ../settings/base.nix

@@ -28,6 +41,7 @@ home.stateVersion = "22.11";
   home.packages = with pkgs; [
     picocom
     ollama
+    llama-cpp
   ];
 
   programs.git = {

@@ -75,6 +89,33 @@ OLLAMA_CONTEXT_LENGTH = "131072";
         };
       };
     };
+    llama-swap = {
+      enable = true;
+      config = {
+        ProgramArguments = [
+          "${pkgs.llama-swap}/bin/llama-swap"
+          "--listen"
+          "0.0.0.0:9292"
+          "--config"
+          "${config.xdg.configHome}/llama-swap/config.yaml"
+          "--watch-config"
+        ];
+        RunAtLoad = true;
+        KeepAlive = true;
+        WorkingDirectory = "/Users/alan";
+        StandardOutPath = "/Users/alan/Library/Logs/llama-swap.log";
+        StandardErrorPath = "/Users/alan/Library/Logs/llama-swap.log";
+        EnvironmentVariables = {
+          PATH = makeDrvBinPath [
+            pkgs.llama-cpp
+          ];
+        };
+      };
+    };
+  };
+
+  xdg.configFile."llama-swap/config.yaml" = {
+    source = direct_symlink "/user/settings/llama/swap.yaml";
   };
 
   targets.darwin.defaults = {