all repos — nixfiles @ 88b5327ab5091e79f8ac6511ed0c352d2f5a9e13

System and user configuration, managed by nix and home-manager

marvin/dev: set up llama.cpp (via llama-swap) Faster inference than ollama, at least with gpt-oss-20b (around 20%)

Alan Pearce
commit

88b5327ab5091e79f8ac6511ed0c352d2f5a9e13

parent

642a58d2d0a83b35cc451927b4be44d952a800a8

3 files changed, 64 insertions(+), 1 deletion(-)

changed files
M system/settings/dev.nix → system/settings/dev.nix
@@ -118,6 +118,18 @@ }
             }
           '';
         };
+        "swap.localhost" = {
+          logFormat = "output discard";
+          extraConfig = ''
+            import local_tls
+            reverse_proxy http://localhost:9292 {
+              transport http {
+                dial_timeout 1s
+                compression off
+              }
+            }
+          '';
+        };
       };
     };
   };
M user/hosts/marvin.nix → user/hosts/marvin.nix
@@ -1,4 +1,17 @@
-{ config, pkgs, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib) removePrefix concatMapStringsSep isDerivation;
+
+  nixfiles = config.home.homeDirectory + "/projects/alin.ovh/nixfiles";
+  direct_symlink = path: config.lib.file.mkOutOfStoreSymlink "${nixfiles}/${removePrefix "/" path}";
+
+  makeDrvBinPath = ps: concatMapStringsSep ":" (p: if isDerivation p then "${p}/bin" else p) ps;
+in
 {
   imports = [
     ../settings/base.nix

@@ -28,6 +41,7 @@ home.stateVersion = "22.11";
   home.packages = with pkgs; [
     picocom
     ollama
+    llama-cpp
   ];
 
   programs.git = {

@@ -75,6 +89,33 @@ OLLAMA_CONTEXT_LENGTH = "131072";
         };
       };
     };
+    llama-swap = {
+      enable = true;
+      config = {
+        ProgramArguments = [
+          "${pkgs.llama-swap}/bin/llama-swap"
+          "--listen"
+          "0.0.0.0:9292"
+          "--config"
+          "${config.xdg.configHome}/llama-swap/config.yaml"
+          "--watch-config"
+        ];
+        RunAtLoad = true;
+        KeepAlive = true;
+        WorkingDirectory = "/Users/alan";
+        StandardOutPath = "/Users/alan/Library/Logs/llama-swap.log";
+        StandardErrorPath = "/Users/alan/Library/Logs/llama-swap.log";
+        EnvironmentVariables = {
+          PATH = makeDrvBinPath [
+            pkgs.llama-cpp
+          ];
+        };
+      };
+    };
+  };
+
+  xdg.configFile."llama-swap/config.yaml" = {
+    source = direct_symlink "/user/settings/llama/swap.yaml";
   };
 
   targets.darwin.defaults = {
A user/settings/llama/swap.yaml
@@ -0,0 +1,10 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json
+models:
+  gpt-oss:20b:
+    cmd: llama-server --port ${PORT} -hf ggml-org/gpt-oss-20b-GGUF --jinja -ub 2048 -b 2048
+  gpt-oss:120b:
+    cmd: llama-server --port ${PORT} -hf ggml-org/gpt-oss-120b-GGUF --jinja -ub 2048 -b 2048
+hooks:
+  on_startup:
+    preload:
+      - "gpt-oss:120b"