@@ -130,6 +130,19 @@ }
             }
           '';
         };
+        "llama.localhost" = {
+          logFormat = "output discard";
+          serverAliases = [ "http://llama.localhost" ];
+          extraConfig = ''
+            import local_tls
+            reverse_proxy http://localhost:11343 {
+              transport http {
+                dial_timeout 1s
+                compression off
+              }
+            }
+          '';
+        };
       };
     };
   };

@@ -3,6 +3,9 @@ config,
   pkgs,
   ...
 }:
+let
+  nixfiles = config.home.homeDirectory + "/projects/alin.ovh/nixfiles";
+in
 {
   imports = [
     ../settings/base.nix
@@ -18,6 +21,7 @@ ../settings/darwin.nix
     ../settings/emacs.nix
     ../settings/fish.nix
     ../settings/git.nix
+    ../settings/local-llm.nix
     ../settings/nixpkgs.nix
     ../settings/mcp.nix
     ../settings/ssh.nix
@@ -137,5 +141,10 @@ };
     "com.apple.timemachine" = {
       DoNotOfferNewDisksForBackup = true;
     };
+  };
+
+  services.llama-swap = {
+    configFile = "${nixfiles}/user/settings/llama/swap-config.yml";
+    extraArgs = [ "--watch-config" ];
   };
 }

@@ -0,0 +1,47 @@
+macros:
+  llama: |
+    llama-server --port ${PORT}
+
+  # - small `n` are not recommended
+  # - MoEs require long drafts
+  # - dense models: can reduce `--draft-min` and `--draft-max`
+  next_edit: |
+    --spec-type ngram-mod
+    --spec-ngram-size-n 24
+    --draft-min 48
+    --draft-max 64
+
+models:
+  sweep-next-edit-1.5b:
+    name: Sweep Next-Edit 1.5B
+    unlisted: true
+    cmd: |
+      ${llama} --hf-repo sweepai/sweep-next-edit-1.5B
+      ${next_edit}
+  sweep-next-edit-v2-7b:
+    name: Sweep Next-Edit v2 7B
+    unlisted: true
+    cmd: |
+      ${llama} --hf-repo henrik3/sweep-next-edit-v2-7B-GGUF
+      ${next_edit}
+  zeta2:
+    name: Zeta 2
+    unlisted: true
+    cmd: |
+      ${llama} --hf-repo bartowski/zed-industries_zeta-2-GGUF
+      ${next_edit}
+
+groups:
+  edit_prediction:
+    persistent: true # not unloaded by other groups
+    swap: true # only run one model from this group
+    exclusive: false
+    members:
+      - sweep-next-edit-1.5b
+      - sweep-next-edit-v2-7b
+      - zeta2
+
+hooks:
+  on_startup:
+    preload:
+      - sweep-next-edit-v2-7b