@@ -1,5 +1,4 @@
 {
-  config,
   pkgs,
   ...
 }:
@@ -25,6 +24,7 @@ ../settings/programs/docker.nix
     ../settings/gaming.nix
 
     ../roles/desktop.nix
+    ../roles/llm-server.nix
     ../roles/nix-remote-builder.nix
     ../roles/home.nix
     ../roles/syncthing.nix

@@ -0,0 +1,83 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  inherit (config.networking) domain fqdn;
+  ls = config.services.llama-swap;
+in
+{
+  services.llama-swap = {
+    enable = true;
+    port = 11343;
+    settings =
+      let
+        llama-cpp = pkgs.llama-cpp.override { cudaSupport = true; };
+        llama-server = lib.getExe' llama-cpp "llama-server";
+        llama = "${llama-server} --port \${PORT}";
+        next_edit = ''
+          --spec-type ngram-mod
+          --spec-ngram-size-n 24
+          --draft-min 24
+          --draft-max 32
+          --no-webui
+        '';
+      in
+      {
+        healthCheckTimeout = 60;
+        models = {
+          sweep-next-edit = {
+            name = "Sweep Next Edit 1.5B";
+            cmd = "${llama} ${next_edit} --hf-repo sweepai/sweep-next-edit-1.5B";
+            aliases = [ "sweep-next-edit-1.5B" ];
+            unlisted = true;
+          };
+          sweep-next-edit-v2-7B = {
+            name = "Sweep Next Edit V2 7B";
+            cmd = "${llama} ${next_edit} --hf-repo henrik3/sweep-next-edit-v2-7B-GGUF";
+            aliases = [ "sweep-next-edit-v2" ];
+            unlisted = true;
+          };
+          zeta2 = {
+            name = "Zeta 2";
+            cmd = "${llama} ${next_edit} --hf-repo bartowski/zed-industries_zeta-2-GGUF";
+            unlisted = true;
+          };
+        };
+        groups = {
+          edit_prediction = {
+            swap = true;
+            members = [
+              "sweep-next-edit"
+              "sweep-next-edit-v2"
+              "zeta2"
+            ];
+          };
+        };
+        hooks.on_startup.preload = [ "sweep-next-edit" ];
+      };
+  };
+  systemd.services.llama-swap = {
+    environment.XDG_CACHE_HOME = "/var/cache/llama-swap";
+    serviceConfig.CacheDirectory = "llama-swap";
+  };
+
+  services.caddy = {
+    enable = true;
+
+    globalConfig = ''
+      acme_ca https://ca.${domain}/directory
+      # acme_ca_root path_to_crt.crt
+    '';
+
+    virtualHosts = {
+      "${fqdn}" = {
+        extraConfig = ''
+          reverse_proxy /llama/ ${ls.listenAddress}:${toString ls.port}
+        '';
+      };
+    };
+  };
+}