marvin: use llama-server's routing instead of llama-swap
5 files changed, 32 insertions(+), 32 deletions(-)
M system/settings/dev.nix → system/settings/dev.nix
@@ -118,11 +118,11 @@ } } ''; }; - "swap.localhost" = { + "chat.localhost" = { logFormat = "output discard"; extraConfig = '' import local_tls - reverse_proxy http://localhost:9292 { + reverse_proxy http://localhost:9090 { transport http { dial_timeout 1s compression off
M user/hosts/marvin.nix → user/hosts/marvin.nix
@@ -5,12 +5,10 @@ lib, ... }: let - inherit (lib) removePrefix concatMapStringsSep isDerivation; + inherit (lib) removePrefix; nixfiles = config.home.homeDirectory + "/projects/alin.ovh/nixfiles"; direct_symlink = path: config.lib.file.mkOutOfStoreSymlink "${nixfiles}/${removePrefix "/" path}"; - - makeDrvBinPath = ps: concatMapStringsSep ":" (p: if isDerivation p then "${p}/bin" else p) ps; in { imports = [@@ -89,33 +87,27 @@ OLLAMA_CONTEXT_LENGTH = "131072"; }; }; }; - llama-swap = { + llama-server = { enable = true; config = { ProgramArguments = [ - "${pkgs.llama-swap}/bin/llama-swap" - "--listen" - "0.0.0.0:9292" - "--config" - "${config.xdg.configHome}/llama-swap/config.yaml" - "--watch-config" + "${pkgs.llama-cpp}/bin/llama-server" + "--port" + "9090" + "--models-preset" + "${config.xdg.configHome}/llama/presets.ini" ]; RunAtLoad = true; KeepAlive = true; WorkingDirectory = "/Users/alan"; - StandardOutPath = "/Users/alan/Library/Logs/llama-swap.log"; - StandardErrorPath = "/Users/alan/Library/Logs/llama-swap.log"; - EnvironmentVariables = { - PATH = makeDrvBinPath [ - pkgs.llama-cpp - ]; - }; + StandardOutPath = "/Users/alan/Library/Logs/llama-server.log"; + StandardErrorPath = "/Users/alan/Library/Logs/llama-server.log"; }; }; }; - xdg.configFile."llama-swap/config.yaml" = { - source = direct_symlink "/user/settings/llama/swap.yaml"; + xdg.configFile."llama/presets.ini" = { + source = direct_symlink "/user/settings/llama/presets.ini"; }; xdg.configFile."raycast/ai/providers.yaml" = {
A user/settings/llama/presets.ini
@@ -0,0 +1,18 @@ +version = 1 + +[ggml-org/gpt-oss-20b-GGUF] +jinja = on + +[ggml-org/gpt-oss-120b-GGUF] +jinja = on + +[henry1477/Qwen3-Embedding-4B-gguf] +embedding = on +pooling = cls +ubatch-size = 8192 + +[ggml-org/Ministral-3-14B-Reasoning-2512-GGUF] + +[ggml-org/Ministral-3-14B-Instruct-2512-GGUF] + +[ggml-org/Nemotron-Nano-3-30B-A3B-GGUF]
D user/settings/llama/swap.yaml
@@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json -models: - gpt-oss:20b: - cmd: llama-server --port ${PORT} -hf ggml-org/gpt-oss-20b-GGUF --jinja -ub 2048 -b 2048 - gpt-oss:120b: - cmd: llama-server --port ${PORT} -hf ggml-org/gpt-oss-120b-GGUF --jinja -ub 2048 -b 2048 -hooks: - on_startup: - preload: - - "gpt-oss:120b"
M user/settings/raycast/ai/providers.yaml → user/settings/raycast/ai/providers.yaml
@@ -1,7 +1,7 @@ providers: - id: llama.cpp name: llama.cpp - base_url: https://swap.localhost/v1/ + base_url: https://chat.localhost/v1/ models: - id: gpt-oss:20b name: gpt-oss-20b