marvin/dev: set up llama.cpp (via llama-swap) Faster inference than ollama, at least with gpt-oss-20b (around 20%)
1 file changed, 10 insertions(+), 0 deletions(-)
changed files
A user/settings/llama/swap.yaml
@@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/mostlygeek/llama-swap/refs/heads/main/config-schema.json +models: + gpt-oss:20b: + cmd: llama-server --port ${PORT} -hf ggml-org/gpt-oss-20b-GGUF --jinja -ub 2048 -b 2048 + gpt-oss:120b: + cmd: llama-server --port ${PORT} -hf ggml-org/gpt-oss-120b-GGUF --jinja -ub 2048 -b 2048 +hooks: + on_startup: + preload: + - "gpt-oss:120b"