marvin: configure llama-swap
1 file changed, 47 insertions(+), 0 deletions(-)
changed files
A user/settings/llama/swap-config.yml
@@ -0,0 +1,47 @@ +macros: + llama: | + llama-server --port ${PORT} + + # - small `n` are not recommended + # - MoEs require long drafts + # - dense models: can reduce `--draft-min` and `--draft-max` + next_edit: | + --spec-type ngram-mod + --spec-ngram-size-n 24 + --draft-min 48 + --draft-max 64 + +models: + sweep-next-edit-1.5b: + name: Sweep Next-Edit 1.5B + unlisted: true + cmd: | + ${llama} --hf-repo sweepai/sweep-next-edit-1.5B + ${next_edit} + sweep-next-edit-v2-7b: + name: Sweep Next-Edit v2 7B + unlisted: true + cmd: | + ${llama} --hf-repo henrik3/sweep-next-edit-v2-7B-GGUF + ${next_edit} + zeta2: + name: Zeta 2 + unlisted: true + cmd: | + ${llama} --hf-repo bartowski/zed-industries_zeta-2-GGUF + ${next_edit} + +groups: + edit_prediction: + persistent: true # not unloaded by other groups + swap: true # only run one model from this group + exclusive: false + members: + - sweep-next-edit-1.5b + - sweep-next-edit-v2-7b + - zeta2 + +hooks: + on_startup: + preload: + - sweep-next-edit-v2-7b