use llama-swap peer config instead of load balancing
2 files changed, 10 insertions(+), 7 deletions(-)
M user/settings/development/web/Caddyfile → user/settings/development/web/Caddyfile
@@ -24,13 +24,7 @@ reverse_proxy http://localhost:1313 } llama.localhost, http://llama.localhost { - reverse_proxy { - to http://prefect:11343 http://localhost:11343 - lb_policy first - lb_try_duration 5s - - health_uri /health - } + reverse_proxy http://localhost:11343 } searchix.localhost {
M user/settings/llama/swap-config.yml → user/settings/llama/swap-config.yml
@@ -65,3 +65,12 @@ hooks: on_startup: preload: - sweep-next-edit-v2-7b + +peers: + prefect: + proxy: http://prefect:11343 + models: + - sweep-next-edit + - sweep-next-edit-v2 + - zeta20 + - zeta21