all repos — nixfiles @ c696706e437ad369e6ca5ceb2e6727e7eb8a3dec

System and user configuration, managed by nix and home-manager

user/settings/local-llm.nix (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
{
  config,
  pkgs,
  ...
}:
let
  srv = config.services.llama-swap;
  apiBase = "http://${srv.listenAddress}:${toString srv.port}";
  completionsPath = "/v1/completions";
in
{
  imports = [
    ../modules/llama-swap.nix
  ];

  home.packages = [
    pkgs.python3Packages.huggingface-hub
  ];

  services.llama-swap = {
    enable = true;
    packages = with pkgs; [
      llama-cpp
      python3Packages.mlx-lm
    ];
    listenAddress = "[::1]";
    port = 11343;
  };

  programs.zed-editor.userSettings = {
    edit_predictions = {
      open_ai_compatible_api = {
        api_url = "${apiBase}${completionsPath}";
      };
      provider = "open_ai_compatible_api";
    };
  };
}