all repos — nixfiles @ f36dcb0db79de68205fc9347f0f42bbf3f3cd2df

System and user configuration, managed by nix and home-manager

user/settings/local-llm.nix (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
{
  pkgs,
  ...
}:
{
  imports = [
    ../modules/llama-swap.nix
  ];

  home.packages = [
    pkgs.python3Packages.huggingface-hub
  ];

  services.llama-swap = {
    enable = true;
    packages = with pkgs; [
      llama-cpp
      python3Packages.mlx-lm
    ];
    listenAddress = "[::1]";
    port = 11343;
  };
}