all repos — nixfiles @ 61e11f9c863ac3119695720802e91e4fc02a5015

System and user configuration, managed by nix and home-manager

system/roles/llm-server.nix (view raw)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
{
  config,
  lib,
  pkgs,
  ...
}:
let
  inherit (config.networking) domain fqdn;
  ls = config.services.llama-swap;
in
{
  environment.systemPackages = [
    pkgs.python3Packages.huggingface-hub
  ];

  services.llama-swap = {
    enable = true;
    listenAddress = "[::]";
    port = 11343;
    settings =
      let
        llama-cpp = pkgs.llama-cpp.override { cudaSupport = true; };
        llama-server = lib.getExe' llama-cpp "llama-server";
        llama = "${llama-server} --port \${PORT}";
        next_edit = ''
          --spec-type ngram-mod
          --spec-ngram-mod-n-match 24
          --spec-ngram-mod-n-min 24
          --spec-ngram-mod-n-max 32
          --no-webui
        '';
      in
      {
        healthCheckTimeout = 60;
        models = {
          sweep-next-edit = {
            name = "Sweep Next Edit 1.5B";
            cmd = "${llama} ${next_edit} --hf-repo sweepai/sweep-next-edit-1.5B";
            aliases = [ "sweep-next-edit-1.5B" ];
          };
          sweep-next-edit-v2 = {
            name = "Sweep Next Edit V2 7B";
            cmd = "${llama} ${next_edit} --hf-repo henrik3/sweep-next-edit-v2-7B-GGUF";
            aliases = [ "sweep-next-edit-v2-7B" ];
          };
          zeta20 = {
            name = "Zeta 2.0";
            cmd = "${llama} ${next_edit} --hf-repo bartowski/zed-industries_zeta-2-GGUF";
            aliases = [ "zeta-2" ];
          };
          zeta21 = {
            name = "Zeta 2.1";
            cmd = "${llama} ${next_edit} --hf-repo mradermacher/zeta-2.1-GGUF";
            aliases = [ "zeta-2.1" ];
          };
        };
        groups = {
          edit_prediction = {
            swap = true;
            members = [
              "sweep-next-edit"
              "sweep-next-edit-v2"
              "zeta20"
              "zeta21"
            ];
          };
        };
        includeAliasesInList = true;
        hooks.on_startup.preload = [ "sweep-next-edit" ];
      };
  };
  systemd.services.llama-swap = {
    environment.XDG_CACHE_HOME = "/var/cache/llama-swap";
    serviceConfig.CacheDirectory = "llama-swap";
  };

  services.caddy = {
    enable = true;

    globalConfig = ''
      acme_ca https://acme.${domain}/directory
    '';

    virtualHosts = {
      "llama.${fqdn}" = {
        extraConfig = ''
          reverse_proxy ${ls.listenAddress}:${toString ls.port}
        '';
      };
    };
  };
}