blob: 70f486b43b1a7c2d29a4ea85ab86e9e0e51d819b (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
|
{ pkgs, config, lib, ... }:
let
cfg = config.services.local-ai;
inherit (lib) mkOption types;
in
{
options.services.local-ai = {
enable = lib.mkEnableOption "Enable service";
package = lib.mkPackageOption pkgs "local-ai" { };
extraArgs = mkOption {
type = types.listOf types.str;
default = [ ];
};
port = mkOption {
type = types.port;
default = 8080;
};
threads = mkOption {
type = types.int;
default = 1;
};
models = mkOption {
type = types.either types.package types.str;
default = "models";
};
parallelRequests = mkOption {
type = types.int;
default = 1;
};
logLevel = mkOption {
type = types.enum [ "error" "warn" "info" "debug" "trace" ];
default = "warn";
};
};
config = lib.mkIf cfg.enable {
systemd.services.local-ai = {
wantedBy = [ "multi-user.target" ];
environment.LLAMACPP_PARALLEL = toString cfg.parallelRequests;
serviceConfig = {
DynamicUser = true;
ExecStart = lib.escapeShellArgs ([
"${cfg.package}/bin/local-ai"
"--address=:${toString cfg.port}"
"--threads=${toString cfg.threads}"
"--localai-config-dir=."
"--models-path=${cfg.models}"
"--log-level=${cfg.logLevel}"
]
++ lib.optional (cfg.parallelRequests > 1) "--parallel-requests"
++ cfg.extraArgs);
RuntimeDirectory = "local-ai";
WorkingDirectory = "%t/local-ai";
};
};
};
}
|
