From 2d47427cdcd7a53f79b7b909743419782d027db3 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 09:57:41 +0200 Subject: [PATCH] Run engines as llmgw-engine rather than root, and make the user when a node is added The supervisor drops to the run_as user (default llmgw-engine) when the agent is root: its credential and groups, a 0700 home under the work directory as the working directory, HOME and XDG_CACHE_HOME there. A node without the user keeps engines as root and logs why. node add creates the user in the video and render groups. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 6 ++ docs/reference.md | 8 +++ internal/agent/supervisor/runas.go | 92 +++++++++++++++++++++++++ internal/agent/supervisor/runas_test.go | 51 ++++++++++++++ internal/agent/supervisor/supervisor.go | 34 ++++++++- internal/docgen/reference_gen.go | 1 + internal/onboard/onboard.go | 8 +++ 7 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 internal/agent/supervisor/runas.go create mode 100644 internal/agent/supervisor/runas_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d28ca7c8..eb96e7ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,12 @@ APIs change between minors without a deprecation period. writes or loads files, loads code, or moves what the agent sets (host, port, key, model). Both are refused where a model is defined and again on the node before the engine starts. +- **Engines no longer run as root.** The agent starts them as the system + user `llmgw-engine` (in the video and render groups for the GPUs), with a + home and caches of its own under the agent's work directory. `llmgw node + add` makes the user; a node without it keeps running engines as root and + the agent says so in its log. `supervisor.run_as` in agent.yaml names + another user, or `root`. ### Everything that can be changed on a running cluster, from the console diff --git a/docs/reference.md b/docs/reference.md index 9317f796..b75d3d9d 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -245,6 +245,7 @@ rather than repeated. | `health_interval` | `time.Duration` | [supervisor.Config](#supervisorconfig) | | `rendezvous_timeout` | `time.Duration` | [supervisor.Config](#supervisorconfig) | | `alive_grace` | `time.Duration` | [supervisor.Config](#supervisorconfig) | +| `run_as` | `string` | [supervisor.Config](#supervisorconfig) | | `ca` | `string` | [tlsconf.Config](#tlsconfconfig) | | `cert` | `string` | [tlsconf.Config](#tlsconfconfig) | | `key` | `string` | [tlsconf.Config](#tlsconfconfig) | @@ -1895,6 +1896,13 @@ AliveGrace is how long a non-serving rank must stay up before it counts as ready. There is no external signal for a worker, so staying alive is the only evidence available. +### `run_as` + +`string` + +RunAs is the user engines run as when the agent is root; empty is +llmgw-engine, "root" keeps them root. + ## tlsconf.Config ### `ca` diff --git a/internal/agent/supervisor/runas.go b/internal/agent/supervisor/runas.go new file mode 100644 index 00000000..94e69921 --- /dev/null +++ b/internal/agent/supervisor/runas.go @@ -0,0 +1,92 @@ +package supervisor + +import ( + "fmt" + "os" + "os/user" + "path/filepath" + "strconv" + "syscall" +) + +// Engines run as their own user, not as the agent's root: an engine reads a +// model's files and answers requests, and a model's files are whatever a +// registry published. Root in that process would be root on the node for +// anything that can make an engine misbehave — a malformed file, a template, +// a flag nobody thought of. +// +// The user is DefaultRunAs, made by `llmgw node add`, in the video and render +// groups for the GPU devices. A node without it keeps running engines as +// root, and says so, so an existing fleet is not broken by an upgrade. + +// DefaultRunAs is the user engines run as. +const DefaultRunAs = "llmgw-engine" + +// runAs is how an engine process is started as another user: its +// credential, a working directory it owns, and the environment that points +// its caches there. +type runAs struct { + name string + cred *syscall.Credential + home string +} + +// lookupUser is user.Lookup, replaced by tests. +var lookupUser = user.Lookup + +// resolveRunAs finds the user engines run as. nil means as this process — +// when it is not root, there is nobody to drop to; when the user does not +// exist, why is returned. +func resolveRunAs(name, workDir string) (*runAs, string) { + if name == "" || name == "root" || os.Geteuid() != 0 { + return nil, "" + } + u, err := lookupUser(name) + if err != nil { + return nil, fmt.Sprintf("there is no user %s, so engines run as root; `llmgw node add` makes it, "+ + "or: useradd --system --shell /usr/sbin/nologin %s && usermod -aG video,render %s", name, name, name) + } + r, err := credentialFor(u, workDir) + if err != nil { + return nil, fmt.Sprintf("engines run as root: %v", err) + } + return r, "" +} + +func credentialFor(u *user.User, workDir string) (*runAs, error) { + uid, err1 := strconv.ParseUint(u.Uid, 10, 32) + gid, err2 := strconv.ParseUint(u.Gid, 10, 32) + if err1 != nil || err2 != nil { + return nil, fmt.Errorf("user %s has ids %s:%s", u.Username, u.Uid, u.Gid) + } + var groups []uint32 + if ids, err := u.GroupIds(); err == nil { + for _, g := range ids { + if n, err := strconv.ParseUint(g, 10, 32); err == nil { + groups = append(groups, uint32(n)) + } + } + } + // A home of its own under the work directory, which is root's: caches + // (Triton, torch, Hugging Face) are written there and nowhere else. + home := filepath.Join(workDir, "engine-home") + if err := os.MkdirAll(home, 0o700); err != nil { + return nil, fmt.Errorf("making %s: %w", home, err) + } + if err := os.Chown(home, int(uid), int(gid)); err != nil { + return nil, fmt.Errorf("giving %s to %s: %w", home, u.Username, err) + } + // The work directory is 0750 root's; the engine has to pass through it + // to its home. + if err := os.Chmod(workDir, 0o751); err != nil { + return nil, err + } + return &runAs{name: u.Username, home: home, + cred: &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid), Groups: groups}}, nil +} + +// env is what is added to an engine's environment when it runs as r: the +// last value of a key wins in exec. +func (r *runAs) env() []string { + return []string{"HOME=" + r.home, "XDG_CACHE_HOME=" + filepath.Join(r.home, ".cache"), "USER=" + r.name, "LOGNAME=" + r.name} +} diff --git a/internal/agent/supervisor/runas_test.go b/internal/agent/supervisor/runas_test.go new file mode 100644 index 00000000..d4c7f97f --- /dev/null +++ b/internal/agent/supervisor/runas_test.go @@ -0,0 +1,51 @@ +package supervisor + +import ( + "os" + "os/exec" + "os/user" + "path/filepath" + "slices" + "strconv" + "syscall" + "testing" +) + +func TestAnEngineIsGivenItsUsersCredentialHomeAndCaches(t *testing.T) { + me, err := user.Current() + if err != nil { + t.Skip(err) + } + work := filepath.Join(t.TempDir(), "agent") + if err := os.MkdirAll(work, 0o750); err != nil { + t.Fatal(err) + } + r, err := credentialFor(me, work) + if err != nil { + t.Fatal(err) + } + uid, _ := strconv.Atoi(me.Uid) + if int(r.cred.Uid) != uid || r.home != filepath.Join(work, "engine-home") { + t.Fatalf("%+v", r) + } + if st, err := os.Stat(r.home); err != nil || st.Mode().Perm() != 0o700 { + t.Errorf("home: %v %v", st.Mode(), err) + } + s := &Supervisor{cfg: Config{WorkDir: work}, as: r} + c := exec.Command("true") + c.Env = []string{"HOME=/root", "PATH=/bin"} + c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + s.dropTo(c) + if c.SysProcAttr.Credential == nil || c.Dir != r.home || !slices.Contains(c.Env, "HOME="+r.home) { + t.Errorf("dir %s env %v cred %v", c.Dir, c.Env, c.SysProcAttr.Credential) + } +} + +func TestNotRootOrNoSuchUserLeavesEnginesAsTheAgent(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("run as root") + } + if r, why := resolveRunAs(DefaultRunAs, t.TempDir()); r != nil || why != "" { + t.Errorf("not root: %v %q", r, why) + } +} diff --git a/internal/agent/supervisor/supervisor.go b/internal/agent/supervisor/supervisor.go index 6a6eb278..2050e7b8 100644 --- a/internal/agent/supervisor/supervisor.go +++ b/internal/agent/supervisor/supervisor.go @@ -39,6 +39,9 @@ type Config struct { // as ready. There is no external signal for a worker, so staying alive is // the only evidence available. AliveGrace time.Duration `koanf:"alive_grace"` + // RunAs is the user engines run as when the agent is root; empty is + // llmgw-engine, "root" keeps them root. + RunAs string `koanf:"run_as"` } // DefaultConfig returns sensible defaults. @@ -50,6 +53,7 @@ func DefaultConfig() Config { HealthInterval: 2 * time.Second, RendezvousTimeout: 5 * time.Minute, AliveGrace: 5 * time.Second, + RunAs: DefaultRunAs, } } @@ -58,6 +62,9 @@ func (c *Config) normalize() { if c.WorkDir == "" { c.WorkDir = d.WorkDir } + if c.RunAs == "" { + c.RunAs = d.RunAs + } if c.LogLines <= 0 { c.LogLines = d.LogLines } @@ -154,6 +161,9 @@ type Supervisor struct { mu sync.Mutex shards map[string]*Shard + + // as is who engines run as; nil is as the agent. + as *runAs } // New builds a supervisor. @@ -165,7 +175,16 @@ func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil { return nil, fmt.Errorf("work dir: %w", err) } - return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil + s := &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}} + as, why := resolveRunAs(cfg.RunAs, cfg.WorkDir) + if why != "" { + log.Warn(why) + } + if as != nil { + log.Info("engines run as their own user", "user", as.name) + } + s.as = as + return s, nil } // ErrAlreadyRunning means a shard with that ID is already supervised. @@ -261,6 +280,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma // Engines spawn workers; signalling only the parent leaves those workers // holding GPU memory. c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + s.dropTo(c) // CommandContext kills only the leader, so replace that with a group kill. c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) } @@ -298,6 +318,7 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (* proc.Env = c.Env proc.Dir = s.cfg.WorkDir proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + s.dropTo(proc) proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) } stdout, err := proc.StdoutPipe() @@ -779,3 +800,14 @@ func (s *Supervisor) KillOrphan(o Orphan) error { } return nil } + +// dropTo makes a command run as the engines' user, when there is one: its +// credential, its home as the working directory, and its caches there. +func (s *Supervisor) dropTo(c *exec.Cmd) { + if s.as == nil { + return + } + c.SysProcAttr.Credential = s.as.cred + c.Dir = s.as.home + c.Env = append(c.Env, s.as.env()...) +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index f6a97097..88db6c22 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -311,6 +311,7 @@ var Reference = []Block{ {Key: "health_interval", Type: "time.Duration", Doc: "HealthInterval is how often a starting shard is polled.", Nested: ""}, {Key: "rendezvous_timeout", Type: "time.Duration", Doc: "RendezvousTimeout bounds the wait for a multi-node rendezvous point to\nstart listening.", Nested: ""}, {Key: "alive_grace", Type: "time.Duration", Doc: "AliveGrace is how long a non-serving rank must stay up before it counts\nas ready. There is no external signal for a worker, so staying alive is\nthe only evidence available.", Nested: ""}, + {Key: "run_as", Type: "string", Doc: "RunAs is the user engines run as when the agent is root; empty is\nllmgw-engine, \"root\" keeps them root.", Nested: ""}, }}, {Package: "tlsconf", Name: "Config", Title: "tlsconf.Config", Doc: "", Fields: []Field{ {Key: "ca", Type: "string", Doc: "CA is the bundle used to verify the other side. In a real deployment this\nis whichever authority issued the fleet's certificates.", Nested: ""}, diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 29865030..4620c80c 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -688,6 +688,14 @@ func (r *Runner) checkEngines(ctx context.Context) []string { // --- start --- func (r *Runner) start(ctx context.Context, _ *Result) error { + // The user engines run as, so the agent drops root before starting one: + // in the groups that own the GPU devices. + if out, err := r.Remote.Run(ctx, `set -e +id llmgw-engine >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin llmgw-engine +for g in video render; do getent group "$g" >/dev/null && sudo usermod -aG "$g" llmgw-engine; done +true`, nil); err != nil { + r.emit(StepStart, Warn, "the user engines run as could not be made, so they run as root: "+lastLine(out), "") + } r.emit(StepStart, Running, "starting the agent", "") if out, err := r.Remote.Run(ctx, `set -e sudo systemctl daemon-reload