Run engines as llmgw-engine rather than root, and make the user when a node is added

The supervisor drops to the run_as user (default llmgw-engine) when the
agent is root: its credential and groups, a 0700 home under the work
directory as the working directory, HOME and XDG_CACHE_HOME there. A node
without the user keeps engines as root and logs why. node add creates the
user in the video and render groups.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 09:57:41 +02:00
1 parent c19abe95e1
commit 2d47427cdc
7 files changed
+199 -1

No files matched your search

+6
View File
@@ -58,6 +58,12 @@ APIs change between minors without a deprecation period.
writes or loads files, loads code, or moves what the agent sets (host,
port, key, model). Both are refused where a model is defined and again on
the node before the engine starts.
- **Engines no longer run as root.** The agent starts them as the system
user `llmgw-engine` (in the video and render groups for the GPUs), with a
home and caches of its own under the agent's work directory. `llmgw node
add` makes the user; a node without it keeps running engines as root and
the agent says so in its log. `supervisor.run_as` in agent.yaml names
another user, or `root`.
### Everything that can be changed on a running cluster, from the console
+8
View File
@@ -245,6 +245,7 @@ rather than repeated.
| `health_interval` | `time.Duration` | [supervisor.Config](#supervisorconfig) |
| `rendezvous_timeout` | `time.Duration` | [supervisor.Config](#supervisorconfig) |
| `alive_grace` | `time.Duration` | [supervisor.Config](#supervisorconfig) |
| `run_as` | `string` | [supervisor.Config](#supervisorconfig) |
| `ca` | `string` | [tlsconf.Config](#tlsconfconfig) |
| `cert` | `string` | [tlsconf.Config](#tlsconfconfig) |
| `key` | `string` | [tlsconf.Config](#tlsconfconfig) |
@@ -1895,6 +1896,13 @@ AliveGrace is how long a non-serving rank must stay up before it counts
as ready. There is no external signal for a worker, so staying alive is
the only evidence available.
### `run_as`
`string`
RunAs is the user engines run as when the agent is root; empty is
llmgw-engine, "root" keeps them root.
## tlsconf.Config
### `ca`
+92
View File
@@ -0,0 +1,92 @@
package supervisor
import (
"fmt"
"os"
"os/user"
"path/filepath"
"strconv"
"syscall"
)
// Engines run as their own user, not as the agent's root: an engine reads a
// model's files and answers requests, and a model's files are whatever a
// registry published. Root in that process would be root on the node for
// anything that can make an engine misbehave — a malformed file, a template,
// a flag nobody thought of.
//
// The user is DefaultRunAs, made by `llmgw node add`, in the video and render
// groups for the GPU devices. A node without it keeps running engines as
// root, and says so, so an existing fleet is not broken by an upgrade.
// DefaultRunAs is the user engines run as.
const DefaultRunAs = "llmgw-engine"
// runAs is how an engine process is started as another user: its
// credential, a working directory it owns, and the environment that points
// its caches there.
type runAs struct {
name string
cred *syscall.Credential
home string
}
// lookupUser is user.Lookup, replaced by tests.
var lookupUser = user.Lookup
// resolveRunAs finds the user engines run as. nil means as this process —
// when it is not root, there is nobody to drop to; when the user does not
// exist, why is returned.
func resolveRunAs(name, workDir string) (*runAs, string) {
if name == "" || name == "root" || os.Geteuid() != 0 {
return nil, ""
}
u, err := lookupUser(name)
if err != nil {
return nil, fmt.Sprintf("there is no user %s, so engines run as root; `llmgw node add` makes it, "+
"or: useradd --system --shell /usr/sbin/nologin %s && usermod -aG video,render %s", name, name, name)
}
r, err := credentialFor(u, workDir)
if err != nil {
return nil, fmt.Sprintf("engines run as root: %v", err)
}
return r, ""
}
func credentialFor(u *user.User, workDir string) (*runAs, error) {
uid, err1 := strconv.ParseUint(u.Uid, 10, 32)
gid, err2 := strconv.ParseUint(u.Gid, 10, 32)
if err1 != nil || err2 != nil {
return nil, fmt.Errorf("user %s has ids %s:%s", u.Username, u.Uid, u.Gid)
}
var groups []uint32
if ids, err := u.GroupIds(); err == nil {
for _, g := range ids {
if n, err := strconv.ParseUint(g, 10, 32); err == nil {
groups = append(groups, uint32(n))
}
}
}
// A home of its own under the work directory, which is root's: caches
// (Triton, torch, Hugging Face) are written there and nowhere else.
home := filepath.Join(workDir, "engine-home")
if err := os.MkdirAll(home, 0o700); err != nil {
return nil, fmt.Errorf("making %s: %w", home, err)
}
if err := os.Chown(home, int(uid), int(gid)); err != nil {
return nil, fmt.Errorf("giving %s to %s: %w", home, u.Username, err)
}
// The work directory is 0750 root's; the engine has to pass through it
// to its home.
if err := os.Chmod(workDir, 0o751); err != nil {
return nil, err
}
return &runAs{name: u.Username, home: home,
cred: &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid), Groups: groups}}, nil
}
// env is what is added to an engine's environment when it runs as r: the
// last value of a key wins in exec.
func (r *runAs) env() []string {
return []string{"HOME=" + r.home, "XDG_CACHE_HOME=" + filepath.Join(r.home, ".cache"), "USER=" + r.name, "LOGNAME=" + r.name}
}
+51
View File
@@ -0,0 +1,51 @@
package supervisor
import (
"os"
"os/exec"
"os/user"
"path/filepath"
"slices"
"strconv"
"syscall"
"testing"
)
func TestAnEngineIsGivenItsUsersCredentialHomeAndCaches(t *testing.T) {
me, err := user.Current()
if err != nil {
t.Skip(err)
}
work := filepath.Join(t.TempDir(), "agent")
if err := os.MkdirAll(work, 0o750); err != nil {
t.Fatal(err)
}
r, err := credentialFor(me, work)
if err != nil {
t.Fatal(err)
}
uid, _ := strconv.Atoi(me.Uid)
if int(r.cred.Uid) != uid || r.home != filepath.Join(work, "engine-home") {
t.Fatalf("%+v", r)
}
if st, err := os.Stat(r.home); err != nil || st.Mode().Perm() != 0o700 {
t.Errorf("home: %v %v", st.Mode(), err)
}
s := &Supervisor{cfg: Config{WorkDir: work}, as: r}
c := exec.Command("true")
c.Env = []string{"HOME=/root", "PATH=/bin"}
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
s.dropTo(c)
if c.SysProcAttr.Credential == nil || c.Dir != r.home || !slices.Contains(c.Env, "HOME="+r.home) {
t.Errorf("dir %s env %v cred %v", c.Dir, c.Env, c.SysProcAttr.Credential)
}
}
func TestNotRootOrNoSuchUserLeavesEnginesAsTheAgent(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("run as root")
}
if r, why := resolveRunAs(DefaultRunAs, t.TempDir()); r != nil || why != "" {
t.Errorf("not root: %v %q", r, why)
}
}
+33 -1
View File
@@ -39,6 +39,9 @@ type Config struct {
// as ready. There is no external signal for a worker, so staying alive is
// the only evidence available.
AliveGrace time.Duration `koanf:"alive_grace"`
// RunAs is the user engines run as when the agent is root; empty is
// llmgw-engine, "root" keeps them root.
RunAs string `koanf:"run_as"`
}
// DefaultConfig returns sensible defaults.
@@ -50,6 +53,7 @@ func DefaultConfig() Config {
HealthInterval: 2 * time.Second,
RendezvousTimeout: 5 * time.Minute,
AliveGrace: 5 * time.Second,
RunAs: DefaultRunAs,
}
}
@@ -58,6 +62,9 @@ func (c *Config) normalize() {
if c.WorkDir == "" {
c.WorkDir = d.WorkDir
}
if c.RunAs == "" {
c.RunAs = d.RunAs
}
if c.LogLines <= 0 {
c.LogLines = d.LogLines
}
@@ -154,6 +161,9 @@ type Supervisor struct {
mu sync.Mutex
shards map[string]*Shard
// as is who engines run as; nil is as the agent.
as *runAs
}
// New builds a supervisor.
@@ -165,7 +175,16 @@ func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error
if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
return nil, fmt.Errorf("work dir: %w", err)
}
return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
s := &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}
as, why := resolveRunAs(cfg.RunAs, cfg.WorkDir)
if why != "" {
log.Warn(why)
}
if as != nil {
log.Info("engines run as their own user", "user", as.name)
}
s.as = as
return s, nil
}
// ErrAlreadyRunning means a shard with that ID is already supervised.
@@ -261,6 +280,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
// Engines spawn workers; signalling only the parent leaves those workers
// holding GPU memory.
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
s.dropTo(c)
// CommandContext kills only the leader, so replace that with a group kill.
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
@@ -298,6 +318,7 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
proc.Env = c.Env
proc.Dir = s.cfg.WorkDir
proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
s.dropTo(proc)
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
stdout, err := proc.StdoutPipe()
@@ -779,3 +800,14 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
}
return nil
}
// dropTo makes a command run as the engines' user, when there is one: its
// credential, its home as the working directory, and its caches there.
func (s *Supervisor) dropTo(c *exec.Cmd) {
if s.as == nil {
return
}
c.SysProcAttr.Credential = s.as.cred
c.Dir = s.as.home
c.Env = append(c.Env, s.as.env()...)
}
+1
View File
@@ -311,6 +311,7 @@ var Reference = []Block{
{Key: "health_interval", Type: "time.Duration", Doc: "HealthInterval is how often a starting shard is polled.", Nested: ""},
{Key: "rendezvous_timeout", Type: "time.Duration", Doc: "RendezvousTimeout bounds the wait for a multi-node rendezvous point to\nstart listening.", Nested: ""},
{Key: "alive_grace", Type: "time.Duration", Doc: "AliveGrace is how long a non-serving rank must stay up before it counts\nas ready. There is no external signal for a worker, so staying alive is\nthe only evidence available.", Nested: ""},
{Key: "run_as", Type: "string", Doc: "RunAs is the user engines run as when the agent is root; empty is\nllmgw-engine, \"root\" keeps them root.", Nested: ""},
}},
{Package: "tlsconf", Name: "Config", Title: "tlsconf.Config", Doc: "", Fields: []Field{
{Key: "ca", Type: "string", Doc: "CA is the bundle used to verify the other side. In a real deployment this\nis whichever authority issued the fleet's certificates.", Nested: ""},
+8
View File
@@ -688,6 +688,14 @@ func (r *Runner) checkEngines(ctx context.Context) []string {
// --- start ---
func (r *Runner) start(ctx context.Context, _ *Result) error {
// The user engines run as, so the agent drops root before starting one:
// in the groups that own the GPU devices.
if out, err := r.Remote.Run(ctx, `set -e
id llmgw-engine >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin llmgw-engine
for g in video render; do getent group "$g" >/dev/null && sudo usermod -aG "$g" llmgw-engine; done
true`, nil); err != nil {
r.emit(StepStart, Warn, "the user engines run as could not be made, so they run as root: "+lastLine(out), "")
}
r.emit(StepStart, Running, "starting the agent", "")
if out, err := r.Remote.Run(ctx, `set -e
sudo systemctl daemon-reload