Run engines as llmgw-engine rather than root, and make the user when a node is added

The supervisor drops to the run_as user (default llmgw-engine) when the
agent is root: its credential and groups, a 0700 home under the work
directory as the working directory, HOME and XDG_CACHE_HOME there. A node
without the user keeps engines as root and logs why. node add creates the
user in the video and render groups.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 09:57:41 +02:00
1 parent c19abe95e1
commit 2d47427cdc
7 files changed
+199 -1

No files matched your search

+92
View File
@@ -0,0 +1,92 @@
package supervisor
import (
"fmt"
"os"
"os/user"
"path/filepath"
"strconv"
"syscall"
)
// Engines run as their own user, not as the agent's root: an engine reads a
// model's files and answers requests, and a model's files are whatever a
// registry published. Root in that process would be root on the node for
// anything that can make an engine misbehave — a malformed file, a template,
// a flag nobody thought of.
//
// The user is DefaultRunAs, made by `llmgw node add`, in the video and render
// groups for the GPU devices. A node without it keeps running engines as
// root, and says so, so an existing fleet is not broken by an upgrade.
// DefaultRunAs is the user engines run as.
const DefaultRunAs = "llmgw-engine"
// runAs is how an engine process is started as another user: its
// credential, a working directory it owns, and the environment that points
// its caches there.
type runAs struct {
name string
cred *syscall.Credential
home string
}
// lookupUser is user.Lookup, replaced by tests.
var lookupUser = user.Lookup
// resolveRunAs finds the user engines run as. nil means as this process —
// when it is not root, there is nobody to drop to; when the user does not
// exist, why is returned.
func resolveRunAs(name, workDir string) (*runAs, string) {
if name == "" || name == "root" || os.Geteuid() != 0 {
return nil, ""
}
u, err := lookupUser(name)
if err != nil {
return nil, fmt.Sprintf("there is no user %s, so engines run as root; `llmgw node add` makes it, "+
"or: useradd --system --shell /usr/sbin/nologin %s && usermod -aG video,render %s", name, name, name)
}
r, err := credentialFor(u, workDir)
if err != nil {
return nil, fmt.Sprintf("engines run as root: %v", err)
}
return r, ""
}
func credentialFor(u *user.User, workDir string) (*runAs, error) {
uid, err1 := strconv.ParseUint(u.Uid, 10, 32)
gid, err2 := strconv.ParseUint(u.Gid, 10, 32)
if err1 != nil || err2 != nil {
return nil, fmt.Errorf("user %s has ids %s:%s", u.Username, u.Uid, u.Gid)
}
var groups []uint32
if ids, err := u.GroupIds(); err == nil {
for _, g := range ids {
if n, err := strconv.ParseUint(g, 10, 32); err == nil {
groups = append(groups, uint32(n))
}
}
}
// A home of its own under the work directory, which is root's: caches
// (Triton, torch, Hugging Face) are written there and nowhere else.
home := filepath.Join(workDir, "engine-home")
if err := os.MkdirAll(home, 0o700); err != nil {
return nil, fmt.Errorf("making %s: %w", home, err)
}
if err := os.Chown(home, int(uid), int(gid)); err != nil {
return nil, fmt.Errorf("giving %s to %s: %w", home, u.Username, err)
}
// The work directory is 0750 root's; the engine has to pass through it
// to its home.
if err := os.Chmod(workDir, 0o751); err != nil {
return nil, err
}
return &runAs{name: u.Username, home: home,
cred: &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid), Groups: groups}}, nil
}
// env is what is added to an engine's environment when it runs as r: the
// last value of a key wins in exec.
func (r *runAs) env() []string {
return []string{"HOME=" + r.home, "XDG_CACHE_HOME=" + filepath.Join(r.home, ".cache"), "USER=" + r.name, "LOGNAME=" + r.name}
}
+51
View File
@@ -0,0 +1,51 @@
package supervisor
import (
"os"
"os/exec"
"os/user"
"path/filepath"
"slices"
"strconv"
"syscall"
"testing"
)
func TestAnEngineIsGivenItsUsersCredentialHomeAndCaches(t *testing.T) {
me, err := user.Current()
if err != nil {
t.Skip(err)
}
work := filepath.Join(t.TempDir(), "agent")
if err := os.MkdirAll(work, 0o750); err != nil {
t.Fatal(err)
}
r, err := credentialFor(me, work)
if err != nil {
t.Fatal(err)
}
uid, _ := strconv.Atoi(me.Uid)
if int(r.cred.Uid) != uid || r.home != filepath.Join(work, "engine-home") {
t.Fatalf("%+v", r)
}
if st, err := os.Stat(r.home); err != nil || st.Mode().Perm() != 0o700 {
t.Errorf("home: %v %v", st.Mode(), err)
}
s := &Supervisor{cfg: Config{WorkDir: work}, as: r}
c := exec.Command("true")
c.Env = []string{"HOME=/root", "PATH=/bin"}
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
s.dropTo(c)
if c.SysProcAttr.Credential == nil || c.Dir != r.home || !slices.Contains(c.Env, "HOME="+r.home) {
t.Errorf("dir %s env %v cred %v", c.Dir, c.Env, c.SysProcAttr.Credential)
}
}
func TestNotRootOrNoSuchUserLeavesEnginesAsTheAgent(t *testing.T) {
if os.Geteuid() == 0 {
t.Skip("run as root")
}
if r, why := resolveRunAs(DefaultRunAs, t.TempDir()); r != nil || why != "" {
t.Errorf("not root: %v %q", r, why)
}
}
+33 -1
View File
@@ -39,6 +39,9 @@ type Config struct {
// as ready. There is no external signal for a worker, so staying alive is
// the only evidence available.
AliveGrace time.Duration `koanf:"alive_grace"`
// RunAs is the user engines run as when the agent is root; empty is
// llmgw-engine, "root" keeps them root.
RunAs string `koanf:"run_as"`
}
// DefaultConfig returns sensible defaults.
@@ -50,6 +53,7 @@ func DefaultConfig() Config {
HealthInterval: 2 * time.Second,
RendezvousTimeout: 5 * time.Minute,
AliveGrace: 5 * time.Second,
RunAs: DefaultRunAs,
}
}
@@ -58,6 +62,9 @@ func (c *Config) normalize() {
if c.WorkDir == "" {
c.WorkDir = d.WorkDir
}
if c.RunAs == "" {
c.RunAs = d.RunAs
}
if c.LogLines <= 0 {
c.LogLines = d.LogLines
}
@@ -154,6 +161,9 @@ type Supervisor struct {
mu sync.Mutex
shards map[string]*Shard
// as is who engines run as; nil is as the agent.
as *runAs
}
// New builds a supervisor.
@@ -165,7 +175,16 @@ func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error
if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
return nil, fmt.Errorf("work dir: %w", err)
}
return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
s := &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}
as, why := resolveRunAs(cfg.RunAs, cfg.WorkDir)
if why != "" {
log.Warn(why)
}
if as != nil {
log.Info("engines run as their own user", "user", as.name)
}
s.as = as
return s, nil
}
// ErrAlreadyRunning means a shard with that ID is already supervised.
@@ -261,6 +280,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
// Engines spawn workers; signalling only the parent leaves those workers
// holding GPU memory.
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
s.dropTo(c)
// CommandContext kills only the leader, so replace that with a group kill.
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
@@ -298,6 +318,7 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
proc.Env = c.Env
proc.Dir = s.cfg.WorkDir
proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
s.dropTo(proc)
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
stdout, err := proc.StdoutPipe()
@@ -779,3 +800,14 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
}
return nil
}
// dropTo makes a command run as the engines' user, when there is one: its
// credential, its home as the working directory, and its caches there.
func (s *Supervisor) dropTo(c *exec.Cmd) {
if s.as == nil {
return
}
c.SysProcAttr.Credential = s.as.cred
c.Dir = s.as.home
c.Env = append(c.Env, s.as.env()...)
}