Run engines as llmgw-engine rather than root, and make the user when a node is added
The supervisor drops to the run_as user (default llmgw-engine) when the agent is root: its credential and groups, a 0700 home under the work directory as the working directory, HOME and XDG_CACHE_HOME there. A node without the user keeps engines as root and logs why. node add creates the user in the video and render groups. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c19abe95e1
commit
2d47427cdc
7 files changed
+199
-1
No files matched your search
@@ -58,6 +58,12 @@ APIs change between minors without a deprecation period.
|
||||
writes or loads files, loads code, or moves what the agent sets (host,
|
||||
port, key, model). Both are refused where a model is defined and again on
|
||||
the node before the engine starts.
|
||||
- **Engines no longer run as root.** The agent starts them as the system
|
||||
user `llmgw-engine` (in the video and render groups for the GPUs), with a
|
||||
home and caches of its own under the agent's work directory. `llmgw node
|
||||
add` makes the user; a node without it keeps running engines as root and
|
||||
the agent says so in its log. `supervisor.run_as` in agent.yaml names
|
||||
another user, or `root`.
|
||||
|
||||
### Everything that can be changed on a running cluster, from the console
|
||||
|
||||
|
||||
@@ -245,6 +245,7 @@ rather than repeated.
|
||||
| `health_interval` | `time.Duration` | [supervisor.Config](#supervisorconfig) |
|
||||
| `rendezvous_timeout` | `time.Duration` | [supervisor.Config](#supervisorconfig) |
|
||||
| `alive_grace` | `time.Duration` | [supervisor.Config](#supervisorconfig) |
|
||||
| `run_as` | `string` | [supervisor.Config](#supervisorconfig) |
|
||||
| `ca` | `string` | [tlsconf.Config](#tlsconfconfig) |
|
||||
| `cert` | `string` | [tlsconf.Config](#tlsconfconfig) |
|
||||
| `key` | `string` | [tlsconf.Config](#tlsconfconfig) |
|
||||
@@ -1895,6 +1896,13 @@ AliveGrace is how long a non-serving rank must stay up before it counts
|
||||
as ready. There is no external signal for a worker, so staying alive is
|
||||
the only evidence available.
|
||||
|
||||
### `run_as`
|
||||
|
||||
`string`
|
||||
|
||||
RunAs is the user engines run as when the agent is root; empty is
|
||||
llmgw-engine, "root" keeps them root.
|
||||
|
||||
## tlsconf.Config
|
||||
|
||||
### `ca`
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package supervisor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// Engines run as their own user, not as the agent's root: an engine reads a
|
||||
// model's files and answers requests, and a model's files are whatever a
|
||||
// registry published. Root in that process would be root on the node for
|
||||
// anything that can make an engine misbehave — a malformed file, a template,
|
||||
// a flag nobody thought of.
|
||||
//
|
||||
// The user is DefaultRunAs, made by `llmgw node add`, in the video and render
|
||||
// groups for the GPU devices. A node without it keeps running engines as
|
||||
// root, and says so, so an existing fleet is not broken by an upgrade.
|
||||
|
||||
// DefaultRunAs is the user engines run as.
|
||||
const DefaultRunAs = "llmgw-engine"
|
||||
|
||||
// runAs is how an engine process is started as another user: its
|
||||
// credential, a working directory it owns, and the environment that points
|
||||
// its caches there.
|
||||
type runAs struct {
|
||||
name string
|
||||
cred *syscall.Credential
|
||||
home string
|
||||
}
|
||||
|
||||
// lookupUser is user.Lookup, replaced by tests.
|
||||
var lookupUser = user.Lookup
|
||||
|
||||
// resolveRunAs finds the user engines run as. nil means as this process —
|
||||
// when it is not root, there is nobody to drop to; when the user does not
|
||||
// exist, why is returned.
|
||||
func resolveRunAs(name, workDir string) (*runAs, string) {
|
||||
if name == "" || name == "root" || os.Geteuid() != 0 {
|
||||
return nil, ""
|
||||
}
|
||||
u, err := lookupUser(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("there is no user %s, so engines run as root; `llmgw node add` makes it, "+
|
||||
"or: useradd --system --shell /usr/sbin/nologin %s && usermod -aG video,render %s", name, name, name)
|
||||
}
|
||||
r, err := credentialFor(u, workDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Sprintf("engines run as root: %v", err)
|
||||
}
|
||||
return r, ""
|
||||
}
|
||||
|
||||
func credentialFor(u *user.User, workDir string) (*runAs, error) {
|
||||
uid, err1 := strconv.ParseUint(u.Uid, 10, 32)
|
||||
gid, err2 := strconv.ParseUint(u.Gid, 10, 32)
|
||||
if err1 != nil || err2 != nil {
|
||||
return nil, fmt.Errorf("user %s has ids %s:%s", u.Username, u.Uid, u.Gid)
|
||||
}
|
||||
var groups []uint32
|
||||
if ids, err := u.GroupIds(); err == nil {
|
||||
for _, g := range ids {
|
||||
if n, err := strconv.ParseUint(g, 10, 32); err == nil {
|
||||
groups = append(groups, uint32(n))
|
||||
}
|
||||
}
|
||||
}
|
||||
// A home of its own under the work directory, which is root's: caches
|
||||
// (Triton, torch, Hugging Face) are written there and nowhere else.
|
||||
home := filepath.Join(workDir, "engine-home")
|
||||
if err := os.MkdirAll(home, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("making %s: %w", home, err)
|
||||
}
|
||||
if err := os.Chown(home, int(uid), int(gid)); err != nil {
|
||||
return nil, fmt.Errorf("giving %s to %s: %w", home, u.Username, err)
|
||||
}
|
||||
// The work directory is 0750 root's; the engine has to pass through it
|
||||
// to its home.
|
||||
if err := os.Chmod(workDir, 0o751); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &runAs{name: u.Username, home: home,
|
||||
cred: &syscall.Credential{Uid: uint32(uid), Gid: uint32(gid), Groups: groups}}, nil
|
||||
}
|
||||
|
||||
// env is what is added to an engine's environment when it runs as r: the
|
||||
// last value of a key wins in exec.
|
||||
func (r *runAs) env() []string {
|
||||
return []string{"HOME=" + r.home, "XDG_CACHE_HOME=" + filepath.Join(r.home, ".cache"), "USER=" + r.name, "LOGNAME=" + r.name}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package supervisor
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAnEngineIsGivenItsUsersCredentialHomeAndCaches(t *testing.T) {
|
||||
me, err := user.Current()
|
||||
if err != nil {
|
||||
t.Skip(err)
|
||||
}
|
||||
work := filepath.Join(t.TempDir(), "agent")
|
||||
if err := os.MkdirAll(work, 0o750); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r, err := credentialFor(me, work)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
uid, _ := strconv.Atoi(me.Uid)
|
||||
if int(r.cred.Uid) != uid || r.home != filepath.Join(work, "engine-home") {
|
||||
t.Fatalf("%+v", r)
|
||||
}
|
||||
if st, err := os.Stat(r.home); err != nil || st.Mode().Perm() != 0o700 {
|
||||
t.Errorf("home: %v %v", st.Mode(), err)
|
||||
}
|
||||
s := &Supervisor{cfg: Config{WorkDir: work}, as: r}
|
||||
c := exec.Command("true")
|
||||
c.Env = []string{"HOME=/root", "PATH=/bin"}
|
||||
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
s.dropTo(c)
|
||||
if c.SysProcAttr.Credential == nil || c.Dir != r.home || !slices.Contains(c.Env, "HOME="+r.home) {
|
||||
t.Errorf("dir %s env %v cred %v", c.Dir, c.Env, c.SysProcAttr.Credential)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotRootOrNoSuchUserLeavesEnginesAsTheAgent(t *testing.T) {
|
||||
if os.Geteuid() == 0 {
|
||||
t.Skip("run as root")
|
||||
}
|
||||
if r, why := resolveRunAs(DefaultRunAs, t.TempDir()); r != nil || why != "" {
|
||||
t.Errorf("not root: %v %q", r, why)
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,9 @@ type Config struct {
|
||||
// as ready. There is no external signal for a worker, so staying alive is
|
||||
// the only evidence available.
|
||||
AliveGrace time.Duration `koanf:"alive_grace"`
|
||||
// RunAs is the user engines run as when the agent is root; empty is
|
||||
// llmgw-engine, "root" keeps them root.
|
||||
RunAs string `koanf:"run_as"`
|
||||
}
|
||||
|
||||
// DefaultConfig returns sensible defaults.
|
||||
@@ -50,6 +53,7 @@ func DefaultConfig() Config {
|
||||
HealthInterval: 2 * time.Second,
|
||||
RendezvousTimeout: 5 * time.Minute,
|
||||
AliveGrace: 5 * time.Second,
|
||||
RunAs: DefaultRunAs,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +62,9 @@ func (c *Config) normalize() {
|
||||
if c.WorkDir == "" {
|
||||
c.WorkDir = d.WorkDir
|
||||
}
|
||||
if c.RunAs == "" {
|
||||
c.RunAs = d.RunAs
|
||||
}
|
||||
if c.LogLines <= 0 {
|
||||
c.LogLines = d.LogLines
|
||||
}
|
||||
@@ -154,6 +161,9 @@ type Supervisor struct {
|
||||
|
||||
mu sync.Mutex
|
||||
shards map[string]*Shard
|
||||
|
||||
// as is who engines run as; nil is as the agent.
|
||||
as *runAs
|
||||
}
|
||||
|
||||
// New builds a supervisor.
|
||||
@@ -165,7 +175,16 @@ func New(cfg Config, health HealthChecker, log *slog.Logger) (*Supervisor, error
|
||||
if err := os.MkdirAll(cfg.WorkDir, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("work dir: %w", err)
|
||||
}
|
||||
return &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}, nil
|
||||
s := &Supervisor{cfg: cfg, log: log, health: health, shards: map[string]*Shard{}}
|
||||
as, why := resolveRunAs(cfg.RunAs, cfg.WorkDir)
|
||||
if why != "" {
|
||||
log.Warn(why)
|
||||
}
|
||||
if as != nil {
|
||||
log.Info("engines run as their own user", "user", as.name)
|
||||
}
|
||||
s.as = as
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// ErrAlreadyRunning means a shard with that ID is already supervised.
|
||||
@@ -261,6 +280,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
// Engines spawn workers; signalling only the parent leaves those workers
|
||||
// holding GPU memory.
|
||||
c.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
s.dropTo(c)
|
||||
// CommandContext kills only the leader, so replace that with a group kill.
|
||||
c.Cancel = func() error { return killGroup(c.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
@@ -298,6 +318,7 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
|
||||
proc.Env = c.Env
|
||||
proc.Dir = s.cfg.WorkDir
|
||||
proc.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
s.dropTo(proc)
|
||||
proc.Cancel = func() error { return killGroup(proc.Process.Pid, syscall.SIGKILL) }
|
||||
|
||||
stdout, err := proc.StdoutPipe()
|
||||
@@ -779,3 +800,14 @@ func (s *Supervisor) KillOrphan(o Orphan) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropTo makes a command run as the engines' user, when there is one: its
|
||||
// credential, its home as the working directory, and its caches there.
|
||||
func (s *Supervisor) dropTo(c *exec.Cmd) {
|
||||
if s.as == nil {
|
||||
return
|
||||
}
|
||||
c.SysProcAttr.Credential = s.as.cred
|
||||
c.Dir = s.as.home
|
||||
c.Env = append(c.Env, s.as.env()...)
|
||||
}
|
||||
@@ -311,6 +311,7 @@ var Reference = []Block{
|
||||
{Key: "health_interval", Type: "time.Duration", Doc: "HealthInterval is how often a starting shard is polled.", Nested: ""},
|
||||
{Key: "rendezvous_timeout", Type: "time.Duration", Doc: "RendezvousTimeout bounds the wait for a multi-node rendezvous point to\nstart listening.", Nested: ""},
|
||||
{Key: "alive_grace", Type: "time.Duration", Doc: "AliveGrace is how long a non-serving rank must stay up before it counts\nas ready. There is no external signal for a worker, so staying alive is\nthe only evidence available.", Nested: ""},
|
||||
{Key: "run_as", Type: "string", Doc: "RunAs is the user engines run as when the agent is root; empty is\nllmgw-engine, \"root\" keeps them root.", Nested: ""},
|
||||
}},
|
||||
{Package: "tlsconf", Name: "Config", Title: "tlsconf.Config", Doc: "", Fields: []Field{
|
||||
{Key: "ca", Type: "string", Doc: "CA is the bundle used to verify the other side. In a real deployment this\nis whichever authority issued the fleet's certificates.", Nested: ""},
|
||||
|
||||
@@ -688,6 +688,14 @@ func (r *Runner) checkEngines(ctx context.Context) []string {
|
||||
// --- start ---
|
||||
|
||||
func (r *Runner) start(ctx context.Context, _ *Result) error {
|
||||
// The user engines run as, so the agent drops root before starting one:
|
||||
// in the groups that own the GPU devices.
|
||||
if out, err := r.Remote.Run(ctx, `set -e
|
||||
id llmgw-engine >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin llmgw-engine
|
||||
for g in video render; do getent group "$g" >/dev/null && sudo usermod -aG "$g" llmgw-engine; done
|
||||
true`, nil); err != nil {
|
||||
r.emit(StepStart, Warn, "the user engines run as could not be made, so they run as root: "+lastLine(out), "")
|
||||
}
|
||||
r.emit(StepStart, Running, "starting the agent", "")
|
||||
if out, err := r.Remote.Run(ctx, `set -e
|
||||
sudo systemctl daemon-reload
|
||||
|
||||
Reference in new issue
Block a user