Slurm's jobs in containers: pyxis and enroot, installed by the agents
CI / test (push) Failing after 6m53s

slurm.containers (ctl slurm containers on) has every Slurm machine's agent
install enroot from NVIDIA's release and build pyxis against its own
Slurm, both pinned and checked against their digests, and on GPU machines
nvidia-container-cli from NVIDIA's repository. enroot keeps each user's
containers under /run/enroot and /var/lib/enroot, needing no login
session, with the shared path in every container. The leader has Slurm
load pyxis only once every machine reports it: plugstack.conf's
"required" fails every job on a machine without the plugin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-14 21:31:20 +01:00
1 parent c3a75dcf3f
commit 57f0fd2ed0
18 files changed
+737 -9

No files matched your search

+32 -6
View File
@@ -5314,7 +5314,10 @@ type Slurm struct {
Users []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"`
// make_homes has this machine make the users' home directories: the one
// that exports the shared path, else the controller.
MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
// containers has the machine install pyxis and enroot, so jobs may run
// in containers (srun --container-image), and set them up.
Containers bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -5426,6 +5429,13 @@ func (x *Slurm) GetMakeHomes() bool {
return false
}
func (x *Slurm) GetContainers() bool {
if x != nil {
return x.Containers
}
return false
}
// SlurmShared is where the Slurm cluster's shared files come from.
type SlurmShared struct {
state protoimpl.MessageState `protogen:"open.v1"`
@@ -5610,7 +5620,10 @@ type SlurmStatus struct {
// "exported", "missing", or empty when there is none.
Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"`
// users is how many of llmgw's users this machine has.
Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
// containers is what this machine has for jobs in containers, as
// "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
Containers string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"`
unknownFields protoimpl.UnknownFields
sizeCache protoimpl.SizeCache
}
@@ -5729,6 +5742,13 @@ func (x *SlurmStatus) GetUsers() int32 {
return 0
}
func (x *SlurmStatus) GetContainers() string {
if x != nil {
return x.Containers
}
return ""
}
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
const file_llmgw_v1_agent_proto_rawDesc = "" +
@@ -6234,7 +6254,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" +
"\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" +
"\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" +
"\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" +
"\x04last\x18\x05 \x01(\bR\x04last\"\xc9\x03\n" +
"\x05Slurm\x12\x1e\n" +
"\n" +
"controller\x18\x01 \x01(\bR\n" +
@@ -6251,7 +6271,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x05users\x18\n" +
" \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" +
"\n" +
"make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" +
"make_homes\x18\v \x01(\bR\tmakeHomes\x12\x1e\n" +
"\n" +
"containers\x18\f \x01(\bR\n" +
"containers\x1a8\n" +
"\n" +
"FilesEntry\x12\x10\n" +
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
@@ -6266,7 +6289,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" +
"\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" +
"\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" +
"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" +
"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\x8b\x03\n" +
"\vSlurmStatus\x12\x18\n" +
"\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" +
"\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" +
@@ -6280,7 +6303,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
"\x10slurmdbd_running\x18\n" +
" \x01(\bR\x0fslurmdbdRunning\x12\x16\n" +
"\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" +
"\x05users\x18\f \x01(\x05R\x05users2O\n" +
"\x05users\x18\f \x01(\x05R\x05users\x12\x1e\n" +
"\n" +
"containers\x18\r \x01(\tR\n" +
"containers2O\n" +
"\fAgentService\x12?\n" +
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
"\fcom.llmgw.v1B\n" +
+6
View File
@@ -1020,6 +1020,9 @@ message Slurm {
// make_homes has this machine make the users' home directories: the one
// that exports the shared path, else the controller.
bool make_homes = 11;
// containers has the machine install pyxis and enroot, so jobs may run
// in containers (srun --container-image), and set them up.
bool containers = 12;
}
// SlurmShared is where the Slurm cluster's shared files come from.
@@ -1077,4 +1080,7 @@ message SlurmStatus {
string shared = 11;
// users is how many of llmgw's users this machine has.
int32 users = 12;
// containers is what this machine has for jobs in containers, as
// "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
string containers = 13;
}