Slurm's jobs in containers: pyxis and enroot, installed by the agents
CI / test (push) Failing after 6m53s
CI / test (push) Failing after 6m53s
slurm.containers (ctl slurm containers on) has every Slurm machine's agent install enroot from NVIDIA's release and build pyxis against its own Slurm, both pinned and checked against their digests, and on GPU machines nvidia-container-cli from NVIDIA's repository. enroot keeps each user's containers under /run/enroot and /var/lib/enroot, needing no login session, with the shared path in every container. The leader has Slurm load pyxis only once every machine reports it: plugstack.conf's "required" fails every job on a machine without the plugin. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
c3a75dcf3f
commit
57f0fd2ed0
18 files changed
+737
-9
No files matched your search
@@ -5314,7 +5314,10 @@ type Slurm struct {
|
||||
Users []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"`
|
||||
// make_homes has this machine make the users' home directories: the one
|
||||
// that exports the shared path, else the controller.
|
||||
MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
|
||||
MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
|
||||
// containers has the machine install pyxis and enroot, so jobs may run
|
||||
// in containers (srun --container-image), and set them up.
|
||||
Containers bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -5426,6 +5429,13 @@ func (x *Slurm) GetMakeHomes() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (x *Slurm) GetContainers() bool {
|
||||
if x != nil {
|
||||
return x.Containers
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SlurmShared is where the Slurm cluster's shared files come from.
|
||||
type SlurmShared struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
@@ -5610,7 +5620,10 @@ type SlurmStatus struct {
|
||||
// "exported", "missing", or empty when there is none.
|
||||
Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"`
|
||||
// users is how many of llmgw's users this machine has.
|
||||
Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
|
||||
Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
|
||||
// containers is what this machine has for jobs in containers, as
|
||||
// "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
|
||||
Containers string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
@@ -5729,6 +5742,13 @@ func (x *SlurmStatus) GetUsers() int32 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *SlurmStatus) GetContainers() string {
|
||||
if x != nil {
|
||||
return x.Containers
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
@@ -6234,7 +6254,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" +
|
||||
"\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" +
|
||||
"\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" +
|
||||
"\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" +
|
||||
"\x04last\x18\x05 \x01(\bR\x04last\"\xc9\x03\n" +
|
||||
"\x05Slurm\x12\x1e\n" +
|
||||
"\n" +
|
||||
"controller\x18\x01 \x01(\bR\n" +
|
||||
@@ -6251,7 +6271,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x05users\x18\n" +
|
||||
" \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" +
|
||||
"\n" +
|
||||
"make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" +
|
||||
"make_homes\x18\v \x01(\bR\tmakeHomes\x12\x1e\n" +
|
||||
"\n" +
|
||||
"containers\x18\f \x01(\bR\n" +
|
||||
"containers\x1a8\n" +
|
||||
"\n" +
|
||||
"FilesEntry\x12\x10\n" +
|
||||
"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
|
||||
@@ -6266,7 +6289,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" +
|
||||
"\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" +
|
||||
"\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" +
|
||||
"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" +
|
||||
"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\x8b\x03\n" +
|
||||
"\vSlurmStatus\x12\x18\n" +
|
||||
"\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" +
|
||||
"\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" +
|
||||
@@ -6280,7 +6303,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x10slurmdbd_running\x18\n" +
|
||||
" \x01(\bR\x0fslurmdbdRunning\x12\x16\n" +
|
||||
"\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" +
|
||||
"\x05users\x18\f \x01(\x05R\x05users2O\n" +
|
||||
"\x05users\x18\f \x01(\x05R\x05users\x12\x1e\n" +
|
||||
"\n" +
|
||||
"containers\x18\r \x01(\tR\n" +
|
||||
"containers2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
|
||||
@@ -1020,6 +1020,9 @@ message Slurm {
|
||||
// make_homes has this machine make the users' home directories: the one
|
||||
// that exports the shared path, else the controller.
|
||||
bool make_homes = 11;
|
||||
// containers has the machine install pyxis and enroot, so jobs may run
|
||||
// in containers (srun --container-image), and set them up.
|
||||
bool containers = 12;
|
||||
}
|
||||
|
||||
// SlurmShared is where the Slurm cluster's shared files come from.
|
||||
@@ -1077,4 +1080,7 @@ message SlurmStatus {
|
||||
string shared = 11;
|
||||
// users is how many of llmgw's users this machine has.
|
||||
int32 users = 12;
|
||||
// containers is what this machine has for jobs in containers, as
|
||||
// "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
|
||||
string containers = 13;
|
||||
}
|
||||
Reference in new issue
Block a user