diff --git a/BACKLOG.md b/BACKLOG.md index 002b1ab..063e9c5 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2057,6 +2057,15 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t controller later with `ctl slurm setup`; batch output lands on the job's first node until there is shared storage; on shared CPUs the judge refuses the first request after a placement change (fail closed, ~10 s), and its advice repeats `--cores 4` to a judge that has 4. +- Shared files and users, built 2026-11-12: `slurm.shared` {path, nfs | server} — the site's export + mounted (a systemd mount unit, hard, nconnect=4), an llmgw node exporting the path itself + (/etc/exports.d, to the Slurm machines' addresses, root_squash; 2049 in the firewall), or the site's + own mount only looked for; `slurm.users: llmgw` — every active account a Linux user everywhere + (`slurm_users`: login from the e-mail, UID from 200000, never reused; homes made on the exporting + machine, else the controller; keys in /etc/llmgw/ssh-keys/%u through an sshd drop-in), SSH keys + from the portal's Profile or `ctl slurm users keys`. A user already on a machine with another UID + is left alone and said. The site has AD (not yet live): users from it, through sssd, are the next + step (`slurm.users` empty today leaves them to the site). - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm: each project an account (`sacctmgr add account`, parent its team and organization), each member a user of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a5321f..e8d66d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,14 @@ APIs change between minors without a deprecation period. machine alone (Slurm takes MySQL or MariaDB only). Slurm's ports (6817–6819, srun's 60001–63000) answer the cluster's networks only, as llmgw's own do. +- **Slurm's shared files and users.** `slurm.shared` (`ctl slurm shared`) is + one path on every Slurm machine: the site's NFS export mounted, an llmgw + node exporting it over NFS (what `server init --slurm` chooses, `/shared` + from the first machine), or a file system the site mounts. `slurm.users: + llmgw` (`ctl slurm users on`) makes each active account a Linux user on + every Slurm machine, the same login and UID (from 200000, never reused) + everywhere, its home on the shared path; people add SSH keys in the + portal's Profile (`ctl slurm users keys` for an operator). ### Changed diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 9913d5f..524ffc6 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5304,7 +5304,17 @@ type Slurm struct { Files map[string]string `protobuf:"bytes,6,rep,name=files,proto3" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // lent says Slurm may run jobs here now: resume the node if llmgw drained // it. Otherwise it is drained, its jobs left to finish. - Lent bool `protobuf:"varint,7,opt,name=lent,proto3" json:"lent,omitempty"` + Lent bool `protobuf:"varint,7,opt,name=lent,proto3" json:"lent,omitempty"` + // shared is the file system every machine of the Slurm cluster sees at + // the same path; unset or with no path, none. + Shared *SlurmShared `protobuf:"bytes,8,opt,name=shared,proto3" json:"shared,omitempty"` + // manage_users has llmgw keep the Linux users below on this machine, + // and take away those it made that are no longer among them. + ManageUsers bool `protobuf:"varint,9,opt,name=manage_users,json=manageUsers,proto3" json:"manage_users,omitempty"` + Users []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"` + // make_homes has this machine make the users' home directories: the one + // that exports the shared path, else the controller. + MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5388,6 +5398,188 @@ func (x *Slurm) GetLent() bool { return false } +func (x *Slurm) GetShared() *SlurmShared { + if x != nil { + return x.Shared + } + return nil +} + +func (x *Slurm) GetManageUsers() bool { + if x != nil { + return x.ManageUsers + } + return false +} + +func (x *Slurm) GetUsers() []*SlurmUser { + if x != nil { + return x.Users + } + return nil +} + +func (x *Slurm) GetMakeHomes() bool { + if x != nil { + return x.MakeHomes + } + return false +} + +// SlurmShared is where the Slurm cluster's shared files come from. +type SlurmShared struct { + state protoimpl.MessageState `protogen:"open.v1"` + // path is where every machine sees them, as /shared. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // nfs is an export, host:/path, to mount at path; empty when this + // machine exports path itself (serve) or the site mounts it. + Nfs string `protobuf:"bytes,2,opt,name=nfs,proto3" json:"nfs,omitempty"` + // serve has this machine export path over NFS to clients. + Serve bool `protobuf:"varint,3,opt,name=serve,proto3" json:"serve,omitempty"` + // clients are the addresses the export answers. + Clients []string `protobuf:"bytes,4,rep,name=clients,proto3" json:"clients,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SlurmShared) Reset() { + *x = SlurmShared{} + mi := &file_llmgw_v1_agent_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SlurmShared) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SlurmShared) ProtoMessage() {} + +func (x *SlurmShared) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[49] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SlurmShared.ProtoReflect.Descriptor instead. +func (*SlurmShared) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{49} +} + +func (x *SlurmShared) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *SlurmShared) GetNfs() string { + if x != nil { + return x.Nfs + } + return "" +} + +func (x *SlurmShared) GetServe() bool { + if x != nil { + return x.Serve + } + return false +} + +func (x *SlurmShared) GetClients() []string { + if x != nil { + return x.Clients + } + return nil +} + +// SlurmUser is a Linux user that runs jobs, the same on every machine. +type SlurmUser struct { + state protoimpl.MessageState `protogen:"open.v1"` + Login string `protobuf:"bytes,1,opt,name=login,proto3" json:"login,omitempty"` + // uid is also the user's own group's gid. + Uid uint32 `protobuf:"varint,2,opt,name=uid,proto3" json:"uid,omitempty"` + // name is the account's, for the GECOS field. + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Home string `protobuf:"bytes,4,opt,name=home,proto3" json:"home,omitempty"` + // ssh_keys are OpenSSH public keys, one a line of authorized_keys. + SshKeys []string `protobuf:"bytes,5,rep,name=ssh_keys,json=sshKeys,proto3" json:"ssh_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SlurmUser) Reset() { + *x = SlurmUser{} + mi := &file_llmgw_v1_agent_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SlurmUser) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SlurmUser) ProtoMessage() {} + +func (x *SlurmUser) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[50] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SlurmUser.ProtoReflect.Descriptor instead. +func (*SlurmUser) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{50} +} + +func (x *SlurmUser) GetLogin() string { + if x != nil { + return x.Login + } + return "" +} + +func (x *SlurmUser) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *SlurmUser) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SlurmUser) GetHome() string { + if x != nil { + return x.Home + } + return "" +} + +func (x *SlurmUser) GetSshKeys() []string { + if x != nil { + return x.SshKeys + } + return nil +} + // SlurmStatus is what this node's Slurm daemons are doing, sent when it // changes and every few minutes while the node has a part in Slurm. type SlurmStatus struct { @@ -5414,13 +5606,18 @@ type SlurmStatus struct { // slurmdbd_running is the accounting daemon on the controller, which keeps // the jobs' records in a MariaDB beside it. SlurmdbdRunning bool `protobuf:"varint,10,opt,name=slurmdbd_running,json=slurmdbdRunning,proto3" json:"slurmdbd_running,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // shared is the shared path as this machine has it: "mounted", + // "exported", "missing", or empty when there is none. + Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"` + // users is how many of llmgw's users this machine has. + Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SlurmStatus) Reset() { *x = SlurmStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[49] + mi := &file_llmgw_v1_agent_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5432,7 +5629,7 @@ func (x *SlurmStatus) String() string { func (*SlurmStatus) ProtoMessage() {} func (x *SlurmStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[49] + mi := &file_llmgw_v1_agent_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5445,7 +5642,7 @@ func (x *SlurmStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use SlurmStatus.ProtoReflect.Descriptor instead. func (*SlurmStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{49} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{51} } func (x *SlurmStatus) GetVersion() string { @@ -5518,6 +5715,20 @@ func (x *SlurmStatus) GetSlurmdbdRunning() bool { return false } +func (x *SlurmStatus) GetShared() string { + if x != nil { + return x.Shared + } + return "" +} + +func (x *SlurmStatus) GetUsers() int32 { + if x != nil { + return x.Users + } + return 0 +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -6023,7 +6234,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" + "\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" + - "\x04last\x18\x05 \x01(\bR\x04last\"\x8d\x02\n" + + "\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" + "\x05Slurm\x12\x1e\n" + "\n" + "controller\x18\x01 \x01(\bR\n" + @@ -6034,11 +6245,28 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "confServer\x12\x1b\n" + "\tmunge_key\x18\x05 \x01(\fR\bmungeKey\x120\n" + "\x05files\x18\x06 \x03(\v2\x1a.llmgw.v1.Slurm.FilesEntryR\x05files\x12\x12\n" + - "\x04lent\x18\a \x01(\bR\x04lent\x1a8\n" + + "\x04lent\x18\a \x01(\bR\x04lent\x12-\n" + + "\x06shared\x18\b \x01(\v2\x15.llmgw.v1.SlurmSharedR\x06shared\x12!\n" + + "\fmanage_users\x18\t \x01(\bR\vmanageUsers\x12)\n" + + "\x05users\x18\n" + + " \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" + + "\n" + + "make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbd\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"c\n" + + "\vSlurmShared\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x10\n" + + "\x03nfs\x18\x02 \x01(\tR\x03nfs\x12\x14\n" + + "\x05serve\x18\x03 \x01(\bR\x05serve\x12\x18\n" + + "\aclients\x18\x04 \x03(\tR\aclients\"v\n" + + "\tSlurmUser\x12\x14\n" + + "\x05login\x18\x01 \x01(\tR\x05login\x12\x10\n" + + "\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" + + "\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" + + "\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" + + "\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" + "\vSlurmStatus\x12\x18\n" + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" + "\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" + @@ -6050,7 +6278,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05error\x18\b \x01(\tR\x05error\x12!\n" + "\ffiles_sha256\x18\t \x01(\tR\vfilesSha256\x12)\n" + "\x10slurmdbd_running\x18\n" + - " \x01(\bR\x0fslurmdbdRunning2O\n" + + " \x01(\bR\x0fslurmdbdRunning\x12\x16\n" + + "\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" + + "\x05users\x18\f \x01(\x05R\x05users2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + @@ -6069,7 +6299,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 59) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 61) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -6121,16 +6351,18 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*TrainStatus)(nil), // 47: llmgw.v1.TrainStatus (*TrainArtifact)(nil), // 48: llmgw.v1.TrainArtifact (*Slurm)(nil), // 49: llmgw.v1.Slurm - (*SlurmStatus)(nil), // 50: llmgw.v1.SlurmStatus - nil, // 51: llmgw.v1.ControlMessage.TraceEntry - nil, // 52: llmgw.v1.Register.LabelsEntry - nil, // 53: llmgw.v1.Register.EngineVersionsEntry - nil, // 54: llmgw.v1.GPU.NicPathsEntry - nil, // 55: llmgw.v1.IBPort.CountersEntry - nil, // 56: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 57: llmgw.v1.EngineWant.OfflineSha256Entry - nil, // 58: llmgw.v1.EngineProfile.EnvEntry - nil, // 59: llmgw.v1.Slurm.FilesEntry + (*SlurmShared)(nil), // 50: llmgw.v1.SlurmShared + (*SlurmUser)(nil), // 51: llmgw.v1.SlurmUser + (*SlurmStatus)(nil), // 52: llmgw.v1.SlurmStatus + nil, // 53: llmgw.v1.ControlMessage.TraceEntry + nil, // 54: llmgw.v1.Register.LabelsEntry + nil, // 55: llmgw.v1.Register.EngineVersionsEntry + nil, // 56: llmgw.v1.GPU.NicPathsEntry + nil, // 57: llmgw.v1.IBPort.CountersEntry + nil, // 58: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 59: llmgw.v1.EngineWant.OfflineSha256Entry + nil, // 60: llmgw.v1.EngineProfile.EnvEntry + nil, // 61: llmgw.v1.Slurm.FilesEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 8, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -6143,7 +6375,7 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 2, // 7: llmgw.v1.AgentMessage.kv_events:type_name -> llmgw.v1.KVEvents 47, // 8: llmgw.v1.AgentMessage.train_status:type_name -> llmgw.v1.TrainStatus 48, // 9: llmgw.v1.AgentMessage.train_artifact:type_name -> llmgw.v1.TrainArtifact - 50, // 10: llmgw.v1.AgentMessage.slurm_status:type_name -> llmgw.v1.SlurmStatus + 52, // 10: llmgw.v1.AgentMessage.slurm_status:type_name -> llmgw.v1.SlurmStatus 30, // 11: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard 33, // 12: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard 34, // 13: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard @@ -6161,11 +6393,11 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 45, // 25: llmgw.v1.ControlMessage.train_start:type_name -> llmgw.v1.TrainStart 46, // 26: llmgw.v1.ControlMessage.train_cancel:type_name -> llmgw.v1.TrainCancel 49, // 27: llmgw.v1.ControlMessage.slurm:type_name -> llmgw.v1.Slurm - 51, // 28: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry - 52, // 29: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 53, // 28: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry + 54, // 29: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 12, // 30: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 14, // 31: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 53, // 32: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 55, // 32: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 11, // 33: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 15, // 34: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 10, // 35: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage @@ -6173,15 +6405,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 21, // 37: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX 0, // 38: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State 27, // 39: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute - 54, // 40: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 56, // 40: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry 13, // 41: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice 18, // 42: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad 17, // 43: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 55, // 44: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 57, // 44: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry 20, // 45: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager 10, // 46: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 15, // 47: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 56, // 48: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 58, // 48: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry 26, // 49: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus 16, // 50: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad 19, // 51: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort @@ -6189,21 +6421,23 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 22, // 53: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState 21, // 54: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX 25, // 55: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 57, // 56: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry + 59, // 56: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry 0, // 57: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State 29, // 58: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress 27, // 59: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute 31, // 60: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 58, // 61: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 60, // 61: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry 44, // 62: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter - 59, // 63: llmgw.v1.Slurm.files:type_name -> llmgw.v1.Slurm.FilesEntry - 1, // 64: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 3, // 65: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 65, // [65:66] is the sub-list for method output_type - 64, // [64:65] is the sub-list for method input_type - 64, // [64:64] is the sub-list for extension type_name - 64, // [64:64] is the sub-list for extension extendee - 0, // [0:64] is the sub-list for field type_name + 61, // 63: llmgw.v1.Slurm.files:type_name -> llmgw.v1.Slurm.FilesEntry + 50, // 64: llmgw.v1.Slurm.shared:type_name -> llmgw.v1.SlurmShared + 51, // 65: llmgw.v1.Slurm.users:type_name -> llmgw.v1.SlurmUser + 1, // 66: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 3, // 67: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 67, // [67:68] is the sub-list for method output_type + 66, // [66:67] is the sub-list for method input_type + 66, // [66:66] is the sub-list for extension type_name + 66, // [66:66] is the sub-list for extension extendee + 0, // [0:66] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -6249,7 +6483,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 59, + NumMessages: 61, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 45318a2..29bbc08 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1010,6 +1010,41 @@ message Slurm { // lent says Slurm may run jobs here now: resume the node if llmgw drained // it. Otherwise it is drained, its jobs left to finish. bool lent = 7; + // shared is the file system every machine of the Slurm cluster sees at + // the same path; unset or with no path, none. + SlurmShared shared = 8; + // manage_users has llmgw keep the Linux users below on this machine, + // and take away those it made that are no longer among them. + bool manage_users = 9; + repeated SlurmUser users = 10; + // make_homes has this machine make the users' home directories: the one + // that exports the shared path, else the controller. + bool make_homes = 11; +} + +// SlurmShared is where the Slurm cluster's shared files come from. +message SlurmShared { + // path is where every machine sees them, as /shared. + string path = 1; + // nfs is an export, host:/path, to mount at path; empty when this + // machine exports path itself (serve) or the site mounts it. + string nfs = 2; + // serve has this machine export path over NFS to clients. + bool serve = 3; + // clients are the addresses the export answers. + repeated string clients = 4; +} + +// SlurmUser is a Linux user that runs jobs, the same on every machine. +message SlurmUser { + string login = 1; + // uid is also the user's own group's gid. + uint32 uid = 2; + // name is the account's, for the GECOS field. + string name = 3; + string home = 4; + // ssh_keys are OpenSSH public keys, one a line of authorized_keys. + repeated string ssh_keys = 5; } // SlurmStatus is what this node's Slurm daemons are doing, sent when it @@ -1037,4 +1072,9 @@ message SlurmStatus { // slurmdbd_running is the accounting daemon on the controller, which keeps // the jobs' records in a MariaDB beside it. bool slurmdbd_running = 10; + // shared is the shared path as this machine has it: "mounted", + // "exported", "missing", or empty when there is none. + string shared = 11; + // users is how many of llmgw's users this machine has. + int32 users = 12; } diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index a8ba48e..0a2df86 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -5109,6 +5109,89 @@ } } }, + "/api/v1/slurm/users": { + "get": { + "operationId": "getSlurmUsers", + "summary": "The Linux users made from llmgw's accounts for the Slurm cluster", + "description": "Every active account, with its login, UID and SSH keys; an account is given its login and UID the first time it is listed. The machines have them when slurm.users is \"llmgw\".", + "tags": [ + "slurm" + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/slurmsvc.User" + } + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/api/v1/slurm/users/{user}/ssh-keys": { + "put": { + "operationId": "putSlurmUsersUserSshKeys", + "summary": "Replace an account's SSH public keys", + "description": "{user} is an ID or e-mail. keys is OpenSSH public keys, one a line, without options; empty takes them all away. The machines have them within a minute.", + "tags": [ + "slurm" + ], + "parameters": [ + { + "name": "user", + "in": "path", + "required": true, + "description": "The person's ID or email.", + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/Actor" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SlurmKeysRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/slurmsvc.User" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/summary": { "get": { "operationId": "getSummary", @@ -9401,6 +9484,15 @@ } } }, + "SlurmKeysRequest": { + "type": "object", + "description": "SlurmKeysRequest is a person's SSH public keys, one a line, as authorized_keys has them; empty takes them all away.", + "properties": { + "keys": { + "type": "string" + } + } + }, "StoreMember": { "type": "object", "description": "StoreMember is one member of a replicated store, as its Patroni reports it.", @@ -10785,6 +10877,24 @@ } } }, + "config.SharedStorage": { + "type": "object", + "description": "SharedStorage is where the Slurm cluster's shared files come from: the site's NFS export (NFS), an NFS export llmgw keeps on one of its nodes (Server), or, with neither, a file system the site mounts itself (Lustre, BeeGFS, GPFS), which llmgw only looks for.", + "properties": { + "Path": { + "type": "string", + "description": "Path is where every machine sees it, as /shared." + }, + "NFS": { + "type": "string", + "description": "NFS is the site's export, host:/path, mounted at Path on every machine of the Slurm cluster." + }, + "Server": { + "type": "string", + "description": "Server is the llmgw node, the controller or one of the nodes, that exports Path over NFS to the others: for a site with no storage of its own. One machine: not highly available, and no faster than its disk and network." + } + } + }, "control.Activity": { "type": "object", "description": "Activity is what a standby's router reports back: when it last sent each replica a request, and the on-demand models it was asked for. The leader evicts and loads by these, and without them a replica busy behind a standby looks idle to the one machine that decides.", @@ -12249,6 +12359,14 @@ "slurmdbd_running": { "type": "boolean" }, + "shared": { + "type": "string", + "description": "Shared is the shared path here: \"mounted\", \"exported\", \"missing\"." + }, + "users": { + "type": "integer", + "description": "Users is how many of llmgw's users the machine has." + }, "state": { "type": "string" }, @@ -12301,6 +12419,40 @@ "type": "string" }, "description": "Waiting are nodes left out of slurm.conf until they report their hardware." + }, + "shared": { + "$ref": "#/components/schemas/config.SharedStorage", + "description": "Shared is where the shared files are and come from; Users who keeps the users, and how many llmgw has." + }, + "users": { + "type": "string" + }, + "user_count": { + "type": "integer" + } + } + }, + "slurmsvc.User": { + "type": "object", + "description": "User is a Linux user made from an llmgw account.", + "properties": { + "login": { + "type": "string" + }, + "uid": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "ssh_keys": { + "type": "array", + "items": { + "type": "string" + } } } }, diff --git a/docs/console.md b/docs/console.md index cdd69a0..6baaf06 100644 --- a/docs/console.md +++ b/docs/console.md @@ -215,7 +215,10 @@ replicas off and then lets Slurm run jobs there; **Take back from Slurm…** starts no new job there and gives it back to llmgw when its jobs have ended. Both change `slurm.lent` in the cluster's settings, as `llmgw ctl slurm lend` and `reclaim` do. Turning Slurm on and choosing its nodes is `llmgw ctl slurm -setup`, or the Slurm part of the first install. +setup`, or the Slurm part of the first install. Above the nodes it says where +the shared files come from and how many users llmgw made; the SHARED column +is each node's (mounted, exported, missing). `llmgw ctl slurm shared` and +`llmgw ctl slurm users` change them. **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. diff --git a/docs/install.md b/docs/install.md index c56c4dd..00a19bb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -195,6 +195,25 @@ On a cluster installed without it, `llmgw ctl slurm setup --controller NODE networks only, as llmgw's own do; a firewall of the site's own between the nodes must let them through. +**Slurm's shared files and users.** Jobs need the same files on every +machine: homes, datasets, output, checkpoints. With `--slurm` this machine +exports `/shared` over NFS to every Slurm machine and they mount it +(`slurm.shared`); NFS's port 2049 answers the cluster's networks only, the +export the Slurm machines alone. That is one machine, not highly available +and no faster than its disk and network: enough for a few nodes. A site with +storage of its own uses it instead — `--slurm-shared HOST:/EXPORT` at +install, or `llmgw ctl slurm shared --path /shared --nfs HOST:/EXPORT` +later; Lustre, BeeGFS or GPFS the site mounts itself, and `llmgw ctl slurm +shared --path /lustre/hpc` only has llmgw look that it is there. Every +active llmgw account is a Linux user on every Slurm machine +(`slurm.users: llmgw`): its login from its e-mail address, a UID from +200000 up that is never given to anyone else, its home under +`/shared/home`. People add their SSH public keys in the portal's +**Profile** and log in to this machine to run `sbatch`; an account suspended +or deleted loses its user, its files stay. A site whose users come from its +directory (sssd against Active Directory or LDAP) passes `--slurm-users +site`, or later `llmgw ctl slurm users off`. + **Secure by default.** What init writes is the hardened configuration, not a demonstration one: @@ -3167,7 +3186,7 @@ loaded at boot by `llmgw-firewall.service`): | Machine | Ports, for the cluster only | | --- | --- | | control plane | 7443 (agents), metrics when not on loopback, 8081 (its mirror), engines and 9100 when it is a node, 2379/2380 when it holds the store's third etcd vote | -| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`) | +| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`); exporting Slurm's shared files, 2049 (NFS) | | store member | 5432, 8008, 2379, 2380, 9100 | | mirror | 8081, 8082, 9100 | diff --git a/docs/reference.md b/docs/reference.md index b6a0de5..0c0c506 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -401,6 +401,9 @@ rather than repeated. | `action` | `string` | [Service](#service) | | `timeout` | `time.Duration` | [Service](#service) | | `stopped` | `bool` | [ServingSettings](#servingsettings) | +| `path` | `string` | [SharedStorage](#sharedstorage) | +| `nfs` | `string` | [SharedStorage](#sharedstorage) | +| `server` | `string` | [SharedStorage](#sharedstorage) | | `bin` | `string` | [SimConfig](#simconfig) | | `enabled` | `bool` | [SlurmSettings](#slurmsettings) | | `cluster_name` | `string` | [SlurmSettings](#slurmsettings) | @@ -409,6 +412,8 @@ rather than repeated. | `all_nodes` | `bool` | [SlurmSettings](#slurmsettings) | | `lent` | `[]string` | [SlurmSettings](#slurmsettings) | | `partition` | `string` | [SlurmSettings](#slurmsettings) | +| `shared` | `SharedStorage` | [SlurmSettings](#slurmsettings) | +| `users` | `string` | [SlurmSettings](#slurmsettings) | | `dir` | `string` | [StrataConfig](#strataconfig) | | `python` | `string` | [StrataConfig](#strataconfig) | | `engine` | `string` | [StrataConfig](#strataconfig) | @@ -3488,6 +3493,30 @@ running and every model's definition as it is: llmgw stop sets it, llmgw start clears it, and the models come back as their catalog entries ask. In force on every control plane within seconds. +## SharedStorage + +### `path` + +`string` + +Path is where every machine sees it, as /shared. + +### `nfs` + +`string` + +NFS is the site's export, host:/path, mounted at Path on every +machine of the Slurm cluster. + +### `server` + +`string` + +Server is the llmgw node, the controller or one of the nodes, that +exports Path over NFS to the others: for a site with no storage of +its own. One machine: not highly available, and no faster than its +disk and network. + ## SimConfig ### `bin` @@ -3548,6 +3577,27 @@ given back to llmgw when its last job ends. Partition is the one partition every node is in. Empty is "gpu". +### `shared` + +`SharedStorage` + +Shared is the file system every machine of the Slurm cluster sees at +the same path: home directories, datasets, the jobs' output and +checkpoints. Empty path: none, and a batch job's output stays on the +node it ran on. + +Takes a [SharedStorage](#sharedstorage) block. + +### `users` + +`string` + +Users is who keeps the Linux users that run jobs, the same on every +machine of the Slurm cluster: "llmgw" makes one for each active llmgw +account (its login from the e-mail address, a UID of llmgw's from +200000 up, its SSH keys from the portal); empty leaves users to the +site (a directory through sssd, or by hand). + ## StrataConfig ### `dir` diff --git a/internal/agent/firewall.go b/internal/agent/firewall.go index bc9193b..fc5faf4 100644 --- a/internal/agent/firewall.go +++ b/internal/agent/firewall.go @@ -20,6 +20,10 @@ var slurmPorts = []firewall.Port{ {From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"}, } +// nfsPort is NFSv4's, on the machine that exports the Slurm cluster's +// shared files. +var nfsPort = firewall.Port{From: slurm.NFSPort, To: slurm.NFSPort, What: "nfs: the Slurm cluster's shared files"} + // applyFirewall keeps llmgw's firewall on this node as the cluster's // settings say: the engines' ports and the node exporter's for the networks // the node reaches its control planes on, and those named, or no table at @@ -69,6 +73,9 @@ func (a *Agent) applyFirewall(m *llmgwv1.Firewall) { if a.slurm != nil && a.slurm.HasPart() { ports = append(ports, slurmPorts...) } + if a.slurm != nil && a.slurm.Serves() { + ports = append(ports, nfsPort) + } changed, err := firewall.Install(ctx, r, firewall.Spec{Ports: ports, Allow: nets}) switch { case err != nil: @@ -79,11 +86,12 @@ func (a *Agent) applyFirewall(m *llmgwv1.Firewall) { } // slurmMessage is the leader's word on this node's part in Slurm. A node -// that gets one has the firewall applied again, with Slurm's ports. +// that gets one, or starts or stops exporting the shared files, has the +// firewall applied again, with Slurm's ports and NFS's. func (a *Agent) slurmMessage(m *llmgwv1.Slurm) { - had := a.slurm.HasPart() + had, served := a.slurm.HasPart(), a.slurm.Serves() a.slurm.Apply(m) - if had || !a.slurm.HasPart() { + if (had || !a.slurm.HasPart()) && served == a.slurm.Serves() { return } a.fwMu.Lock() diff --git a/internal/agent/slurmnode/shared.go b/internal/agent/slurmnode/shared.go new file mode 100644 index 0000000..0c84d89 --- /dev/null +++ b/internal/agent/slurmnode/shared.go @@ -0,0 +1,150 @@ +package slurmnode + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// What a machine says of the shared path. +const ( + SharedMounted = "mounted" + SharedExported = "exported" + SharedMissing = "missing" +) + +// shared keeps the shared path as want says: exported from here, mounted +// from an export, or looked for where the site mounts it. Answers how it +// is. +func (k *Keeper) shared(ctx context.Context, want *llmgwv1.Slurm) (string, error) { + sh := want.GetShared() + path := sh.GetPath() + k.unshare(ctx, path, sh.GetServe(), sh.GetNfs()) + if path == "" { + return "", nil + } + if !strings.HasPrefix(path, "/") || strings.Contains(path, "..") || strings.ContainsAny(path, " \n\t\"'\\$`") { + return SharedMissing, fmt.Errorf("the leader sent a path %q", path) + } + switch { + case sh.GetServe(): + if err := k.host.MkdirAll(path, 0o755, ""); err != nil { + return SharedMissing, err + } + changed, err := k.write(slurm.ExportsPath, []byte(slurm.Exports(path, sh.GetClients())), 0o644, "") + if err != nil { + return SharedMissing, err + } + if !k.active(ctx, "nfs-server") { + k.log.Info("NFS starts on this machine, exporting the Slurm cluster's shared files", "path", path) + if err := k.systemctl(ctx, "enable", "--now", "nfs-server"); err != nil { + return SharedMissing, err + } + changed = true + } + if changed { + if out, err := k.host.Run(ctx, nil, "exportfs", "-ra"); err != nil { + return SharedMissing, fmt.Errorf("exportfs -ra: %v: %s", err, lastLine(out)) + } + } + return SharedExported, nil + case sh.GetNfs() != "": + nfs := sh.GetNfs() + if strings.ContainsAny(nfs, " \n\t\"'\\$`") || !strings.Contains(nfs, ":/") { + return SharedMissing, fmt.Errorf("the leader sent an export %q", nfs) + } + if err := k.host.MkdirAll(path, 0o755, ""); err != nil { + return SharedMissing, err + } + unit := slurm.MountUnitName(path) + changed, err := k.write(filepath.Join(slurm.MountUnitDir, unit), []byte(slurm.MountUnit(nfs, path)), 0o644, "") + if err != nil { + return SharedMissing, err + } + if changed { + k.log.Info("the Slurm cluster's shared files are mounted here", "from", nfs, "at", path) + if err := k.systemctl(ctx, "daemon-reload"); err != nil { + return SharedMissing, err + } + if err := k.systemctl(ctx, "enable", unit); err != nil { + return SharedMissing, err + } + // From another export: a restart mounts the new one, and fails + // while a job still has files open on the old. + if err := k.systemctl(ctx, "restart", unit); err != nil { + return SharedMissing, err + } + } else if !k.active(ctx, unit) { + if err := k.systemctl(ctx, "start", unit); err != nil { + return SharedMissing, err + } + } + } + if _, err := k.host.Run(ctx, nil, "mountpoint", "-q", path); err != nil { + if sh.GetNfs() == "" { + return SharedMissing, fmt.Errorf("%s is not mounted here: the site mounts it (slurm.shared names no export)", path) + } + return SharedMissing, fmt.Errorf("%s is not mounted from %s", path, sh.GetNfs()) + } + return SharedMounted, nil +} + +// unshare takes away the export or the mount llmgw kept here that want no +// longer has: path is where the files are wanted now, serve and nfs how. +func (k *Keeper) unshare(ctx context.Context, path string, serve bool, nfs string) { + if _, err := k.host.ReadFile(slurm.ExportsPath); err == nil && (!serve || path == "") { + k.log.Info("this machine no longer exports the Slurm cluster's shared files") + _ = k.host.Remove(slurm.ExportsPath) + _, _ = k.host.Run(ctx, nil, "exportfs", "-ra") + } + out, err := k.host.Run(ctx, nil, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--type=mount") + if err != nil { + return + } + keep := "" + if path != "" && nfs != "" && !serve { + keep = slurm.MountUnitName(path) + } + for _, l := range strings.Split(string(out), "\n") { + f := strings.Fields(l) + if len(f) == 0 || f[0] == keep { + continue + } + file := filepath.Join(slurm.MountUnitDir, f[0]) + b, err := k.host.ReadFile(file) + if err != nil || !strings.HasPrefix(string(b), "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files.") { + continue + } + k.log.Info("the Slurm cluster's old shared mount is taken away", "unit", f[0]) + if err := k.systemctl(ctx, "disable", "--now", f[0]); err != nil { + k.log.Warn("could not unmount the old shared files", "unit", f[0], "err", err) + continue + } + _ = k.host.Remove(file) + _ = k.systemctl(ctx, "daemon-reload") + } +} + +// sharedPackages are what the shared path needs here, by the command that +// shows each is installed. +func sharedPackages(want *llmgwv1.Slurm, need map[string][2]string) { + switch sh := want.GetShared(); { + case sh.GetPath() == "": + case sh.GetServe(): + need["exportfs"] = [2]string{"nfs-kernel-server", "nfs-utils"} + case sh.GetNfs() != "": + need["mount.nfs"] = [2]string{"nfs-common", "nfs-utils"} + } +} + +// Serves is whether this machine exports the shared files, so NFS's port +// is kept for the cluster's networks. +func (k *Keeper) Serves() bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.want.GetShared().GetServe() && k.want.GetShared().GetPath() != "" +} diff --git a/internal/agent/slurmnode/shared_test.go b/internal/agent/slurmnode/shared_test.go new file mode 100644 index 0000000..3fd286b --- /dev/null +++ b/internal/agent/slurmnode/shared_test.go @@ -0,0 +1,124 @@ +package slurmnode + +import ( + "context" + "slices" + "strings" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +func ayse() *llmgwv1.SlurmUser { + return &llmgwv1.SlurmUser{Login: "ayse", Uid: 200000, Name: "Ayşe Y.", Home: "/shared/home/ayse", + SshKeys: []string{"ssh-ed25519 AAAA ayse@laptop"}} +} + +func TestTheServerExportsAndMakesUsersWithHomes(t *testing.T) { + h := newFake() + h.active["ssh"] = true + inst := &fakeInst{host: h} + k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) + m := nodeMsg(false) + m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Serve: true, Clients: []string{"10.0.0.22", "10.0.0.21"}} + m.ManageUsers, m.MakeHomes, m.Users = true, true, []*llmgwv1.SlurmUser{ayse()} + h.fail["test -d /shared/home/ayse"] = true + st := k.Reconcile(context.Background(), m) + if st.GetError() != "" { + t.Fatal(st.GetError()) + } + if st.GetShared() != SharedExported || st.GetUsers() != 1 { + t.Fatalf("status %v", st) + } + if !slices.Contains(inst.got["deb"], "nfs-kernel-server") { + t.Errorf("packages %v", inst.got) + } + if e := string(h.files[slurm.ExportsPath]); !strings.Contains(e, `"/shared" 10.0.0.21(`) { + t.Errorf("exports:\n%s", e) + } + for _, want := range []string{"systemctl enable --now nfs-server", "exportfs -ra", "groupadd -g 200000 ayse", + "useradd -u 200000 -g 200000 -M -d /shared/home/ayse -s /bin/bash -c Ayşe Y. ayse", + "install -d -m 0700 -o 200000 -g 200000 /shared/home/ayse", "sshd -t", "systemctl reload ssh"} { + if !slices.Contains(h.ran, want) { + t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n")) + } + } + if string(h.files[slurm.AuthorizedKeysDir+"/ayse"]) != "ssh-ed25519 AAAA ayse@laptop\n" || + string(h.files[k.usersMarker()]) != "ayse\n" || h.files[slurm.SSHDropIn] == nil { + t.Errorf("files: keys %q, marker %q", h.files[slurm.AuthorizedKeysDir+"/ayse"], h.files[k.usersMarker()]) + } + + // Again, made: nothing is run again. + h.outputs["getent passwd ayse"] = "ayse:x:200000:200000:Ayşe Y.:/shared/home/ayse:/bin/bash\n" + delete(h.fail, "test -d /shared/home/ayse") + h.ran = nil + k.Reconcile(context.Background(), m) + for _, c := range h.ran { + if strings.HasPrefix(c, "useradd") || strings.HasPrefix(c, "usermod") || strings.HasPrefix(c, "exportfs") || + strings.HasPrefix(c, "install") || c == "systemctl reload ssh" { + t.Errorf("ran %q again", c) + } + } + + // Gone from the accounts: taken away, its home left. + m.Users = nil + k.Reconcile(context.Background(), m) + if !slices.Contains(h.ran, "userdel ayse") || h.files[k.usersMarker()] != nil || h.files[slurm.SSHDropIn] != nil || + h.files[slurm.AuthorizedKeysDir+"/ayse"] != nil { + t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n")) + } + if slices.ContainsFunc(h.ran, func(c string) bool { return strings.HasPrefix(c, "rm") || strings.Contains(c, "userdel -r") }) { + t.Error("a home was removed") + } +} + +func TestAClientMountsAndLeavesOthersUsersAlone(t *testing.T) { + h := newFake() + inst := &fakeInst{host: h} + k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) + m := nodeMsg(false) + m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"} + bob := &llmgwv1.SlurmUser{Login: "bob", Uid: 200001, Home: "/shared/home/bob"} + m.ManageUsers, m.Users = true, []*llmgwv1.SlurmUser{ayse(), bob} + // The site's own ayse, with its own UID. + h.outputs["getent passwd ayse"] = "ayse:x:1001:1001::/home/ayse:/bin/bash\n" + st := k.Reconcile(context.Background(), m) + if st.GetShared() != SharedMounted || st.GetUsers() != 1 { + t.Fatalf("status %v", st) + } + if !strings.Contains(st.GetError(), "ayse is already a user here with UID 1001") { + t.Errorf("error %q", st.GetError()) + } + if u := string(h.files["/etc/systemd/system/shared.mount"]); !strings.Contains(u, "What=10.0.0.11:/shared") { + t.Fatalf("unit:\n%s", u) + } + for _, want := range []string{"systemctl enable shared.mount", "systemctl restart shared.mount", "mountpoint -q /shared", + "useradd -u 200001 -g 200001 -M -d /shared/home/bob -s /bin/bash -c bob"} { + if !slices.Contains(h.ran, want) { + t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n")) + } + } + if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "ayse") && !strings.HasPrefix(c, "getent") }) || + slices.ContainsFunc(h.ran, func(c string) bool { return strings.HasPrefix(c, "install -d") }) { + t.Errorf("touched the site's ayse, or made a home:\n%s", strings.Join(h.ran, "\n")) + } + if string(h.files[k.usersMarker()]) != "bob\n" { + t.Errorf("marker %q", h.files[k.usersMarker()]) + } + + // Not mounted: said. + h.fail["mountpoint -q /shared"] = true + if st := k.Reconcile(context.Background(), m); st.GetShared() != SharedMissing || + !strings.Contains(st.GetError(), "not mounted from 10.0.0.11:/shared") { + t.Errorf("status %v", st) + } + + // No shared path any more: the mount is taken away. + h.outputs["systemctl list-units --all --plain --no-legend --type=mount"] = "shared.mount loaded active mounted llmgw\n" + m.Shared = nil + k.Reconcile(context.Background(), m) + if !slices.Contains(h.ran, "systemctl disable --now shared.mount") || h.files["/etc/systemd/system/shared.mount"] != nil { + t.Errorf("mount kept:\n%s", strings.Join(h.ran, "\n")) + } +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 8f15ae6..6c5176b 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -210,6 +210,31 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl if err := k.munge(ctx, want.GetMungeKey()); err != nil { return fail("munge", err) } + // The shared files, then the users whose homes are on them. Neither + // keeps Slurm from running: what goes wrong is said, and Slurm kept. + var shErr, usErr error + st.Shared, shErr = k.shared(ctx, want) + if shErr != nil { + k.log.Warn("keeping the Slurm cluster's shared files on this node", "err", shErr) + } + if shErr == nil || !want.GetMakeHomes() || want.GetShared().GetPath() == "" { + st.Users, usErr = k.users(ctx, want) + } else { + // Homes made now would be made beside the shared files, not on them. + usErr = errors.New("waiting for the shared files, where the homes are") + } + if usErr != nil { + k.log.Warn("keeping the Slurm users on this node", "err", usErr) + } + defer func() { + switch { + case st.Error != "": + case shErr != nil: + st.Error = "shared files: " + shErr.Error() + case usErr != nil: + st.Error = "users: " + usErr.Error() + } + }() if want.GetController() { // Accounting first: slurmctld reports to it from its first start. // Without it slurmctld still runs jobs, a pass of backfill late @@ -252,6 +277,7 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error if want.GetNode() { need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"} } + sharedPackages(want, need) pkgs := map[string][]string{} for bin, p := range need { if k.installed[bin] { @@ -518,6 +544,10 @@ func (k *Keeper) stop(ctx context.Context) { } _ = k.host.Remove(DropIn) _ = k.systemctl(ctx, "daemon-reload") + k.unshare(ctx, "", false, "") + if err := k.removeUsers(ctx, k.madeUsers(), nil); err != nil { + k.log.Warn("could not take away every Slurm user llmgw made", "err", err) + } _ = k.host.Remove(k.marker) k.files, k.acctDone, k.clusterAdded = "", "", "" } diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index 2f70c0d..c4b26d8 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -60,7 +60,7 @@ func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...str case "disable": f.active[u] = false } - return nil, nil + return []byte(f.outputs[c]), nil case strings.HasPrefix(c, "scontrol show node"): return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil case strings.HasPrefix(c, "scontrol update"): @@ -119,7 +119,7 @@ type fakeInst struct { func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error { i.got = by - for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { + for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} { i.host.bins[b] = true } return nil @@ -337,7 +337,7 @@ func TestBadNames(t *testing.T) { func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) { h := newFake() - for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { + for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} { h.bins[b] = true } k := New(h, &fakeInst{host: h}, "/m", nil) diff --git a/internal/agent/slurmnode/users.go b/internal/agent/slurmnode/users.go new file mode 100644 index 0000000..91fcd18 --- /dev/null +++ b/internal/agent/slurmnode/users.go @@ -0,0 +1,238 @@ +package slurmnode + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "slices" + "strconv" + "strings" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// usersMarker lists, a line each, the logins llmgw made here: the ones it +// may change and take away. A user that was here before is never touched. +func (k *Keeper) usersMarker() string { return k.marker + "-users" } + +// users keeps the Linux users want has, the same login and UID as on every +// other machine, their keys where sshd reads them, and their homes when +// this machine makes them; and takes away those llmgw made that are gone. +// Answers how many it has. A user that cannot be made is said, the others +// are made anyway. +func (k *Keeper) users(ctx context.Context, want *llmgwv1.Slurm) (int32, error) { + made := k.madeUsers() + if !want.GetManageUsers() { + return 0, k.removeUsers(ctx, made, nil) + } + var errs []error + var keep []string + n := int32(0) + for _, u := range want.GetUsers() { + err := k.user(ctx, u, slices.Contains(made, u.GetLogin()), want.GetMakeHomes()) + if errors.Is(err, errNotMade) { + errs = append(errs, err) + continue + } + keep = append(keep, u.GetLogin()) + if err != nil { + errs = append(errs, err) + continue + } + n++ + } + if err := k.keepMade(keep); err != nil { + return n, err + } + if err := k.removeUsers(ctx, made, keep); err != nil { + errs = append(errs, err) + } + if err := k.sshConf(ctx, len(keep) > 0); err != nil { + errs = append(errs, err) + } + return n, errors.Join(errs...) +} + +// errNotMade is a user llmgw could not make here, nor took for its own. +var errNotMade = errors.New("not made") + +type notMade struct{ msg string } + +func (e notMade) Error() string { return e.msg } +func (e notMade) Is(target error) bool { return target == errNotMade } + +// user keeps one user; ours is whether llmgw made it here before. +func (k *Keeper) user(ctx context.Context, u *llmgwv1.SlurmUser, ours, makeHome bool) error { + login, uid, home := u.GetLogin(), u.GetUid(), u.GetHome() + if !slurm.ValidLogin(login) || uid < slurm.MinUID || !filepath.IsAbs(home) || strings.Contains(home, "..") { + return notMade{fmt.Sprintf("the leader sent a user %q, UID %d, home %q", login, uid, home)} + } + id := strconv.FormatUint(uint64(uid), 10) + gecos := strings.Map(func(r rune) rune { + if r == ':' || r == ',' || r == '\n' || r < ' ' { + return -1 + } + return r + }, u.GetName()) + byName, okName := k.passwd(ctx, login) + byID, okID := k.passwd(ctx, id) + switch { + case okName && byName[2] != id: + return notMade{fmt.Sprintf("%s is already a user here with UID %s, not llmgw's %s", login, byName[2], id)} + case !okName && okID: + return notMade{fmt.Sprintf("UID %s is already %s's here, so %s is not made", id, byID[0], login)} + case !okName: + if _, ok := k.group(ctx, login); !ok { + if out, err := k.host.Run(ctx, nil, "groupadd", "-g", id, login); err != nil { + return notMade{fmt.Sprintf("groupadd %s: %v: %s", login, err, lastLine(out))} + } + } + k.log.Info("a Slurm user is made here, as llmgw's accounts say", "login", login, "uid", uid) + if out, err := k.host.Run(ctx, nil, "useradd", "-u", id, "-g", id, "-M", "-d", home, "-s", "/bin/bash", + "-c", gecos, login); err != nil { + return notMade{fmt.Sprintf("useradd %s: %v: %s", login, err, lastLine(out))} + } + case ours && (byName[4] != gecos || byName[5] != home): + if out, err := k.host.Run(ctx, nil, "usermod", "-c", gecos, "-d", home, login); err != nil { + return fmt.Errorf("usermod %s: %v: %s", login, err, lastLine(out)) + } + } + keys := strings.Join(u.GetSshKeys(), "\n") + file := filepath.Join(slurm.AuthorizedKeysDir, login) + if keys == "" { + _ = k.host.Remove(file) + } else if _, err := k.write(file, []byte(keys+"\n"), 0o644, ""); err != nil { + return err + } + if makeHome { + return k.home(ctx, login, id, home) + } + return nil +} + +// home makes a user's home directory, from /etc/skel, when it is not there. +func (k *Keeper) home(ctx context.Context, login, id, home string) error { + if _, err := k.host.Run(ctx, nil, "test", "-d", home); err == nil { + return nil + } + if err := k.host.MkdirAll(filepath.Dir(home), 0o755, ""); err != nil { + return err + } + k.log.Info("a Slurm user's home is made", "login", login, "home", home) + for _, c := range [][]string{ + {"install", "-d", "-m", "0700", "-o", id, "-g", id, home}, + {"cp", "-rT", "/etc/skel", home}, + {"chown", "-R", id + ":" + id, home}, + } { + if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil { + return fmt.Errorf("making %s's home: %s: %v: %s", login, c[0], err, lastLine(out)) + } + } + return nil +} + +// removeUsers takes away the users llmgw made here that are not in keep. +// Their homes and files stay; their UID is never given to another. +func (k *Keeper) removeUsers(ctx context.Context, made, keep []string) error { + var errs []error + left := slices.Clone(keep) + for _, login := range made { + if slices.Contains(keep, login) { + continue + } + k.log.Info("a Slurm user llmgw made is taken away; its home stays", "login", login) + _ = k.host.Remove(filepath.Join(slurm.AuthorizedKeysDir, login)) + if _, ok := k.passwd(ctx, login); ok { + if out, err := k.host.Run(ctx, nil, "userdel", login); err != nil { + // Still logged in, or running something: tried again. + errs = append(errs, fmt.Errorf("userdel %s: %v: %s", login, err, lastLine(out))) + left = append(left, login) + continue + } + } + } + if !slices.Equal(left, made) { + if err := k.keepMade(left); err != nil { + errs = append(errs, err) + } + } + if len(left) == 0 { + if err := k.sshConf(ctx, false); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (k *Keeper) madeUsers() []string { + b, err := k.host.ReadFile(k.usersMarker()) + if err != nil { + return nil + } + return strings.Fields(string(b)) +} + +func (k *Keeper) keepMade(logins []string) error { + s := slices.Clone(logins) + slices.Sort(s) + s = slices.Compact(s) + if len(s) == 0 { + return k.host.Remove(k.usersMarker()) + } + _, err := k.write(k.usersMarker(), []byte(strings.Join(s, "\n")+"\n"), 0o644, "") + return err +} + +// sshConf has sshd read the users' keys from AuthorizedKeysDir, or no +// longer. +func (k *Keeper) sshConf(ctx context.Context, on bool) error { + if !on { + if _, err := k.host.ReadFile(slurm.SSHDropIn); err != nil { + return nil + } + _ = k.host.Remove(slurm.SSHDropIn) + return k.reloadSSH(ctx) + } + changed, err := k.write(slurm.SSHDropIn, []byte(slurm.SSHConf), 0o644, "") + if err != nil || !changed { + return err + } + if out, err := k.host.Run(ctx, nil, "sshd", "-t"); err != nil { + _ = k.host.Remove(slurm.SSHDropIn) + return fmt.Errorf("sshd refuses llmgw's setting for the users' keys, so it is not kept: %v: %s", err, lastLine(out)) + } + k.log.Info("sshd reads the Slurm users' keys from llmgw too", "dir", slurm.AuthorizedKeysDir) + return k.reloadSSH(ctx) +} + +// reloadSSH has sshd read its settings again, under Debian's name for it +// or Red Hat's; one that is not running reads them when it starts. +func (k *Keeper) reloadSSH(ctx context.Context) error { + for _, u := range []string{"ssh", "sshd"} { + if k.active(ctx, u) { + return k.systemctl(ctx, "reload", u) + } + } + return nil +} + +// passwd is getent's line for a user, by login or UID, in its fields. +func (k *Keeper) passwd(ctx context.Context, key string) ([]string, bool) { + out, err := k.host.Run(ctx, nil, "getent", "passwd", key) + f := strings.Split(strings.TrimSpace(string(out)), ":") + if err != nil || len(f) < 7 { + return nil, false + } + return f, true +} + +func (k *Keeper) group(ctx context.Context, key string) ([]string, bool) { + out, err := k.host.Run(ctx, nil, "getent", "group", key) + f := strings.Split(strings.TrimSpace(string(out)), ":") + if err != nil || len(f) < 3 { + return nil, false + } + return f, true +} diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index 1ba5abe..e03fdcc 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -3,12 +3,16 @@ package cli import ( "encoding/json" "fmt" + "io" + "net/url" + "os" "slices" "strings" "text/tabwriter" "github.com/spf13/cobra" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" ) @@ -27,7 +31,148 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client), newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true), - newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false)) + newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false), + newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON)) + return cmd +} + +func newSlurmSharedCmd(client func() *ctlClient) *cobra.Command { + var path, server, nfs, note string + var off bool + c := &cobra.Command{ + Use: "shared --path DIR (--server NODE | --nfs HOST:/EXPORT) | --off", + Short: "Files every machine of the Slurm cluster sees at one path", + Long: "Home directories, datasets, the jobs' output and checkpoints, at the same path on the controller\n" + + "and every node. Three ways:\n" + + " --nfs HOST:/EXPORT the site's own NFS export, mounted at --path on every machine;\n" + + " --server NODE an llmgw node (the controller or a node) exports --path over NFS to the\n" + + " others, for a site with no storage of its own: one machine, not highly\n" + + " available, no faster than its disk and network;\n" + + " neither the site mounts --path itself (Lustre, BeeGFS, GPFS); llmgw only looks\n" + + " that it is there.\n" + + "The export answers the Slurm machines alone, NFS's port the cluster's networks. A machine\n" + + "that gives up its mount does so when no job holds files on it.", + Example: " llmgw ctl slurm shared --path /shared --server node06\n" + + " llmgw ctl slurm shared --path /shared --nfs filer.site:/export/hpc\n" + + " llmgw ctl slurm shared --path /lustre/hpc\n" + + " llmgw ctl slurm shared --off", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if off == (path != "") { + return fmt.Errorf("give --path, or --off") + } + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if off { + if doc.Node([]string{"slurm", "shared"}) != nil { + doc.Unset([]string{"slurm", "shared"}) + } + return nil + } + v := map[string]string{"path": path} + if server != "" { + v["server"] = server + } + if nfs != "" { + v["nfs"] = nfs + } + b, _ := json.Marshal(v) + return doc.SetYAML([]string{"slurm", "shared"}, string(b)) + }) + }, + } + c.Flags().StringVar(&path, "path", "", "where every machine sees the files, as /shared") + c.Flags().StringVar(&server, "server", "", "the llmgw node that exports them over NFS") + c.Flags().StringVar(&nfs, "nfs", "", "the site's NFS export, HOST:/EXPORT") + c.Flags().BoolVar(&off, "off", false, "no shared files; mounts and exports llmgw made are taken away") + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c +} + +func newSlurmUsersCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + cmd := &cobra.Command{ + Use: "users", + Short: "The Linux users that run jobs, made from llmgw's accounts", + Long: "With slurm.users \"llmgw\" every active account is a Linux user on every machine of the Slurm\n" + + "cluster: its login from its e-mail address, a UID of llmgw's from 200000 up (never given to\n" + + "another, even when the account goes), its home under the shared path. It logs in over SSH\n" + + "with the keys given in the portal (Profile) or here. An account suspended or deleted loses\n" + + "its user; its home and files stay. Off, users are the site's: from its directory (sssd), or\n" + + "made by hand.", + } + list := &cobra.Command{ + Use: "list", + Short: "Every user, with its login, UID and how many keys it has", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var us []slurmsvc.User + if err := client().get(cmd.Context(), "/api/v1/slurm/users", &us); err != nil { + return err + } + w := cmd.OutOrStdout() + if *asJSON { + return json.NewEncoder(w).Encode(us) + } + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "LOGIN\tUID\tACCOUNT\tSSH KEYS") + for _, u := range us { + fmt.Fprintf(tw, "%s\t%d\t%s\t%d\n", u.Login, u.UID, u.Email, len(u.SSHKeys)) + } + return tw.Flush() + }, + } + var note string + toggle := func(use, short, value string) *cobra.Command { + c := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if value == "" { + if doc.Node([]string{"slurm", "users"}) != nil { + doc.Unset([]string{"slurm", "users"}) + } + return nil + } + return doc.SetYAML([]string{"slurm", "users"}, yamlString(value)) + }) + }, + } + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c + } + keys := &cobra.Command{ + Use: "keys ACCOUNT FILE", + Short: "Replace an account's SSH public keys, from a file (- for standard input)", + Long: "FILE holds OpenSSH public keys, one a line, as ~/.ssh/id_ed25519.pub; an empty one takes them all away.", + Example: " llmgw ctl slurm users keys ayse@site.example ayse.pub\n" + + " ssh-add -L | llmgw ctl slurm users keys ayse@site.example -", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + var b []byte + var err error + if args[1] == "-" { + b, err = io.ReadAll(io.LimitReader(cmd.InOrStdin(), 64<<10)) + } else { + b, err = os.ReadFile(args[1]) + } + if err != nil { + return err + } + body, _ := json.Marshal(map[string]string{"keys": string(b)}) + var u slurmsvc.User + if err := client().put(cmd.Context(), "/api/v1/slurm/users/"+url.PathEscape(args[0])+"/ssh-keys", body, &u); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s (UID %d) has %d SSH key(s); the machines have them within a minute\n", + u.Login, u.UID, len(u.SSHKeys)) + return nil + }, + } + cmd.AddCommand(list, + toggle("on", "Make a Linux user for every active account, on every machine of the Slurm cluster", "llmgw"), + toggle("off", "Leave users to the site; those llmgw made are taken away, their homes left", ""), + keys) return cmd } @@ -62,14 +207,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { default: fmt.Fprintln(w, ": slurmctld not running") } + fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared)) + if st.Users != "" { + fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount) + } tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tNOTE") + fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE") for _, n := range st.Nodes { - state, jobs, slurmd, note := "-", "-", "-", n.Describe() + state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe() if r := n.Status; r != nil { state = or(r.State, "-") jobs = fmt.Sprint(r.Jobs) slurmd = map[bool]string{true: "running", false: "stopped"}[r.SlurmdRunning] + shared = or(r.Shared, "-") if r.Error != "" { note = r.Error } @@ -77,7 +227,7 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if !n.Connected { note = "agent not connected; " + note } - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, note) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, shared, note) } if err := tw.Flush(); err != nil { return err @@ -172,6 +322,19 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command { return c } +// sharedLine says where the shared files are and come from. +func sharedLine(sh config.SharedStorage) string { + switch { + case sh.Path == "": + return "none (`llmgw ctl slurm shared` adds them; a batch job's output stays on its node)" + case sh.Server != "": + return fmt.Sprintf("%s, exported over NFS by %s", sh.Path, sh.Server) + case sh.NFS != "": + return fmt.Sprintf("%s, mounted from %s", sh.Path, sh.NFS) + } + return sh.Path + ", mounted by the site" +} + // newSlurmListCmd adds nodes to slurm.lent, or takes them out. func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cobra.Command { var note string diff --git a/internal/cli/server.go b/internal/cli/server.go index 3e3ca58..7ce7d4d 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -256,6 +256,7 @@ func newServerCmd() *cobra.Command { // The settings in force, for what reads them at each use. var current atomic.Pointer[config.Server] current.Store(&cfg) + rt.SetSlurm(func() config.SlurmSettings { return current.Load().Slurm }) var agentSvc *agentsvc.Service var agentHandler http.Handler var agentPath string @@ -452,6 +453,16 @@ func newServerCmd() *cobra.Command { }, log) // What slurm.conf held, for the leader that comes next. slurmSvc.SetMemory(slurmsvc.StoreMemory(st)) + // slurm.users "llmgw": a Linux user for each active account. + slurmSvc.SetUsers(func(ctx context.Context) ([]slurmsvc.User, error) { + us, err := st.SlurmUsers(ctx) + out := make([]slurmsvc.User, 0, len(us)) + for _, u := range us { + out = append(out, adminapi.SlurmUserView(u)) + } + return out, err + }) + api.SetSlurmUsers(st) agentSvc.SetSlurm(slurmSvc) go slurmSvc.Run(ctx) api.SetSlurm(slurmSvc) diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 22d2f90..4a52eb1 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -127,6 +127,10 @@ type serverInit struct { // slurm runs Slurm beside llmgw: slurmctld on this machine's node, // slurmd on every node, each llmgw's until it is lent. slurm bool + // slurmShared is where Slurm's shared files come from: "here" (this + // machine exports /shared), the site's export as HOST:/EXPORT, or + // "none". slurmUsers is "llmgw" (users from the accounts) or "site". + slurmShared, slurmUsers string // evictAfterIdle and minLifetime are on_demand's; zero leaves its own. evictAfterIdle, minLifetime time.Duration // standbys makes each machine of the replicated store a control plane @@ -263,6 +267,10 @@ func newServerInitCmd() *cobra.Command { "pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one") f.BoolVar(&in.slurm, "slurm", false, "Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") + f.StringVar(&in.slurmShared, "slurm-shared", "here", + "with --slurm, the jobs' shared files at /shared: here (this machine exports them over NFS), the site's HOST:/EXPORT, or none") + f.StringVar(&in.slurmUsers, "slurm-users", config.UsersLlmgw, + "with --slurm, who makes the Linux users that run jobs: llmgw (one for each account, SSH keys in the portal) or site") f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0, "how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)") f.DurationVar(&in.minLifetime, "min-lifetime", 0, diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index c28fc9f..cb4c4f5 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -144,6 +144,13 @@ func (in *serverInit) checkChoices() error { if err := (config.SlurmSettings{Enabled: true, Controller: localNodeID()}).Check(); err != nil { return fmt.Errorf("slurm: this machine's hostname: %w", err) } + set := config.SlurmSettings{Enabled: true, Controller: localNodeID(), Shared: slurmShared(in.slurmShared)} + if err := set.Check(); err != nil { + return fmt.Errorf("--slurm-shared %q: here, none, or the site's HOST:/EXPORT: %w", in.slurmShared, err) + } + if u := in.slurmUsers; u != "" && u != config.UsersLlmgw && u != "site" { + return fmt.Errorf("--slurm-users %q: llmgw or site", u) + } } if in.vip != "" { ip, _, err := net.ParseCIDR(in.vip) @@ -329,7 +336,7 @@ func (in *serverInit) tuneConfig(body string) string { "engines:\n comfyui: "+provision.ComfyUIVersion()+"\npresets:\n auto: true\n") } if in.slurm { - extra = append(extra, slurmSection(localNodeID())) + extra = append(extra, slurmSection(localNodeID(), slurmShared(in.slurmShared), in.slurmUsers != "site")) } if in.evictAfterIdle > 0 || in.minLifetime > 0 { b := "# How models that load when asked for take turns on a card, chosen at\n# install.\non_demand:\n" @@ -347,15 +354,43 @@ func (in *serverInit) tuneConfig(body string) string { return body } +// slurmShared is --slurm-shared as settings: "here" is /shared exported +// by this machine's node, "none" nothing, anything else the site's export. +func slurmShared(v string) config.SharedStorage { + switch v { + case "", "none": + return config.SharedStorage{} + case "here": + return config.SharedStorage{Path: "/shared", Server: localNodeID()} + } + return config.SharedStorage{Path: "/shared", NFS: v} +} + // slurmSection is server.yaml's slurm block as an install chooses it: -// slurmctld on this machine's node, slurmd on every node. -func slurmSection(controller string) string { - return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" + +// slurmctld on this machine's node, slurmd on every node, the shared files +// and the users as chosen. +func slurmSection(controller string, shared config.SharedStorage, users bool) string { + b := "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" + "# for the jobs' records, on this machine; slurmd on\n" + "# every node, those that join later too. Each node is llmgw's, drained in\n" + "# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" + "# reclaim NODE' takes it back once its jobs end.\n" + "slurm:\n enabled: true\n controller: " + yamlString(controller) + "\n all_nodes: true\n" + switch { + case shared.Server != "": + b += " # /shared on every Slurm machine, exported over NFS from this one\n" + + " # ('llmgw ctl slurm shared' points it at the site's storage instead).\n" + + " shared:\n path: " + yamlString(shared.Path) + "\n server: " + yamlString(shared.Server) + "\n" + case shared.NFS != "": + b += " # /shared on every Slurm machine, mounted from the site's export.\n" + + " shared:\n path: " + yamlString(shared.Path) + "\n nfs: " + yamlString(shared.NFS) + "\n" + } + if users { + b += " # A Linux user for each llmgw account, the same UID everywhere; SSH keys\n" + + " # in the portal's Profile. Empty leaves users to the site (sssd, a directory).\n" + + " users: llmgw\n" + } + return b } // localNodeID is the node id this machine's agent takes: its hostname. @@ -501,6 +536,12 @@ func (in *serverInit) initFlags() []string { } if in.slurm { add("--slurm") + if in.slurmShared != "" && in.slurmShared != "here" { + add("--slurm-shared", in.slurmShared) + } + if in.slurmUsers == "site" { + add("--slurm-users", "site") + } } if in.evictAfterIdle > 0 { add("--evict-after-idle", durationYAML(in.evictAfterIdle)) diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 4f0b154..53d36b6 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -17,7 +17,7 @@ import ( func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) { in := &serverInit{ language: "tr", signupOpen: true, defaultRPM: 120, defaultTPM: 50000, maxKeyTTL: 30 * 24 * time.Hour, - media: true, slurm: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second, + media: true, slurm: true, slurmShared: "here", slurmUsers: "llmgw", evictAfterIdle: time.Minute, minLifetime: 90 * time.Second, judge: judgeSpec{Fail: "open", Categories: []string{"Violent", "PII"}, Replicas: 3, Budget: 3 * time.Second}, } j := in.judgeFor() @@ -43,6 +43,8 @@ func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) { t.Errorf("pictures and 3D: presets %v comfyui %q", cfg.Presets.Auto, cfg.Engines["comfyui"]) case !cfg.Slurm.Enabled || !cfg.Slurm.AllNodes || cfg.Slurm.Controller != localNodeID(): t.Errorf("slurm %+v", cfg.Slurm) + case cfg.Slurm.Shared.Path != "/shared" || cfg.Slurm.Shared.Server != localNodeID() || cfg.Slurm.Users != "llmgw": + t.Errorf("slurm's shared files and users %+v", cfg.Slurm) case cfg.OnDemand.EvictAfterIdle != time.Minute || cfg.OnDemand.MinLifetime != 90*time.Second: t.Errorf("on demand %+v", cfg.OnDemand) } diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index 0a49056..be6d153 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -292,9 +292,13 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta "cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" + "Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " + "its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " + - "are submitted on this machine with sbatch and srun.", + "are submitted on this machine with sbatch and srun.\n\n" + + "Every Slurm machine sees /shared, exported over NFS from this machine (homes, datasets, the jobs' " + + "output); 'llmgw ctl slurm shared' points it at the site's own storage instead. Each llmgw account " + + "is a Linux user there, the same UID on every machine, logging in with the SSH keys given in the " + + "portal; 'llmgw ctl slurm users off' leaves users to the site's directory.", off: "no Slurm; 'llmgw ctl slurm setup' adds it later", - summary: func() string { return "slurmctld here, slurmd on every node; none lent" }, + summary: func() string { return "slurmctld here, slurmd on every node, /shared from here; none lent" }, needs: func() string { if !f.node.on { return "slurmctld runs on this machine's node: turn the node on, or Slurm off" diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go index 6cb845d..ba297b6 100644 --- a/internal/cli/tui_slurm.go +++ b/internal/cli/tui_slurm.go @@ -44,6 +44,16 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { } head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl) head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller)) + if st.Shared.Path == "" { + head += "\n shared files: " + sWarn.Render("none") + sDim.Render(" (llmgw ctl slurm shared; output stays on the job's node)") + } else { + head += "\n shared files: " + sharedLine(st.Shared) + } + if st.Users != "" { + head += "\n " + fmt.Sprintf("users: %d from llmgw's accounts", st.UserCount) + sDim.Render(", SSH keys in the portal's Profile") + } else { + head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)") + } head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.") if len(st.Waiting) > 0 { head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting)) @@ -56,13 +66,20 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { for _, n := range st.Nodes { rows = append(rows, slurmRow(n)) } - return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys + return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "SHARED", "NOTE"}, rows, m.row, pw, m.inPanel), keys } func slurmRow(n slurmsvc.NodeStatus) []string { - state, jobs, slurmd, note := "-", "-", "-", n.Describe() + state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe() if r := n.Status; r != nil { state = or(r.State, "-") + switch r.Shared { + case "": + case "missing": + shared = sBad.Render(r.Shared) + default: + shared = sOK.Render(r.Shared) + } jobs = fmt.Sprint(r.Jobs) slurmd = sBad.Render("stopped") if r.SlurmdRunning { @@ -84,7 +101,7 @@ func slurmRow(n slurmsvc.NodeStatus) []string { case slurmsvc.Lending, slurmsvc.Reclaiming: whose = sWarn.Render(whose) } - return []string{n.Node, whose, state, jobs, slurmd, note} + return []string{n.Node, whose, state, jobs, slurmd, shared, note} } // slurmMenu is what can be done with one of Slurm's nodes. diff --git a/internal/cli/tui_slurm_test.go b/internal/cli/tui_slurm_test.go index 87393ce..2cc0500 100644 --- a/internal/cli/tui_slurm_test.go +++ b/internal/cli/tui_slurm_test.go @@ -11,6 +11,7 @@ import ( tea "github.com/charmbracelet/bubbletea" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" ) @@ -37,16 +38,17 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { m := newConsoleModel(srv.URL, "/nonexistent/server.yaml") m.loaded, m.menu, m.inPanel = true, cSlurm, true m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu", - Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"}, + Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"}, + Shared: config.SharedStorage{Path: "/shared", Server: "node06"}, Users: "llmgw", UserCount: 2, Nodes: []slurmsvc.NodeStatus{ {Node: "node05", Phase: slurmsvc.Lent, Connected: true, - Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}}, + Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true, Shared: "mounted"}}, {Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true, Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}}, }} var tm tea.Model = m view := tm.View() - for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { + for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "/shared, exported over NFS by node06", "users: 2", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { if !strings.Contains(view, want) { t.Errorf("the Slurm view has no %q:\n%s", want, view) } diff --git a/internal/config/slurm.go b/internal/config/slurm.go index e1b946e..c2dfb7a 100644 --- a/internal/config/slurm.go +++ b/internal/config/slurm.go @@ -2,8 +2,10 @@ package config import ( "fmt" + "path" "regexp" "slices" + "strings" ) // SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes @@ -38,6 +40,83 @@ type SlurmSettings struct { // Partition is the one partition every node is in. Empty is "gpu". Partition string `koanf:"partition"` + + // Shared is the file system every machine of the Slurm cluster sees at + // the same path: home directories, datasets, the jobs' output and + // checkpoints. Empty path: none, and a batch job's output stays on the + // node it ran on. + Shared SharedStorage `koanf:"shared"` + + // Users is who keeps the Linux users that run jobs, the same on every + // machine of the Slurm cluster: "llmgw" makes one for each active llmgw + // account (its login from the e-mail address, a UID of llmgw's from + // 200000 up, its SSH keys from the portal); empty leaves users to the + // site (a directory through sssd, or by hand). + Users string `koanf:"users"` +} + +// SharedStorage is where the Slurm cluster's shared files come from: the +// site's NFS export (NFS), an NFS export llmgw keeps on one of its nodes +// (Server), or, with neither, a file system the site mounts itself +// (Lustre, BeeGFS, GPFS), which llmgw only looks for. +type SharedStorage struct { + // Path is where every machine sees it, as /shared. + Path string `koanf:"path"` + // NFS is the site's export, host:/path, mounted at Path on every + // machine of the Slurm cluster. + NFS string `koanf:"nfs"` + // Server is the llmgw node, the controller or one of the nodes, that + // exports Path over NFS to the others: for a site with no storage of + // its own. One machine: not highly available, and no faster than its + // disk and network. + Server string `koanf:"server"` +} + +// UsersLlmgw has llmgw make the Slurm cluster's users from its accounts. +const UsersLlmgw = "llmgw" + +// HomeDir is where a user's home directory is: under the shared path when +// there is one, else on each machine's own disk. +func (s SlurmSettings) HomeDir(login string) string { + if s.Shared.Path != "" { + return path.Join(s.Shared.Path, "home", login) + } + return path.Join("/home", login) +} + +// nfsExport is host:/path. +var nfsExport = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:\[\]-]*:/[A-Za-z0-9_./-]*$`) + +// sharedPath is an absolute path of plain names, which a mount unit's name +// is made from. +var sharedPath = regexp.MustCompile(`^(/[A-Za-z0-9_.-]+)+$`) + +func (s SlurmSettings) checkShared() error { + sh := s.Shared + if sh.Path == "" { + if sh.NFS != "" || sh.Server != "" { + return fmt.Errorf("slurm.shared.path is empty: say where every machine sees the shared files, as /shared") + } + return nil + } + if !sharedPath.MatchString(sh.Path) || strings.Contains(sh.Path, "/.") { + return fmt.Errorf("slurm.shared.path %q is not an absolute path of plain names, as /shared", sh.Path) + } + for _, sys := range []string{"/", "/etc", "/usr", "/var", "/var/lib", "/home", "/root", "/boot", "/opt", "/srv", "/tmp", "/proc", "/sys", "/dev", "/run"} { + if path.Clean(sh.Path) == sys { + return fmt.Errorf("slurm.shared.path %s is the system's own directory: take one of its own, as /shared", sh.Path) + } + } + if sh.NFS != "" && sh.Server != "" { + return fmt.Errorf("slurm.shared names both the site's export (nfs) and an llmgw node to export it (server): one of them") + } + if sh.NFS != "" && !nfsExport.MatchString(sh.NFS) { + return fmt.Errorf("slurm.shared.nfs %q is not host:/path", sh.NFS) + } + if sh.Server != "" && sh.Server != s.Controller && !s.AllNodes && !slices.Contains(s.Nodes, sh.Server) { + return fmt.Errorf("slurm.shared.server %q is neither the controller nor one of slurm.nodes", sh.Server) + } + return nil } // slurmName is what Slurm takes as a node, cluster or partition name. @@ -111,5 +190,8 @@ func (s SlurmSettings) Check() error { return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n) } } - return nil + if s.Users != "" && s.Users != UsersLlmgw { + return fmt.Errorf("slurm.users %q: %q has llmgw make them, empty leaves them to the site", s.Users, UsersLlmgw) + } + return s.checkShared() } diff --git a/internal/config/slurm_test.go b/internal/config/slurm_test.go index 78dbeb8..b2175a1 100644 --- a/internal/config/slurm_test.go +++ b/internal/config/slurm_test.go @@ -35,6 +35,49 @@ func TestSlurmCheck(t *testing.T) { } } +func TestSlurmShared(t *testing.T) { + base := SlurmSettings{Enabled: true, Controller: "c", Nodes: []string{"a"}} + for _, sh := range []SharedStorage{{}, {Path: "/shared"}, {Path: "/shared", NFS: "filer.site:/export/hpc"}, + {Path: "/data/hpc", Server: "c"}, {Path: "/shared", Server: "a"}} { + s := base + s.Shared = sh + if err := s.Check(); err != nil { + t.Errorf("%+v refused: %v", sh, err) + } + } + for want, sh := range map[string]SharedStorage{ + "path is empty": {NFS: "f:/x"}, + "plain names": {Path: "shared"}, + "system's own": {Path: "/home"}, + "one of them": {Path: "/shared", NFS: "f:/x", Server: "c"}, + "not host:/path": {Path: "/shared", NFS: "f/x"}, + "neither": {Path: "/shared", Server: "z"}, + } { + s := base + s.Shared = sh + if err := s.Check(); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%+v: got %v, want %q", sh, err, want) + } + } + s := base + s.Shared = SharedStorage{Path: "/a/../etc"} + if s.Check() == nil { + t.Error("a path with .. taken") + } + s.Shared, s.Users = SharedStorage{}, "ldap" + if s.Check() == nil { + t.Error("slurm.users ldap taken") + } + s.Users, s.Shared = UsersLlmgw, SharedStorage{Path: "/shared"} + if s.Check() != nil || s.HomeDir("ayse") != "/shared/home/ayse" { + t.Errorf("home %s", s.HomeDir("ayse")) + } + s.Shared = SharedStorage{} + if s.HomeDir("ayse") != "/home/ayse" { + t.Errorf("home %s", s.HomeDir("ayse")) + } +} + func TestSlurmAllNodes(t *testing.T) { s := SlurmSettings{Enabled: true, Controller: "c", AllNodes: true, Lent: []string{"joined"}} if err := s.Check(); err != nil { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 583d571..43143cb 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -98,6 +98,8 @@ type API struct { usage Usage quotas Quotas ctl *control.Controller + // slurmUsers are the Linux users made from the accounts; nil without a store. + slurmUsers SlurmUsers } // New builds the API over a controller. diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index d003789..2af4ec5 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -85,6 +85,7 @@ var PathParams = map[string]string{ "projects/{id}": "The project's ID.", "keys/{id}": "The key's ID, as listed (not the key itself).", "users/{id}": "The person's ID or email.", + "users/{user}": "The person's ID or email.", "artifacts/{kind}": "What the job made: peft, a gzipped tar of the PEFT adapter, or gguf, the adapter for llama.cpp.", } @@ -398,6 +399,14 @@ var Routes = []Route{ "(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " + "still running) or unknown (its agent has not said).", Response: slurmsvc.Status{}}, + {Method: "GET", Path: "/api/v1/slurm/users", Summary: "The Linux users made from llmgw's accounts for the Slurm cluster", + Description: "Every active account, with its login, UID and SSH keys; an account is given its login and UID the first " + + "time it is listed. The machines have them when slurm.users is \"llmgw\".", + Response: []slurmsvc.User{}}, + {Method: "PUT", Path: "/api/v1/slurm/users/{user}/ssh-keys", Summary: "Replace an account's SSH public keys", + Description: "{user} is an ID or e-mail. keys is OpenSSH public keys, one a line, without options; empty takes them " + + "all away. The machines have them within a minute.", + Request: SlurmKeysRequest{}, Response: slurmsvc.User{}}, {Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node", Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " + diff --git a/internal/control/adminapi/slurm.go b/internal/control/adminapi/slurm.go index f899acc..1935af6 100644 --- a/internal/control/adminapi/slurm.go +++ b/internal/control/adminapi/slurm.go @@ -1,15 +1,43 @@ package adminapi import ( + "context" + "encoding/json" + "errors" "net/http" + "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader // keeps it. type SlurmService interface { Status() slurmsvc.Status + UsersChanged() +} + +// SlurmUsers are the Linux users llmgw makes from its accounts. +type SlurmUsers interface { + SlurmUsers(ctx context.Context) ([]store.SlurmUser, error) + SlurmUserByID(ctx context.Context, userID string) (store.SlurmUser, error) + SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error +} + +// SetSlurmUsers enables the Slurm users' endpoints. +func (a *API) SetSlurmUsers(u SlurmUsers) { a.slurmUsers = u } + +// SlurmKeysRequest is a person's SSH public keys, one a line, as +// authorized_keys has them; empty takes them all away. +type SlurmKeysRequest struct { + Keys string `json:"keys"` +} + +// SlurmUserView is the user from a store row. +func SlurmUserView(u store.SlurmUser) slurmsvc.User { + return slurmsvc.User{Login: u.Login, UID: u.UID, Name: u.Name, Email: u.Email, SSHKeys: u.SSHKeys} } // SetSlurm gives the API the Slurm service. @@ -17,6 +45,69 @@ func (a *API) SetSlurm(s SlurmService) { a.slurm = s } func (a *API) registerSlurm(mux Mux) { mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus) + mux.HandleFunc("GET /api/v1/slurm/users", a.slurmUserList) + mux.HandleFunc("PUT /api/v1/slurm/users/{user}/ssh-keys", a.slurmUserKeys) +} + +func (a *API) slurmUserList(w http.ResponseWriter, r *http.Request) { + if a.slurmUsers == nil { + writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without") + return + } + us, err := a.slurmUsers.SlurmUsers(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + out := make([]slurmsvc.User, 0, len(us)) + for _, u := range us { + out = append(out, SlurmUserView(u)) + } + writeJSON(w, http.StatusOK, out) +} + +// slurmUserKeys replaces a person's SSH keys; {user} is an ID or e-mail. +func (a *API) slurmUserKeys(w http.ResponseWriter, r *http.Request) { + if a.slurmUsers == nil { + writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without") + return + } + if !a.tenancyReady(w) { + return + } + var in SlurmKeysRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&in); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + keys, err := slurm.ParseSSHKeys(in.Keys) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ctx := r.Context() + u, err := a.findUser(ctx, r.PathValue("user")) + if err != nil { + writeError(w, http.StatusNotFound, strings.TrimPrefix(err.Error(), store.ErrNotFound.Error()+": ")) + return + } + if err := a.slurmUsers.SetSlurmSSHKeys(ctx, u.ID, keys); err != nil { + code := http.StatusInternalServerError + if errors.Is(err, store.ErrNotFound) { + code = http.StatusNotFound + } + writeError(w, code, err.Error()) + return + } + if a.slurm != nil { + a.slurm.UsersChanged() + } + su, err := a.slurmUsers.SlurmUserByID(ctx, u.ID) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, SlurmUserView(su)) } func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) { diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 53cabd9..8f29747 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -71,17 +71,21 @@ type NodeStatus struct { // Reported is the agent's SlurmStatus, for the admin API. type Reported struct { - Version string `json:"version,omitempty"` - Hardware string `json:"hardware,omitempty"` - SlurmdRunning bool `json:"slurmd_running"` - SlurmctldRunning bool `json:"slurmctld_running"` - SlurmdbdRunning bool `json:"slurmdbd_running"` - State string `json:"state,omitempty"` - Reason string `json:"reason,omitempty"` - Jobs int `json:"jobs"` - Error string `json:"error,omitempty"` - FilesSHA256 string `json:"files_sha256,omitempty"` - At time.Time `json:"at"` + Version string `json:"version,omitempty"` + Hardware string `json:"hardware,omitempty"` + SlurmdRunning bool `json:"slurmd_running"` + SlurmctldRunning bool `json:"slurmctld_running"` + SlurmdbdRunning bool `json:"slurmdbd_running"` + // Shared is the shared path here: "mounted", "exported", "missing". + Shared string `json:"shared,omitempty"` + // Users is how many of llmgw's users the machine has. + Users int `json:"users"` + State string `json:"state,omitempty"` + Reason string `json:"reason,omitempty"` + Jobs int `json:"jobs"` + Error string `json:"error,omitempty"` + FilesSHA256 string `json:"files_sha256,omitempty"` + At time.Time `json:"at"` } // Status is the whole Slurm cluster, for `ctl slurm status`. @@ -95,6 +99,20 @@ type Status struct { // Waiting are nodes left out of slurm.conf until they report their // hardware. Waiting []string `json:"waiting,omitempty"` + // Shared is where the shared files are and come from; Users who keeps + // the users, and how many llmgw has. + Shared config.SharedStorage `json:"shared"` + Users string `json:"users,omitempty"` + UserCount int `json:"user_count,omitempty"` +} + +// User is a Linux user made from an llmgw account. +type User struct { + Login string `json:"login"` + UID int `json:"uid"` + Name string `json:"name"` + Email string `json:"email"` + SSHKeys []string `json:"ssh_keys,omitempty"` } // Service keeps the Slurm cluster. @@ -127,6 +145,46 @@ type Service struct { // started is the first pass, from which a node neither heard from nor // kept is waited for, for HoldFor. started time.Time + // userList reads the users from the accounts; users is what it last + // said, read at usersAt. Until it has been read once no node is told + // anything of users: an empty list would take every user away. + userList func(ctx context.Context) ([]User, error) + users []User + usersAt time.Time + usersOK bool +} + +// UsersEvery is how often the users are read again without being asked. +var UsersEvery = 30 * time.Second + +// SetUsers has the users read with list, for slurm.users "llmgw". +func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) { + s.mu.Lock() + s.userList = list + s.mu.Unlock() +} + +// UsersChanged has the users read again now: an account or its keys +// changed. +func (s *Service) UsersChanged() { + s.mu.Lock() + s.usersAt = time.Time{} + s.mu.Unlock() + s.Kick() +} + +// readUsers reads the users when they are due. Under s.mu. +func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) { + if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery { + return + } + us, err := s.userList(ctx) + s.usersAt = time.Now() + if err != nil { + s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err) + return + } + s.users, s.usersOK = us, true } // HoldFor is how long a new leader waits for a node it neither has heard @@ -198,6 +256,7 @@ func (s *Service) Kick() { // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), + Shared: st.GetShared(), Users: int(st.GetUsers()), SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() @@ -227,6 +286,7 @@ func (s *Service) Tick(ctx context.Context) { s.mu.Lock() defer s.mu.Unlock() s.recall(ctx) + s.readUsers(ctx, set) if s.started.IsZero() { s.started = time.Now() } @@ -477,16 +537,103 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key m.Controller = true m.Files = maps.Clone(files) } + if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) { + return nil + } return m } +// sharedPart puts the shared files in m, false when where they come from +// is not known yet: told nothing rather than no shared files, which would +// unmount them under running jobs. Under s.mu. +func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool { + sh := set.Shared + if sh.Path == "" { + return true + } + m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS} + switch { + case sh.Server == "": + case sh.Server == node: + m.Shared.Serve = true + for _, n := range s.machines(set) { + if n == node { + continue + } + if a := s.addr(set, n); a != "" { + m.Shared.Clients = append(m.Shared.Clients, a) + } + } + slices.Sort(m.Shared.Clients) + default: + a := s.addr(set, sh.Server) + if a == "" { + return false + } + if strings.Contains(a, ":") { + a = "[" + a + "]" + } + m.Shared.Nfs = a + ":" + sh.Path + } + return true +} + +// usersPart puts the users in m, false while they have not been read. +// Under s.mu. +func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool { + if set.Users != config.UsersLlmgw { + return true + } + if !s.usersOK { + return false + } + m.ManageUsers = true + switch { + case set.Shared.Path == "": + m.MakeHomes = true // each machine's own /home + case set.Shared.Server != "": + m.MakeHomes = set.Shared.Server == node + default: + m.MakeHomes = set.Controller == node + } + for _, u := range s.users { + m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up + Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)}) + } + return true +} + +// machines are the controller and the nodes. +func (s *Service) machines(set config.SlurmSettings) []string { + out := slices.Clone(set.Nodes) + if !slices.Contains(out, set.Controller) { + out = append(out, set.Controller) + } + return out +} + +// addr is a machine's management address, from the inventory, else kept +// from before. Under s.mu. +func (s *Service) addr(set config.SlurmSettings, node string) string { + if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" { + return n.MgmtAddr + } + if node == set.Controller { + return s.controllerAddr(set) + } + return s.known.Nodes[node].Addr +} + // Status is the cluster as the service sees it now. func (s *Service) Status() Status { set := s.current() s.mu.Lock() defer s.mu.Unlock() out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller, - Partition: set.PartitionName(), Ctld: s.status[set.Controller]} + Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users} + if set.Users == config.UsersLlmgw { + out.UserCount = len(s.users) + } if !set.Enabled { return out } diff --git a/internal/control/slurmsvc/shared_test.go b/internal/control/slurmsvc/shared_test.go new file mode 100644 index 0000000..3a147f1 --- /dev/null +++ b/internal/control/slurmsvc/shared_test.go @@ -0,0 +1,85 @@ +package slurmsvc + +import ( + "context" + "errors" + "slices" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" +) + +func TestSharedFilesFromTheController(t *testing.T) { + r := newRig() + r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"} + r.tick() + ctl, node := r.agents.last("node06").GetShared(), r.agents.last("cuda01").GetShared() + if !ctl.GetServe() || !slices.Equal(ctl.GetClients(), []string{"10.0.0.147"}) || ctl.GetPath() != "/shared" { + t.Fatalf("server told %v", ctl) + } + if node.GetServe() || node.GetNfs() != "10.0.0.146:/shared" { + t.Fatalf("client told %v", node) + } + // The site's own export: every machine mounts it. + r.set.Shared = config.SharedStorage{Path: "/shared", NFS: "filer:/hpc"} + r.tick() + if r.agents.last("node06").GetShared().GetNfs() != "filer:/hpc" || r.agents.last("cuda01").GetShared().GetServe() { + t.Fatal("the site's export not sent") + } +} + +func TestSharedServerNotKnownTellsNothing(t *testing.T) { + r := newRig() + r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"} + delete(r.inv.nodes, "node06") + r.svc.known.ControllerAddr, r.svc.known.Controller = "10.0.0.146", "node06" + r.tick() + if r.agents.last("cuda01").GetShared().GetNfs() != "10.0.0.146:/shared" { + t.Fatalf("the kept address not used: %v", r.agents.last("cuda01")) + } + r2 := newRig() + r2.set.Nodes = []string{"cuda01", "site2"} + r2.agents.conn = append(r2.agents.conn, "site2") + r2.set.Shared = config.SharedStorage{Path: "/shared", Server: "site2"} + r2.tick() + if r2.agents.last("cuda01") != nil { + t.Fatal("told with no address for the server: its mount would go") + } +} + +func TestUsersSentOnceRead(t *testing.T) { + r := newRig() + r.set.Users = config.UsersLlmgw + r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"} + fail := true + r.svc.SetUsers(func(context.Context) ([]User, error) { + if fail { + return nil, errors.New("store down") + } + return []User{{Login: "ayse", UID: 200000, Name: "Ayşe", SSHKeys: []string{"ssh-ed25519 AAAA"}}}, nil + }) + r.tick() + if len(r.agents.got["cuda01"]) != 0 { + t.Fatal("told before the users were read: every user would go") + } + fail = false + r.svc.UsersChanged() + r.tick() + ctl, node := r.agents.last("node06"), r.agents.last("cuda01") + if !ctl.GetManageUsers() || !ctl.GetMakeHomes() || node.GetMakeHomes() || len(node.GetUsers()) != 1 { + t.Fatalf("controller %v\nnode %v", ctl, node) + } + if u := node.GetUsers()[0]; u.GetHome() != "/shared/home/ayse" || u.GetUid() != 200000 || len(u.GetSshKeys()) != 1 { + t.Fatalf("user %v", u) + } + // The store down later: the last users are kept. + fail = true + r.svc.UsersChanged() + r.tick() + if len(r.agents.last("cuda01").GetUsers()) != 1 { + t.Fatal("users lost when the store could not be read") + } + if st := r.svc.Status(); st.UserCount != 1 || st.Shared.Server != "node06" { + t.Fatalf("status %+v", st) + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 77d7366..3cbc98a 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -510,6 +510,11 @@ var Reference = []Block{ {Package: "config", Name: "ServingSettings", Title: "ServingSettings", Doc: "", Fields: []Field{ {Key: "stopped", Type: "bool", Doc: "Stopped takes every model down and answers every request for one with\n503 (system_stopped), leaving the control planes, agents and mirror\nrunning and every model's definition as it is: llmgw stop sets it,\nllmgw start clears it, and the models come back as their catalog\nentries ask. In force on every control plane within seconds.", Nested: ""}, }}, + {Package: "config", Name: "SharedStorage", Title: "SharedStorage", Doc: "", Fields: []Field{ + {Key: "path", Type: "string", Doc: "Path is where every machine sees it, as /shared.", Nested: ""}, + {Key: "nfs", Type: "string", Doc: "NFS is the site's export, host:/path, mounted at Path on every\nmachine of the Slurm cluster.", Nested: ""}, + {Key: "server", Type: "string", Doc: "Server is the llmgw node, the controller or one of the nodes, that\nexports Path over NFS to the others: for a site with no storage of\nits own. One machine: not highly available, and no faster than its\ndisk and network.", Nested: ""}, + }}, {Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{ {Key: "bin", Type: "string", Doc: "Bin is the llmgw binary itself; the engine is one of its subcommands.\nEmpty means whatever started this agent, which is right almost always.", Nested: ""}, }}, @@ -521,6 +526,8 @@ var Reference = []Block{ {Key: "all_nodes", Type: "bool", Doc: "AllNodes runs slurmd on every llmgw node, those that join later too,\nbesides any named in Nodes. Each is llmgw's until it is lent, as any.", Nested: ""}, {Key: "lent", Type: "[]string", Doc: "Lent are the nodes of Nodes Slurm may run jobs on now. A node put here\nis drained of llmgw's replicas first, as `ctl drain` does, and resumed\nin Slurm once none is left; a node taken out is drained in Slurm and\ngiven back to llmgw when its last job ends.", Nested: ""}, {Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""}, + {Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"}, + {Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal); empty leaves users to the\nsite (a directory through sssd, or by hand).", Nested: ""}, }}, {Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{ {Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""}, diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 3689369..cba291d 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -253,6 +253,9 @@ func (r *Router) Handler() http.Handler { pages.SetPersonalDefaults(r.personalDefaults) if r.store != nil { pages.SetAudit(r.store) + if r.slurmSettings != nil { + pages.SetSlurm(r.store, r.slurmSettings) + } // A forgotten password: a link mailed when the cluster's // settings say how, otherwise asked of an operator. pages.SetPasswordReset(r.store, r.mail, r.mail.BaseURL) diff --git a/internal/router/router.go b/internal/router/router.go index cf60adf..d851fc8 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -208,6 +208,9 @@ type Router struct { // settingsCheck and settingsLocal mount the portal's settings page. settingsCheck func([]byte) error settingsLocal []string + // slurmSettings is the cluster's Slurm settings, for the portal; nil + // offers nothing of Slurm. + slurmSettings func() config.SlurmSettings } // SetClusterView gives the router the data the /admin page renders. @@ -238,6 +241,10 @@ func (r *Router) SetSettingsCheck(check func([]byte) error, local []string) { r.settingsCheck, r.settingsLocal = check, local } +// SetSlurm has the portal offer the Slurm cluster's login and SSH keys as +// settings say. +func (r *Router) SetSlurm(s func() config.SlurmSettings) { r.slurmSettings = s } + // SetStopped refuses, or again serves, every request for a model. func (r *Router) SetStopped(stopped bool) { r.stopped.Store(stopped) } diff --git a/internal/slurm/shared.go b/internal/slurm/shared.go new file mode 100644 index 0000000..1645d32 --- /dev/null +++ b/internal/slurm/shared.go @@ -0,0 +1,127 @@ +package slurm + +import ( + "fmt" + "regexp" + "slices" + "strings" + + "golang.org/x/crypto/ssh" +) + +// Shared storage: one path every machine of the Slurm cluster sees, from +// the site's NFS export, from an llmgw node that exports it, or mounted by +// the site itself. +const ( + // NFSPort is NFSv4's one port, which is all a client needs. + NFSPort = 2049 + // ExportsPath is the export llmgw keeps on the machine that serves. + ExportsPath = "/etc/exports.d/llmgw-slurm.exports" + // MountUnitDir is where the mount unit is written. + MountUnitDir = "/etc/systemd/system" +) + +// Exports is the exports file for path, read-write to clients alone. Root +// on a client is nobody here: a client's root cannot take another's files. +func Exports(path string, clients []string) string { + cs := slices.Clone(clients) + slices.Sort(cs) + cs = slices.Compact(cs) + var b strings.Builder + b.WriteString("# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, to its machines alone.\n") + if len(cs) == 0 { + return b.String() + } + fmt.Fprintf(&b, "%q", path) + for _, c := range cs { + fmt.Fprintf(&b, " %s(rw,sync,no_subtree_check,root_squash)", c) + } + b.WriteString("\n") + return b.String() +} + +// MountUnitName is systemd's name for the mount unit of path, as +// systemd-escape --path makes it. +func MountUnitName(path string) string { + p := strings.Trim(path, "/") + var b strings.Builder + for i := 0; i < len(p); i++ { + c := p[i] + switch { + case c == '/': + b.WriteByte('-') + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '_', c == '.' && i > 0: + b.WriteByte(c) + default: + fmt.Fprintf(&b, `\x%02x`, c) + } + } + return b.String() + ".mount" +} + +// MountUnit mounts the export nfs at path. hard: a job waits for a server +// that is gone rather than reading errors; nconnect spreads a client's +// traffic over several connections. The version is the highest both +// sides speak. +func MountUnit(nfs, path string) string { + return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files.\n" + + "[Unit]\nDescription=llmgw: the Slurm cluster's shared files\n" + + "After=network-online.target\nWants=network-online.target\n\n" + + "[Mount]\nWhat=" + nfs + "\nWhere=" + path + "\nType=nfs\n" + + "Options=hard,nconnect=4,_netdev\nTimeoutSec=60\n\n" + + "[Install]\nWantedBy=remote-fs.target\n" +} + +// Users: Linux users llmgw makes from its accounts, the same login and UID +// on every machine, logging in with the SSH keys given in the portal. +const ( + // AuthorizedKeysDir holds each user's keys, root's to write, so a + // user's own home (on a file system shared with others) need hold none. + AuthorizedKeysDir = "/etc/llmgw/ssh-keys" + // SSHDropIn has sshd look there too, after the user's own files. + SSHDropIn = "/etc/ssh/sshd_config.d/50-llmgw-slurm.conf" + // MinUID is the lowest UID llmgw gives. + MinUID = 200000 +) + +// SSHConf is SSHDropIn's settings. +const SSHConf = "# Written by llmgw (BACKLOG P): the Slurm users' keys, from llmgw's portal.\n" + + "AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys2 " + AuthorizedKeysDir + "/%u\n" + +// login is what useradd takes, as llmgw gives it. +var login = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,31}$`) + +// ValidLogin is whether a login is one llmgw gives. +func ValidLogin(s string) bool { return login.MatchString(s) } + +// ParseSSHKeys reads OpenSSH public keys, one a line, as authorized_keys +// takes them, without options: what a person pastes into the portal. +// Blank lines and # comments are skipped; each key comes back as its type, +// its base64 and its comment. +func ParseSSHKeys(text string) ([]string, error) { + var out []string + for i, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + pk, comment, opts, _, err := ssh.ParseAuthorizedKey([]byte(line)) + if err != nil { + return nil, fmt.Errorf("line %d is not an SSH public key (as ~/.ssh/id_ed25519.pub has it)", i+1) + } + if len(opts) > 0 { + return nil, fmt.Errorf("line %d has options before its key; only the key is taken", i+1) + } + k := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(pk))) + if comment = strings.Join(strings.Fields(comment), " "); comment != "" { + k += " " + comment + } + if !slices.Contains(out, k) { + out = append(out, k) + } + } + if len(out) > 20 { + return nil, fmt.Errorf("%d keys: 20 at most", len(out)) + } + return out, nil +} diff --git a/internal/slurm/shared_test.go b/internal/slurm/shared_test.go new file mode 100644 index 0000000..4849940 --- /dev/null +++ b/internal/slurm/shared_test.go @@ -0,0 +1,42 @@ +package slurm + +import ( + "strings" + "testing" +) + +func TestExportsAndMountUnit(t *testing.T) { + e := Exports("/shared", []string{"10.0.0.22", "10.0.0.21", "10.0.0.22"}) + if !strings.Contains(e, `"/shared" 10.0.0.21(rw,sync,no_subtree_check,root_squash) 10.0.0.22(rw,`) || + strings.Count(e, "10.0.0.22") != 1 { + t.Fatalf("exports:\n%s", e) + } + if strings.Contains(Exports("/shared", nil), "/shared\"") { + t.Fatal("an export to nobody") + } + for path, want := range map[string]string{"/shared": "shared.mount", "/data/hpc": "data-hpc.mount", + "/hpc-data": `hpc\x2ddata.mount`, "/.b": `\x2eb.mount`} { + if got := MountUnitName(path); got != want { + t.Errorf("%s: %s, want %s", path, got, want) + } + } + if u := MountUnit("10.0.0.11:/shared", "/shared"); !strings.Contains(u, "What=10.0.0.11:/shared\nWhere=/shared\nType=nfs") { + t.Fatalf("unit:\n%s", u) + } +} + +func TestParseSSHKeys(t *testing.T) { + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILq3vMdLAKlaDdxwrSgCtP/j/qrF4nzxNBNGuJmK5vxh" + got, err := ParseSSHKeys("# mine\n\n" + key + " ayse@laptop \n" + key + " ayse@laptop\n") + if err != nil || len(got) != 1 || got[0] != key+" ayse@laptop" { + t.Fatalf("%q %v", got, err) + } + for _, bad := range []string{"not a key", `command="rm -rf /" ` + key} { + if _, err := ParseSSHKeys(bad); err == nil { + t.Errorf("%q taken", bad) + } + } + if !ValidLogin("ali_veli") || ValidLogin("9x") || ValidLogin("Ayse") { + t.Error("ValidLogin") + } +} diff --git a/internal/store/copy.go b/internal/store/copy.go index 589e936..fc96479 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -29,6 +29,7 @@ var copyOrder = []string{ "user_excluded_licences", // After users, which it points at. "user_directory", + "slurm_users", "preset_state", "api_keys", "key_claims", diff --git a/internal/store/migrations/postgres/0052_slurm_users.sql b/internal/store/migrations/postgres/0052_slurm_users.sql new file mode 100644 index 0000000..929f75d --- /dev/null +++ b/internal/store/migrations/postgres/0052_slurm_users.sql @@ -0,0 +1,12 @@ +-- Ported from the SQLite migration of the same number. +-- The Linux user llmgw makes for an account on every machine of the Slurm +-- cluster, when slurm.users is "llmgw": its login and UID (also its group's +-- GID), given once and never again to another, and the SSH public keys it +-- logs in with. No reference to users: a deleted account's row stays, so its +-- login and UID, still on the files it left, are not handed to someone else. +CREATE TABLE slurm_users ( + user_id TEXT PRIMARY KEY, + login TEXT NOT NULL UNIQUE, + uid INTEGER NOT NULL UNIQUE, + ssh_keys TEXT NOT NULL DEFAULT '' +); diff --git a/internal/store/migrations/sqlite/0052_slurm_users.sql b/internal/store/migrations/sqlite/0052_slurm_users.sql new file mode 100644 index 0000000..bc57477 --- /dev/null +++ b/internal/store/migrations/sqlite/0052_slurm_users.sql @@ -0,0 +1,11 @@ +-- The Linux user llmgw makes for an account on every machine of the Slurm +-- cluster, when slurm.users is "llmgw": its login and UID (also its group's +-- GID), given once and never again to another, and the SSH public keys it +-- logs in with. No reference to users: a deleted account's row stays, so its +-- login and UID, still on the files it left, are not handed to someone else. +CREATE TABLE slurm_users ( + user_id TEXT PRIMARY KEY, + login TEXT NOT NULL UNIQUE, + uid INTEGER NOT NULL UNIQUE, + ssh_keys TEXT NOT NULL DEFAULT '' +); diff --git a/internal/store/slurm_users.go b/internal/store/slurm_users.go new file mode 100644 index 0000000..f7dfc95 --- /dev/null +++ b/internal/store/slurm_users.go @@ -0,0 +1,174 @@ +package store + +import ( + "context" + "database/sql" + "strconv" + "strings" +) + +// SlurmUser is the Linux user llmgw makes for an active account on every +// machine of the Slurm cluster (slurm.users "llmgw"). +type SlurmUser struct { + UserID string + Email string + Name string + Login string + UID int + // SSHKeys are OpenSSH public keys, one a line. + SSHKeys []string +} + +// FirstSlurmUID is the first UID llmgw gives: far above what a +// distribution or a site's own users take, and below the range systemd +// keeps for its containers (524288). +const FirstSlurmUID = 200000 + +// reservedLogins are names an account's login is never given: the +// system's, Slurm's and llmgw's own. +var reservedLogins = map[string]bool{ + "root": true, "daemon": true, "bin": true, "sys": true, "sync": true, "games": true, "man": true, + "lp": true, "mail": true, "news": true, "uucp": true, "proxy": true, "backup": true, "list": true, + "irc": true, "nobody": true, "admin": true, "administrator": true, "slurm": true, "munge": true, + "mysql": true, "mariadb": true, "postgres": true, "llmgw": true, "sshd": true, "ubuntu": true, + "debian": true, "centos": true, "rocky": true, "ec2-user": true, "cloud-user": true, "syslog": true, + "messagebus": true, "polkitd": true, "nfsnobody": true, "operator": true, "ftp": true, "adm": true, + "halt": true, "shutdown": true, "www-data": true, "gnats": true, "user": true, "guest": true, +} + +// loginFrom is an e-mail address's part before the @, as a Linux login: +// lower case, letters, digits, _ and -, a letter first, 24 at most. +func loginFrom(email string) string { + local, _, _ := strings.Cut(strings.ToLower(email), "@") + var b strings.Builder + for _, r := range local { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_', r == '-': + b.WriteRune(r) + case r == '.' || r == '+': + b.WriteByte('_') + } + } + l := strings.Trim(b.String(), "_-") + if l == "" || l[0] < 'a' || l[0] > 'z' { + l = "u" + l + } + if len(l) > 24 { + l = l[:24] + } + return l +} + +// SlurmUsers are the active accounts as Linux users, each given its login +// and UID the first time it is asked for. +func (s *DB) SlurmUsers(ctx context.Context) ([]SlurmUser, error) { + type row struct { + SlurmUser + has bool + } + rows, err := queryAll(ctx, s.db, + `SELECT u.id, u.email, u.name, COALESCE(su.login, ''), COALESCE(su.uid, 0), COALESCE(su.ssh_keys, '') + FROM users u LEFT JOIN slurm_users su ON su.user_id = u.id + WHERE u.state = ? ORDER BY u.created_at`, + []any{string(UserActive)}, func(r *sql.Rows) (row, error) { + var x row + var keys string + err := r.Scan(&x.UserID, &x.Email, &x.Name, &x.Login, &x.UID, &keys) + x.SSHKeys, x.has = splitKeys(keys), x.Login != "" + return x, err + }) + if err != nil { + return nil, err + } + out := make([]SlurmUser, 0, len(rows)) + for _, x := range rows { + if !x.has { + login, uid, err := s.giveSlurmLogin(ctx, x.UserID, x.Email) + if err != nil { + return nil, err + } + x.Login, x.UID = login, uid + } + out = append(out, x.SlurmUser) + } + return out, nil +} + +// SlurmUserByID is one account's Linux user, given its login and UID if it +// has none yet. +func (s *DB) SlurmUserByID(ctx context.Context, userID string) (SlurmUser, error) { + u, err := s.UserByID(ctx, userID) + if err != nil { + return SlurmUser{}, err + } + out := SlurmUser{UserID: u.ID, Email: u.Email, Name: u.Name} + var keys string + err = s.db.QueryRowContext(ctx, `SELECT login, uid, ssh_keys FROM slurm_users WHERE user_id = ?`, userID). + Scan(&out.Login, &out.UID, &keys) + switch { + case err == sql.ErrNoRows: + out.Login, out.UID, err = s.giveSlurmLogin(ctx, u.ID, u.Email) + case err == nil: + out.SSHKeys = splitKeys(keys) + } + return out, err +} + +// SetSlurmSSHKeys replaces an account's SSH public keys, checked by the +// caller. +func (s *DB) SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error { + if _, err := s.SlurmUserByID(ctx, userID); err != nil { + return err + } + _, err := s.db.ExecContext(ctx, `UPDATE slurm_users SET ssh_keys = ? WHERE user_id = ?`, + strings.Join(keys, "\n"), userID) + return err +} + +// giveSlurmLogin gives an account its login, its e-mail's local part with a +// number after it if that is taken, and the next UID. +func (s *DB) giveSlurmLogin(ctx context.Context, userID, email string) (string, int, error) { + var login string + var uid int + err := s.inTx(ctx, func(c conn) error { + // Another pass may have given it meanwhile. + err := c.QueryRowContext(ctx, `SELECT login, uid FROM slurm_users WHERE user_id = ?`, userID).Scan(&login, &uid) + if err == nil { + return nil + } + if err != sql.ErrNoRows { + return err + } + if err := c.QueryRowContext(ctx, `SELECT COALESCE(MAX(uid), 0) FROM slurm_users`).Scan(&uid); err != nil { + return err + } + uid = max(uid+1, FirstSlurmUID) + base := loginFrom(email) + login = base + for n := 2; ; n++ { + if !reservedLogins[login] { + var taken int + if err := c.QueryRowContext(ctx, `SELECT COUNT(*) FROM slurm_users WHERE login = ?`, login).Scan(&taken); err != nil { + return err + } + if taken == 0 { + break + } + } + login = base + strconv.Itoa(n) + } + _, err = c.ExecContext(ctx, `INSERT INTO slurm_users (user_id, login, uid) VALUES (?, ?, ?)`, userID, login, uid) + return err + }) + return login, uid, err +} + +func splitKeys(s string) []string { + var out []string + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + out = append(out, l) + } + } + return out +} diff --git a/internal/store/slurm_users_test.go b/internal/store/slurm_users_test.go new file mode 100644 index 0000000..700b050 --- /dev/null +++ b/internal/store/slurm_users_test.go @@ -0,0 +1,57 @@ +package store + +import ( + "context" + "testing" + "time" +) + +func TestSlurmUsersGetLoginsAndUIDsOnce(t *testing.T) { + ctx := context.Background() + db := openDB(t) + for i, e := range []string{"ayse@a.test", "Ayse@b.test", "root@c.test", "9lives@d.test", "ali.veli@e.test"} { + id := "u" + string(rune('0'+i)) + if err := db.CreateUser(ctx, User{ID: id, Email: e, Name: e, PasswordHash: "x", State: UserActive, + CreatedAt: time.Unix(int64(i), 0)}); err != nil { + t.Fatal(err) + } + } + if err := db.CreateUser(ctx, User{ID: "p", Email: "pending@x.test", PasswordHash: "x", State: UserPending, + CreatedAt: time.Unix(9, 0)}); err != nil { + t.Fatal(err) + } + us, err := db.SlurmUsers(ctx) + if err != nil { + t.Fatal(err) + } + want := []struct { + login string + uid int + }{{"ayse", 200000}, {"ayse2", 200001}, {"root2", 200002}, {"u9lives", 200003}, {"ali_veli", 200004}} + if len(us) != len(want) { + t.Fatalf("%d users: %+v", len(us), us) + } + for i, w := range want { + if us[i].Login != w.login || us[i].UID != w.uid { + t.Errorf("%s: %s %d, want %s %d", us[i].Email, us[i].Login, us[i].UID, w.login, w.uid) + } + } + if err := db.SetSlurmSSHKeys(ctx, "u0", []string{"ssh-ed25519 AAAA a", "ssh-ed25519 BBBB b"}); err != nil { + t.Fatal(err) + } + again, _ := db.SlurmUsers(ctx) + if again[0].UID != 200000 || len(again[0].SSHKeys) != 2 { + t.Fatalf("asked again: %+v", again[0]) + } + // A deleted account's UID is not given again. + if _, err := db.raw.ExecContext(ctx, `DELETE FROM users WHERE id = 'u4'`); err != nil { + t.Fatal(err) + } + if err := db.ApproveUser(ctx, "p", "", "admin", time.Now()); err != nil { + t.Fatal(err) + } + u, err := db.SlurmUserByID(ctx, "p") + if err != nil || u.UID != 200005 || u.Login != "pending" { + t.Fatalf("%+v %v", u, err) + } +} diff --git a/internal/web/account.go b/internal/web/account.go index aca225a..67569ae 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -17,6 +17,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/account" "gitea.mynodes.xyz/ilker/llmgw/internal/audit" "gitea.mynodes.xyz/ilker/llmgw/internal/clientip" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/vscodeext" @@ -151,6 +152,10 @@ type AccountPages struct { resets account.ResetStore mailer Mailer publicURL string + // slurmKeys and slurmSettings offer the Slurm cluster's login; see + // SetSlurm. + slurmKeys SlurmKeys + slurmSettings func() config.SlurmSettings } // ServedModels supplies the names the profile page's example should use. @@ -207,6 +212,7 @@ func (p *AccountPages) Register(mux *http.ServeMux) { mux.HandleFunc("POST /profile/keys/{id}/revoke", p.selfAudited(p.revokeKey)) mux.HandleFunc("POST /profile/password", p.selfAudited(p.changePassword)) mux.HandleFunc("POST /profile/licences", p.selfAudited(p.setLicences)) + mux.HandleFunc("POST /profile/ssh-keys", p.selfAudited(p.setSlurmKeys)) mux.HandleFunc("POST /profile/connect", p.selfAudited(p.connect)) mux.HandleFunc("GET /setup/{tool}", p.setupScript) mux.HandleFunc("GET /setup/vscode/"+vscodeext.FileName, p.vsix) @@ -415,6 +421,9 @@ type profileData struct { Licences *LicenceChoices LicencesSaved bool LicenceError string + // Slurm is the person's login to the Slurm cluster and its SSH keys; + // nil unless llmgw makes the cluster's users. + Slurm *slurmView PasswordChanged bool PasswordError string @@ -558,6 +567,7 @@ func (p *AccountPages) render(w http.ResponseWriter, r *http.Request, d.Keys = keys p.fillPlaces(r.Context(), langOf(r), user, &d) p.fillLicences(r.Context(), user.ID, &d) + p.fillSlurm(r.Context(), user, &d) render(w, r, "profile", d) } diff --git a/internal/web/locales/en/account.json b/internal/web/locales/en/account.json index ad007f6..0985fe1 100644 --- a/internal/web/locales/en/account.json +++ b/internal/web/locales/en/account.json @@ -188,5 +188,13 @@ "reset.bad": "This link does not work: it was used already, it expired, or a newer one was sent.", "reset.err.mismatch": "the two passwords are not the same", "reset.mail.subject": "Set a new password for llmgw", - "reset.mail.body": "Hello %s,\n\nA link to set a new password for your llmgw account was asked for:\n\n%s\n\nIt works once, for %d minutes. If you did not ask for it, ignore this message: your password stays as it is.\n" + "reset.mail.body": "Hello %s,\n\nA link to set a new password for your llmgw account was asked for:\n\n%s\n\nIt works once, for %d minutes. If you did not ask for it, ignore this message: your password stays as it is.\n", + "profile.slurm": "Slurm: login and SSH keys", + "profile.slurm.saved": "Your keys are kept; every machine of the Slurm cluster has them within a minute.", + "profile.slurm.intro": "You are %s (UID %d) on every machine of the Slurm cluster, your home %s. Log in to the controller with one of the keys below and submit jobs there.", + "profile.slurm.keys": "SSH public keys, one a line (as ~/.ssh/id_ed25519.pub)", + "profile.slurm.save": "Save keys", + "profile.slurm.note": "Only the public key: the private one never leaves your computer. An empty box takes every key away.", + "profile.slurm.err_keys": "Not kept: %s.", + "profile.slurm.placeholder": "ssh-ed25519 AAAA… you@laptop" } diff --git a/internal/web/locales/tr/account.json b/internal/web/locales/tr/account.json index 73e44b8..a5217a8 100644 --- a/internal/web/locales/tr/account.json +++ b/internal/web/locales/tr/account.json @@ -188,5 +188,13 @@ "reset.bad": "Bu bağlantı çalışmıyor: daha önce kullanılmış, süresi dolmuş ya da daha yenisi gönderilmiş.", "reset.err.mismatch": "iki şifre aynı değil", "reset.mail.subject": "llmgw için yeni şifre belirleyin", - "reset.mail.body": "Merhaba %s,\n\nllmgw hesabınız için yeni şifre belirleme bağlantısı istendi:\n\n%s\n\nBir kez ve %d dakika boyunca geçerli. Siz istemediyseniz bu mesajı dikkate almayın: şifreniz değişmez.\n" + "reset.mail.body": "Merhaba %s,\n\nllmgw hesabınız için yeni şifre belirleme bağlantısı istendi:\n\n%s\n\nBir kez ve %d dakika boyunca geçerli. Siz istemediyseniz bu mesajı dikkate almayın: şifreniz değişmez.\n", + "profile.slurm": "Slurm: giriş ve SSH anahtarları", + "profile.slurm.saved": "Anahtarlarınız kaydedildi; Slurm cluster'ının her makinesine bir dakika içinde ulaşır.", + "profile.slurm.intro": "Slurm cluster'ının her makinesinde %s (UID %d) olarak varsınız, ev dizininiz %s. Aşağıdaki anahtarlardan biriyle kontrolcüye bağlanıp işlerinizi oradan gönderin.", + "profile.slurm.keys": "SSH açık anahtarları, satır başına bir tane (~/.ssh/id_ed25519.pub gibi)", + "profile.slurm.save": "Anahtarları kaydet", + "profile.slurm.note": "Yalnızca açık anahtar: gizli anahtar bilgisayarınızdan hiç çıkmaz. Kutuyu boşaltmak bütün anahtarları kaldırır.", + "profile.slurm.err_keys": "Kaydedilmedi: %s.", + "profile.slurm.placeholder": "ssh-ed25519 AAAA… siz@dizüstü" } diff --git a/internal/web/slurmkeys.go b/internal/web/slurmkeys.go new file mode 100644 index 0000000..9de773f --- /dev/null +++ b/internal/web/slurmkeys.go @@ -0,0 +1,98 @@ +package web + +import ( + "context" + "net/http" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// SlurmKeys keeps the SSH keys a person logs in to the Slurm cluster with. +type SlurmKeys interface { + SlurmUserByID(ctx context.Context, userID string) (store.SlurmUser, error) + SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error +} + +// SetSlurm has the profile page offer the Slurm cluster's login and SSH +// keys while settings say llmgw makes its users. +func (p *AccountPages) SetSlurm(keys SlurmKeys, settings func() config.SlurmSettings) { + p.slurmKeys, p.slurmSettings = keys, settings +} + +// slurmView is the profile's Slurm part. +type slurmView struct { + Login string + UID int + Home string + // Host is where jobs are submitted: the controller. + Host string + Keys string + Saved bool + Error string +} + +// slurmOn is the settings, and whether the page offers the part. +func (p *AccountPages) slurmOn() (config.SlurmSettings, bool) { + if p.slurmKeys == nil || p.slurmSettings == nil { + return config.SlurmSettings{}, false + } + s := p.slurmSettings() + return s, s.Enabled && s.Users == config.UsersLlmgw +} + +// fillSlurm puts the person's Slurm login in d, keeping what d says of a +// save. +func (p *AccountPages) fillSlurm(ctx context.Context, user store.User, d *profileData) { + set, on := p.slurmOn() + if !on || !user.CanSignIn() { + return + } + u, err := p.slurmKeys.SlurmUserByID(ctx, user.ID) + if err != nil { + p.log.Error("could not read a person's Slurm login", "user", user.ID, "err", err) + return + } + v := &slurmView{Login: u.Login, UID: u.UID, Home: set.HomeDir(u.Login), Host: set.Controller, + Keys: strings.Join(u.SSHKeys, "\n")} + if d.Slurm != nil { + v.Saved, v.Error = d.Slurm.Saved, d.Slurm.Error + if v.Error != "" { + v.Keys = d.Slurm.Keys + } + } + d.Slurm = v +} + +func (p *AccountPages) setSlurmKeys(w http.ResponseWriter, r *http.Request) { + noStore(w) + sess, user, ok := p.session(r) + if !ok { + http.Redirect(w, r, "/login", http.StatusSeeOther) + return + } + if _, on := p.slurmOn(); !on { + http.NotFound(w, r) + return + } + if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) { + badForm(w, r) + return + } + l := langOf(r) + text := r.FormValue("keys") + keys, err := slurm.ParseSSHKeys(text) + if err != nil { + p.render(w, r, sess, user, profileData{Slurm: &slurmView{Keys: text, Error: l.T("profile.slurm.err_keys", err.Error())}}) + return + } + if err := p.slurmKeys.SetSlurmSSHKeys(r.Context(), user.ID, keys); err != nil { + p.log.Error("could not keep a person's SSH keys", "user", user.ID, "err", err) + p.render(w, r, sess, user, profileData{Slurm: &slurmView{Keys: text, Error: l.T("profile.err.try_again")}}) + return + } + p.log.Info("a user changed their SSH keys for Slurm", "user", user.ID, "keys", len(keys)) + p.render(w, r, sess, user, profileData{Slurm: &slurmView{Saved: true}}) +} diff --git a/internal/web/slurmkeys_test.go b/internal/web/slurmkeys_test.go new file mode 100644 index 0000000..c0b6d82 --- /dev/null +++ b/internal/web/slurmkeys_test.go @@ -0,0 +1,74 @@ +package web + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func TestTheProfileKeepsAPersonsSSHKeysForSlurm(t *testing.T) { + ctx := t.Context() + st, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + ayse := store.User{ID: "usr-a", Email: "ayse@llmt.test", Name: "Ayşe", State: store.UserActive, CreatedAt: time.Now()} + if err := st.CreateUser(ctx, ayse); err != nil { + t.Fatal(err) + } + set := config.SlurmSettings{Enabled: true, Controller: "llmt-1", Users: config.UsersLlmgw, + Shared: config.SharedStorage{Path: "/shared", Server: "llmt-1"}} + p := NewAccountPages(fakeAccounts{user: ayse}, st, nil, "v0", slog.New(slog.DiscardHandler)) + p.SetSlurm(st, func() config.SlurmSettings { return set }) + mux := http.NewServeMux() + p.Register(mux) + do := func(form url.Values) (int, string) { + req := httptest.NewRequest(http.MethodGet, "/profile", nil) + if form != nil { + form.Set("csrf", "tok") + req = httptest.NewRequest(http.MethodPost, "/profile/ssh-keys", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + req.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"}) + req.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"}) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec.Code, rec.Body.String() + } + _, page := do(nil) + for _, want := range []string{`action="/profile/ssh-keys"`, "ssh ayse@llmt-1", "UID 200000", "/shared/home/ayse"} { + if !strings.Contains(page, want) { + t.Errorf("the page lacks %q", want) + } + } + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILq3vMdLAKlaDdxwrSgCtP/j/qrF4nzxNBNGuJmK5vxh ayse@laptop" + _, saved := do(url.Values{"keys": {key + "\n"}}) + u, _ := st.SlurmUserByID(ctx, ayse.ID) + if len(u.SSHKeys) != 1 || u.SSHKeys[0] != key || !strings.Contains(saved, "within a minute") { + t.Fatalf("kept %v", u.SSHKeys) + } + _, bad := do(url.Values{"keys": {"not a key"}}) + if !strings.Contains(bad, "Not kept") || !strings.Contains(bad, "not a key") { + t.Error("a bad key was not refused, or the box lost what was typed") + } + if u, _ := st.SlurmUserByID(ctx, ayse.ID); len(u.SSHKeys) != 1 { + t.Error("a refused form changed the keys") + } + // Off: neither the part nor the form. + set.Users = "" + if _, page := do(nil); strings.Contains(page, "ssh-keys") { + t.Error("offered with slurm.users off") + } + if code, _ := do(url.Values{"keys": {key}}); code != http.StatusNotFound { + t.Errorf("posted with slurm.users off: %d", code) + } +} diff --git a/internal/web/templates/account.html b/internal/web/templates/account.html index 11114b7..f1d79ba 100644 --- a/internal/web/templates/account.html +++ b/internal/web/templates/account.html @@ -232,6 +232,21 @@ document.addEventListener("click", (e) => {

{{t "profile.licences.note"}}

{{end}} +{{with .Slurm}} +

{{t "profile.slurm"}}

+{{if .Saved}}

{{t "profile.slurm.saved"}}

{{end}} +{{if .Error}}

{{.Error}}

{{end}} +{{if .Login}}

{{t "profile.slurm.intro" .Login .UID .Home}}

+
ssh {{.Login}}@{{.Host}}
+sbatch job.sh
{{end}} +
+ + + +
+

{{t "profile.slurm.note"}}

+{{end}} +

{{t "profile.password"}}

{{if .PasswordChanged}}

{{t "profile.password_changed"}}

{{end}} {{if .PasswordError}}

{{.PasswordError}}

{{end}}