From d8ce573bdbdfd27381e639974de3e290491439c7 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 12 Nov 2026 21:32:56 +0100 Subject: [PATCH] Slurm's shared files and users: one path everywhere, accounts as Linux users slurm.shared mounts the site's NFS export, has an llmgw node export the path over NFS to the Slurm machines alone (2049 in the firewall), or looks for a file system the site mounts. slurm.users "llmgw" makes each active account a Linux user on every Slurm machine: login from its e-mail, a UID from 200000 kept in slurm_users and never reused, its home on the shared path, its SSH keys from the portal's Profile (or ctl slurm users keys) read by sshd from /etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first machine and llmgw's users; --slurm-shared and --slurm-users change that. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 9 + CHANGELOG.md | 8 + api/gen/llmgw/v1/agent.pb.go | 312 +++++++++++++++--- api/proto/llmgw/v1/agent.proto | 40 +++ docs/admin-api.openapi.json | 152 +++++++++ docs/console.md | 5 +- docs/install.md | 21 +- docs/reference.md | 50 +++ internal/agent/firewall.go | 14 +- internal/agent/slurmnode/shared.go | 150 +++++++++ internal/agent/slurmnode/shared_test.go | 124 +++++++ internal/agent/slurmnode/slurmnode.go | 30 ++ internal/agent/slurmnode/slurmnode_test.go | 6 +- internal/agent/slurmnode/users.go | 238 +++++++++++++ internal/cli/ctl_slurm.go | 171 +++++++++- internal/cli/server.go | 11 + internal/cli/serverinit.go | 8 + internal/cli/serverinit_choices.go | 49 ++- internal/cli/serverinit_choices_test.go | 4 +- internal/cli/tui_choices.go | 8 +- internal/cli/tui_slurm.go | 23 +- internal/cli/tui_slurm_test.go | 8 +- internal/config/slurm.go | 84 ++++- internal/config/slurm_test.go | 43 +++ internal/control/adminapi/api.go | 2 + internal/control/adminapi/routes.go | 9 + internal/control/adminapi/slurm.go | 91 +++++ internal/control/slurmsvc/service.go | 171 +++++++++- internal/control/slurmsvc/shared_test.go | 85 +++++ internal/docgen/reference_gen.go | 7 + internal/router/handlers.go | 3 + internal/router/router.go | 7 + internal/slurm/shared.go | 127 +++++++ internal/slurm/shared_test.go | 42 +++ internal/store/copy.go | 1 + .../migrations/postgres/0052_slurm_users.sql | 12 + .../migrations/sqlite/0052_slurm_users.sql | 11 + internal/store/slurm_users.go | 174 ++++++++++ internal/store/slurm_users_test.go | 57 ++++ internal/web/account.go | 10 + internal/web/locales/en/account.json | 10 +- internal/web/locales/tr/account.json | 10 +- internal/web/slurmkeys.go | 98 ++++++ internal/web/slurmkeys_test.go | 74 +++++ internal/web/templates/account.html | 15 + 45 files changed, 2505 insertions(+), 79 deletions(-) create mode 100644 internal/agent/slurmnode/shared.go create mode 100644 internal/agent/slurmnode/shared_test.go create mode 100644 internal/agent/slurmnode/users.go create mode 100644 internal/control/slurmsvc/shared_test.go create mode 100644 internal/slurm/shared.go create mode 100644 internal/slurm/shared_test.go create mode 100644 internal/store/migrations/postgres/0052_slurm_users.sql create mode 100644 internal/store/migrations/sqlite/0052_slurm_users.sql create mode 100644 internal/store/slurm_users.go create mode 100644 internal/store/slurm_users_test.go create mode 100644 internal/web/slurmkeys.go create mode 100644 internal/web/slurmkeys_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 002b1ab..063e9c5 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2057,6 +2057,15 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t controller later with `ctl slurm setup`; batch output lands on the job's first node until there is shared storage; on shared CPUs the judge refuses the first request after a placement change (fail closed, ~10 s), and its advice repeats `--cores 4` to a judge that has 4. +- Shared files and users, built 2026-11-12: `slurm.shared` {path, nfs | server} — the site's export + mounted (a systemd mount unit, hard, nconnect=4), an llmgw node exporting the path itself + (/etc/exports.d, to the Slurm machines' addresses, root_squash; 2049 in the firewall), or the site's + own mount only looked for; `slurm.users: llmgw` — every active account a Linux user everywhere + (`slurm_users`: login from the e-mail, UID from 200000, never reused; homes made on the exporting + machine, else the controller; keys in /etc/llmgw/ssh-keys/%u through an sshd drop-in), SSH keys + from the portal's Profile or `ctl slurm users keys`. A user already on a machine with another UID + is left alone and said. The site has AD (not yet live): users from it, through sssd, are the next + step (`slurm.users` empty today leaves them to the site). - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm: each project an account (`sacctmgr add account`, parent its team and organization), each member a user of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a5321f..e8d66d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,14 @@ APIs change between minors without a deprecation period. machine alone (Slurm takes MySQL or MariaDB only). Slurm's ports (6817–6819, srun's 60001–63000) answer the cluster's networks only, as llmgw's own do. +- **Slurm's shared files and users.** `slurm.shared` (`ctl slurm shared`) is + one path on every Slurm machine: the site's NFS export mounted, an llmgw + node exporting it over NFS (what `server init --slurm` chooses, `/shared` + from the first machine), or a file system the site mounts. `slurm.users: + llmgw` (`ctl slurm users on`) makes each active account a Linux user on + every Slurm machine, the same login and UID (from 200000, never reused) + everywhere, its home on the shared path; people add SSH keys in the + portal's Profile (`ctl slurm users keys` for an operator). ### Changed diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index 9913d5f..524ffc6 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5304,7 +5304,17 @@ type Slurm struct { Files map[string]string `protobuf:"bytes,6,rep,name=files,proto3" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // lent says Slurm may run jobs here now: resume the node if llmgw drained // it. Otherwise it is drained, its jobs left to finish. - Lent bool `protobuf:"varint,7,opt,name=lent,proto3" json:"lent,omitempty"` + Lent bool `protobuf:"varint,7,opt,name=lent,proto3" json:"lent,omitempty"` + // shared is the file system every machine of the Slurm cluster sees at + // the same path; unset or with no path, none. + Shared *SlurmShared `protobuf:"bytes,8,opt,name=shared,proto3" json:"shared,omitempty"` + // manage_users has llmgw keep the Linux users below on this machine, + // and take away those it made that are no longer among them. + ManageUsers bool `protobuf:"varint,9,opt,name=manage_users,json=manageUsers,proto3" json:"manage_users,omitempty"` + Users []*SlurmUser `protobuf:"bytes,10,rep,name=users,proto3" json:"users,omitempty"` + // make_homes has this machine make the users' home directories: the one + // that exports the shared path, else the controller. + MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -5388,6 +5398,188 @@ func (x *Slurm) GetLent() bool { return false } +func (x *Slurm) GetShared() *SlurmShared { + if x != nil { + return x.Shared + } + return nil +} + +func (x *Slurm) GetManageUsers() bool { + if x != nil { + return x.ManageUsers + } + return false +} + +func (x *Slurm) GetUsers() []*SlurmUser { + if x != nil { + return x.Users + } + return nil +} + +func (x *Slurm) GetMakeHomes() bool { + if x != nil { + return x.MakeHomes + } + return false +} + +// SlurmShared is where the Slurm cluster's shared files come from. +type SlurmShared struct { + state protoimpl.MessageState `protogen:"open.v1"` + // path is where every machine sees them, as /shared. + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + // nfs is an export, host:/path, to mount at path; empty when this + // machine exports path itself (serve) or the site mounts it. + Nfs string `protobuf:"bytes,2,opt,name=nfs,proto3" json:"nfs,omitempty"` + // serve has this machine export path over NFS to clients. + Serve bool `protobuf:"varint,3,opt,name=serve,proto3" json:"serve,omitempty"` + // clients are the addresses the export answers. + Clients []string `protobuf:"bytes,4,rep,name=clients,proto3" json:"clients,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SlurmShared) Reset() { + *x = SlurmShared{} + mi := &file_llmgw_v1_agent_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SlurmShared) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SlurmShared) ProtoMessage() {} + +func (x *SlurmShared) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[49] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SlurmShared.ProtoReflect.Descriptor instead. +func (*SlurmShared) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{49} +} + +func (x *SlurmShared) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *SlurmShared) GetNfs() string { + if x != nil { + return x.Nfs + } + return "" +} + +func (x *SlurmShared) GetServe() bool { + if x != nil { + return x.Serve + } + return false +} + +func (x *SlurmShared) GetClients() []string { + if x != nil { + return x.Clients + } + return nil +} + +// SlurmUser is a Linux user that runs jobs, the same on every machine. +type SlurmUser struct { + state protoimpl.MessageState `protogen:"open.v1"` + Login string `protobuf:"bytes,1,opt,name=login,proto3" json:"login,omitempty"` + // uid is also the user's own group's gid. + Uid uint32 `protobuf:"varint,2,opt,name=uid,proto3" json:"uid,omitempty"` + // name is the account's, for the GECOS field. + Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Home string `protobuf:"bytes,4,opt,name=home,proto3" json:"home,omitempty"` + // ssh_keys are OpenSSH public keys, one a line of authorized_keys. + SshKeys []string `protobuf:"bytes,5,rep,name=ssh_keys,json=sshKeys,proto3" json:"ssh_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SlurmUser) Reset() { + *x = SlurmUser{} + mi := &file_llmgw_v1_agent_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SlurmUser) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SlurmUser) ProtoMessage() {} + +func (x *SlurmUser) ProtoReflect() protoreflect.Message { + mi := &file_llmgw_v1_agent_proto_msgTypes[50] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SlurmUser.ProtoReflect.Descriptor instead. +func (*SlurmUser) Descriptor() ([]byte, []int) { + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{50} +} + +func (x *SlurmUser) GetLogin() string { + if x != nil { + return x.Login + } + return "" +} + +func (x *SlurmUser) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *SlurmUser) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SlurmUser) GetHome() string { + if x != nil { + return x.Home + } + return "" +} + +func (x *SlurmUser) GetSshKeys() []string { + if x != nil { + return x.SshKeys + } + return nil +} + // SlurmStatus is what this node's Slurm daemons are doing, sent when it // changes and every few minutes while the node has a part in Slurm. type SlurmStatus struct { @@ -5414,13 +5606,18 @@ type SlurmStatus struct { // slurmdbd_running is the accounting daemon on the controller, which keeps // the jobs' records in a MariaDB beside it. SlurmdbdRunning bool `protobuf:"varint,10,opt,name=slurmdbd_running,json=slurmdbdRunning,proto3" json:"slurmdbd_running,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // shared is the shared path as this machine has it: "mounted", + // "exported", "missing", or empty when there is none. + Shared string `protobuf:"bytes,11,opt,name=shared,proto3" json:"shared,omitempty"` + // users is how many of llmgw's users this machine has. + Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SlurmStatus) Reset() { *x = SlurmStatus{} - mi := &file_llmgw_v1_agent_proto_msgTypes[49] + mi := &file_llmgw_v1_agent_proto_msgTypes[51] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5432,7 +5629,7 @@ func (x *SlurmStatus) String() string { func (*SlurmStatus) ProtoMessage() {} func (x *SlurmStatus) ProtoReflect() protoreflect.Message { - mi := &file_llmgw_v1_agent_proto_msgTypes[49] + mi := &file_llmgw_v1_agent_proto_msgTypes[51] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5445,7 +5642,7 @@ func (x *SlurmStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use SlurmStatus.ProtoReflect.Descriptor instead. func (*SlurmStatus) Descriptor() ([]byte, []int) { - return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{49} + return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{51} } func (x *SlurmStatus) GetVersion() string { @@ -5518,6 +5715,20 @@ func (x *SlurmStatus) GetSlurmdbdRunning() bool { return false } +func (x *SlurmStatus) GetShared() string { + if x != nil { + return x.Shared + } + return "" +} + +func (x *SlurmStatus) GetUsers() int32 { + if x != nil { + return x.Users + } + return 0 +} + var File_llmgw_v1_agent_proto protoreflect.FileDescriptor const file_llmgw_v1_agent_proto_rawDesc = "" + @@ -6023,7 +6234,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" + "\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" + - "\x04last\x18\x05 \x01(\bR\x04last\"\x8d\x02\n" + + "\x04last\x18\x05 \x01(\bR\x04last\"\xa9\x03\n" + "\x05Slurm\x12\x1e\n" + "\n" + "controller\x18\x01 \x01(\bR\n" + @@ -6034,11 +6245,28 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "confServer\x12\x1b\n" + "\tmunge_key\x18\x05 \x01(\fR\bmungeKey\x120\n" + "\x05files\x18\x06 \x03(\v2\x1a.llmgw.v1.Slurm.FilesEntryR\x05files\x12\x12\n" + - "\x04lent\x18\a \x01(\bR\x04lent\x1a8\n" + + "\x04lent\x18\a \x01(\bR\x04lent\x12-\n" + + "\x06shared\x18\b \x01(\v2\x15.llmgw.v1.SlurmSharedR\x06shared\x12!\n" + + "\fmanage_users\x18\t \x01(\bR\vmanageUsers\x12)\n" + + "\x05users\x18\n" + + " \x03(\v2\x13.llmgw.v1.SlurmUserR\x05users\x12\x1d\n" + + "\n" + + "make_homes\x18\v \x01(\bR\tmakeHomes\x1a8\n" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbd\x02\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"c\n" + + "\vSlurmShared\x12\x12\n" + + "\x04path\x18\x01 \x01(\tR\x04path\x12\x10\n" + + "\x03nfs\x18\x02 \x01(\tR\x03nfs\x12\x14\n" + + "\x05serve\x18\x03 \x01(\bR\x05serve\x12\x18\n" + + "\aclients\x18\x04 \x03(\tR\aclients\"v\n" + + "\tSlurmUser\x12\x14\n" + + "\x05login\x18\x01 \x01(\tR\x05login\x12\x10\n" + + "\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" + + "\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" + + "\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" + + "\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xeb\x02\n" + "\vSlurmStatus\x12\x18\n" + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" + "\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" + @@ -6050,7 +6278,9 @@ const file_llmgw_v1_agent_proto_rawDesc = "" + "\x05error\x18\b \x01(\tR\x05error\x12!\n" + "\ffiles_sha256\x18\t \x01(\tR\vfilesSha256\x12)\n" + "\x10slurmdbd_running\x18\n" + - " \x01(\bR\x0fslurmdbdRunning2O\n" + + " \x01(\bR\x0fslurmdbdRunning\x12\x16\n" + + "\x06shared\x18\v \x01(\tR\x06shared\x12\x14\n" + + "\x05users\x18\f \x01(\x05R\x05users2O\n" + "\fAgentService\x12?\n" + "\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" + "\fcom.llmgw.v1B\n" + @@ -6069,7 +6299,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte { } var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 59) +var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 61) var file_llmgw_v1_agent_proto_goTypes = []any{ (ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State (*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage @@ -6121,16 +6351,18 @@ var file_llmgw_v1_agent_proto_goTypes = []any{ (*TrainStatus)(nil), // 47: llmgw.v1.TrainStatus (*TrainArtifact)(nil), // 48: llmgw.v1.TrainArtifact (*Slurm)(nil), // 49: llmgw.v1.Slurm - (*SlurmStatus)(nil), // 50: llmgw.v1.SlurmStatus - nil, // 51: llmgw.v1.ControlMessage.TraceEntry - nil, // 52: llmgw.v1.Register.LabelsEntry - nil, // 53: llmgw.v1.Register.EngineVersionsEntry - nil, // 54: llmgw.v1.GPU.NicPathsEntry - nil, // 55: llmgw.v1.IBPort.CountersEntry - nil, // 56: llmgw.v1.Heartbeat.EngineVersionsEntry - nil, // 57: llmgw.v1.EngineWant.OfflineSha256Entry - nil, // 58: llmgw.v1.EngineProfile.EnvEntry - nil, // 59: llmgw.v1.Slurm.FilesEntry + (*SlurmShared)(nil), // 50: llmgw.v1.SlurmShared + (*SlurmUser)(nil), // 51: llmgw.v1.SlurmUser + (*SlurmStatus)(nil), // 52: llmgw.v1.SlurmStatus + nil, // 53: llmgw.v1.ControlMessage.TraceEntry + nil, // 54: llmgw.v1.Register.LabelsEntry + nil, // 55: llmgw.v1.Register.EngineVersionsEntry + nil, // 56: llmgw.v1.GPU.NicPathsEntry + nil, // 57: llmgw.v1.IBPort.CountersEntry + nil, // 58: llmgw.v1.Heartbeat.EngineVersionsEntry + nil, // 59: llmgw.v1.EngineWant.OfflineSha256Entry + nil, // 60: llmgw.v1.EngineProfile.EnvEntry + nil, // 61: llmgw.v1.Slurm.FilesEntry } var file_llmgw_v1_agent_proto_depIdxs = []int32{ 8, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register @@ -6143,7 +6375,7 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 2, // 7: llmgw.v1.AgentMessage.kv_events:type_name -> llmgw.v1.KVEvents 47, // 8: llmgw.v1.AgentMessage.train_status:type_name -> llmgw.v1.TrainStatus 48, // 9: llmgw.v1.AgentMessage.train_artifact:type_name -> llmgw.v1.TrainArtifact - 50, // 10: llmgw.v1.AgentMessage.slurm_status:type_name -> llmgw.v1.SlurmStatus + 52, // 10: llmgw.v1.AgentMessage.slurm_status:type_name -> llmgw.v1.SlurmStatus 30, // 11: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard 33, // 12: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard 34, // 13: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard @@ -6161,11 +6393,11 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 45, // 25: llmgw.v1.ControlMessage.train_start:type_name -> llmgw.v1.TrainStart 46, // 26: llmgw.v1.ControlMessage.train_cancel:type_name -> llmgw.v1.TrainCancel 49, // 27: llmgw.v1.ControlMessage.slurm:type_name -> llmgw.v1.Slurm - 51, // 28: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry - 52, // 29: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry + 53, // 28: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry + 54, // 29: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry 12, // 30: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU 14, // 31: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment - 53, // 32: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry + 55, // 32: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry 11, // 33: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard 15, // 34: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights 10, // 35: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage @@ -6173,15 +6405,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 21, // 37: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX 0, // 38: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State 27, // 39: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute - 54, // 40: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry + 56, // 40: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry 13, // 41: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice 18, // 42: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad 17, // 43: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor - 55, // 44: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry + 57, // 44: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry 20, // 45: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager 10, // 46: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage 15, // 47: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights - 56, // 48: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry + 58, // 48: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry 26, // 49: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus 16, // 50: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad 19, // 51: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort @@ -6189,21 +6421,23 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{ 22, // 53: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState 21, // 54: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX 25, // 55: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant - 57, // 56: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry + 59, // 56: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry 0, // 57: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State 29, // 58: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress 27, // 59: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute 31, // 60: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile - 58, // 61: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry + 60, // 61: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry 44, // 62: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter - 59, // 63: llmgw.v1.Slurm.files:type_name -> llmgw.v1.Slurm.FilesEntry - 1, // 64: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage - 3, // 65: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage - 65, // [65:66] is the sub-list for method output_type - 64, // [64:65] is the sub-list for method input_type - 64, // [64:64] is the sub-list for extension type_name - 64, // [64:64] is the sub-list for extension extendee - 0, // [0:64] is the sub-list for field type_name + 61, // 63: llmgw.v1.Slurm.files:type_name -> llmgw.v1.Slurm.FilesEntry + 50, // 64: llmgw.v1.Slurm.shared:type_name -> llmgw.v1.SlurmShared + 51, // 65: llmgw.v1.Slurm.users:type_name -> llmgw.v1.SlurmUser + 1, // 66: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage + 3, // 67: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage + 67, // [67:68] is the sub-list for method output_type + 66, // [66:67] is the sub-list for method input_type + 66, // [66:66] is the sub-list for extension type_name + 66, // [66:66] is the sub-list for extension extendee + 0, // [0:66] is the sub-list for field type_name } func init() { file_llmgw_v1_agent_proto_init() } @@ -6249,7 +6483,7 @@ func file_llmgw_v1_agent_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)), NumEnums: 1, - NumMessages: 59, + NumMessages: 61, NumExtensions: 0, NumServices: 1, }, diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index 45318a2..29bbc08 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1010,6 +1010,41 @@ message Slurm { // lent says Slurm may run jobs here now: resume the node if llmgw drained // it. Otherwise it is drained, its jobs left to finish. bool lent = 7; + // shared is the file system every machine of the Slurm cluster sees at + // the same path; unset or with no path, none. + SlurmShared shared = 8; + // manage_users has llmgw keep the Linux users below on this machine, + // and take away those it made that are no longer among them. + bool manage_users = 9; + repeated SlurmUser users = 10; + // make_homes has this machine make the users' home directories: the one + // that exports the shared path, else the controller. + bool make_homes = 11; +} + +// SlurmShared is where the Slurm cluster's shared files come from. +message SlurmShared { + // path is where every machine sees them, as /shared. + string path = 1; + // nfs is an export, host:/path, to mount at path; empty when this + // machine exports path itself (serve) or the site mounts it. + string nfs = 2; + // serve has this machine export path over NFS to clients. + bool serve = 3; + // clients are the addresses the export answers. + repeated string clients = 4; +} + +// SlurmUser is a Linux user that runs jobs, the same on every machine. +message SlurmUser { + string login = 1; + // uid is also the user's own group's gid. + uint32 uid = 2; + // name is the account's, for the GECOS field. + string name = 3; + string home = 4; + // ssh_keys are OpenSSH public keys, one a line of authorized_keys. + repeated string ssh_keys = 5; } // SlurmStatus is what this node's Slurm daemons are doing, sent when it @@ -1037,4 +1072,9 @@ message SlurmStatus { // slurmdbd_running is the accounting daemon on the controller, which keeps // the jobs' records in a MariaDB beside it. bool slurmdbd_running = 10; + // shared is the shared path as this machine has it: "mounted", + // "exported", "missing", or empty when there is none. + string shared = 11; + // users is how many of llmgw's users this machine has. + int32 users = 12; } diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index a8ba48e..0a2df86 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -5109,6 +5109,89 @@ } } }, + "/api/v1/slurm/users": { + "get": { + "operationId": "getSlurmUsers", + "summary": "The Linux users made from llmgw's accounts for the Slurm cluster", + "description": "Every active account, with its login, UID and SSH keys; an account is given its login and UID the first time it is listed. The machines have them when slurm.users is \"llmgw\".", + "tags": [ + "slurm" + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/slurmsvc.User" + } + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/api/v1/slurm/users/{user}/ssh-keys": { + "put": { + "operationId": "putSlurmUsersUserSshKeys", + "summary": "Replace an account's SSH public keys", + "description": "{user} is an ID or e-mail. keys is OpenSSH public keys, one a line, without options; empty takes them all away. The machines have them within a minute.", + "tags": [ + "slurm" + ], + "parameters": [ + { + "name": "user", + "in": "path", + "required": true, + "description": "The person's ID or email.", + "schema": { + "type": "string" + } + }, + { + "$ref": "#/components/parameters/Actor" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SlurmKeysRequest" + } + } + } + }, + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/slurmsvc.User" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/summary": { "get": { "operationId": "getSummary", @@ -9401,6 +9484,15 @@ } } }, + "SlurmKeysRequest": { + "type": "object", + "description": "SlurmKeysRequest is a person's SSH public keys, one a line, as authorized_keys has them; empty takes them all away.", + "properties": { + "keys": { + "type": "string" + } + } + }, "StoreMember": { "type": "object", "description": "StoreMember is one member of a replicated store, as its Patroni reports it.", @@ -10785,6 +10877,24 @@ } } }, + "config.SharedStorage": { + "type": "object", + "description": "SharedStorage is where the Slurm cluster's shared files come from: the site's NFS export (NFS), an NFS export llmgw keeps on one of its nodes (Server), or, with neither, a file system the site mounts itself (Lustre, BeeGFS, GPFS), which llmgw only looks for.", + "properties": { + "Path": { + "type": "string", + "description": "Path is where every machine sees it, as /shared." + }, + "NFS": { + "type": "string", + "description": "NFS is the site's export, host:/path, mounted at Path on every machine of the Slurm cluster." + }, + "Server": { + "type": "string", + "description": "Server is the llmgw node, the controller or one of the nodes, that exports Path over NFS to the others: for a site with no storage of its own. One machine: not highly available, and no faster than its disk and network." + } + } + }, "control.Activity": { "type": "object", "description": "Activity is what a standby's router reports back: when it last sent each replica a request, and the on-demand models it was asked for. The leader evicts and loads by these, and without them a replica busy behind a standby looks idle to the one machine that decides.", @@ -12249,6 +12359,14 @@ "slurmdbd_running": { "type": "boolean" }, + "shared": { + "type": "string", + "description": "Shared is the shared path here: \"mounted\", \"exported\", \"missing\"." + }, + "users": { + "type": "integer", + "description": "Users is how many of llmgw's users the machine has." + }, "state": { "type": "string" }, @@ -12301,6 +12419,40 @@ "type": "string" }, "description": "Waiting are nodes left out of slurm.conf until they report their hardware." + }, + "shared": { + "$ref": "#/components/schemas/config.SharedStorage", + "description": "Shared is where the shared files are and come from; Users who keeps the users, and how many llmgw has." + }, + "users": { + "type": "string" + }, + "user_count": { + "type": "integer" + } + } + }, + "slurmsvc.User": { + "type": "object", + "description": "User is a Linux user made from an llmgw account.", + "properties": { + "login": { + "type": "string" + }, + "uid": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "email": { + "type": "string" + }, + "ssh_keys": { + "type": "array", + "items": { + "type": "string" + } } } }, diff --git a/docs/console.md b/docs/console.md index cdd69a0..6baaf06 100644 --- a/docs/console.md +++ b/docs/console.md @@ -215,7 +215,10 @@ replicas off and then lets Slurm run jobs there; **Take back from Slurm…** starts no new job there and gives it back to llmgw when its jobs have ended. Both change `slurm.lent` in the cluster's settings, as `llmgw ctl slurm lend` and `reclaim` do. Turning Slurm on and choosing its nodes is `llmgw ctl slurm -setup`, or the Slurm part of the first install. +setup`, or the Slurm part of the first install. Above the nodes it says where +the shared files come from and how many users llmgw made; the SHARED column +is each node's (mounted, exported, missing). `llmgw ctl slurm shared` and +`llmgw ctl slurm users` change them. **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. diff --git a/docs/install.md b/docs/install.md index c56c4dd..00a19bb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -195,6 +195,25 @@ On a cluster installed without it, `llmgw ctl slurm setup --controller NODE networks only, as llmgw's own do; a firewall of the site's own between the nodes must let them through. +**Slurm's shared files and users.** Jobs need the same files on every +machine: homes, datasets, output, checkpoints. With `--slurm` this machine +exports `/shared` over NFS to every Slurm machine and they mount it +(`slurm.shared`); NFS's port 2049 answers the cluster's networks only, the +export the Slurm machines alone. That is one machine, not highly available +and no faster than its disk and network: enough for a few nodes. A site with +storage of its own uses it instead — `--slurm-shared HOST:/EXPORT` at +install, or `llmgw ctl slurm shared --path /shared --nfs HOST:/EXPORT` +later; Lustre, BeeGFS or GPFS the site mounts itself, and `llmgw ctl slurm +shared --path /lustre/hpc` only has llmgw look that it is there. Every +active llmgw account is a Linux user on every Slurm machine +(`slurm.users: llmgw`): its login from its e-mail address, a UID from +200000 up that is never given to anyone else, its home under +`/shared/home`. People add their SSH public keys in the portal's +**Profile** and log in to this machine to run `sbatch`; an account suspended +or deleted loses its user, its files stay. A site whose users come from its +directory (sssd against Active Directory or LDAP) passes `--slurm-users +site`, or later `llmgw ctl slurm users off`. + **Secure by default.** What init writes is the hardened configuration, not a demonstration one: @@ -3167,7 +3186,7 @@ loaded at boot by `llmgw-firewall.service`): | Machine | Ports, for the cluster only | | --- | --- | | control plane | 7443 (agents), metrics when not on loopback, 8081 (its mirror), engines and 9100 when it is a node, 2379/2380 when it holds the store's third etcd vote | -| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`) | +| node | engine ports (`control.port_range`, 20000–20999), 9100; with a part in Slurm also 6817–6819 (slurmctld, slurmd, slurmdbd) and 60001–63000 (srun's `SrunPortRange`); exporting Slurm's shared files, 2049 (NFS) | | store member | 5432, 8008, 2379, 2380, 9100 | | mirror | 8081, 8082, 9100 | diff --git a/docs/reference.md b/docs/reference.md index b6a0de5..0c0c506 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -401,6 +401,9 @@ rather than repeated. | `action` | `string` | [Service](#service) | | `timeout` | `time.Duration` | [Service](#service) | | `stopped` | `bool` | [ServingSettings](#servingsettings) | +| `path` | `string` | [SharedStorage](#sharedstorage) | +| `nfs` | `string` | [SharedStorage](#sharedstorage) | +| `server` | `string` | [SharedStorage](#sharedstorage) | | `bin` | `string` | [SimConfig](#simconfig) | | `enabled` | `bool` | [SlurmSettings](#slurmsettings) | | `cluster_name` | `string` | [SlurmSettings](#slurmsettings) | @@ -409,6 +412,8 @@ rather than repeated. | `all_nodes` | `bool` | [SlurmSettings](#slurmsettings) | | `lent` | `[]string` | [SlurmSettings](#slurmsettings) | | `partition` | `string` | [SlurmSettings](#slurmsettings) | +| `shared` | `SharedStorage` | [SlurmSettings](#slurmsettings) | +| `users` | `string` | [SlurmSettings](#slurmsettings) | | `dir` | `string` | [StrataConfig](#strataconfig) | | `python` | `string` | [StrataConfig](#strataconfig) | | `engine` | `string` | [StrataConfig](#strataconfig) | @@ -3488,6 +3493,30 @@ running and every model's definition as it is: llmgw stop sets it, llmgw start clears it, and the models come back as their catalog entries ask. In force on every control plane within seconds. +## SharedStorage + +### `path` + +`string` + +Path is where every machine sees it, as /shared. + +### `nfs` + +`string` + +NFS is the site's export, host:/path, mounted at Path on every +machine of the Slurm cluster. + +### `server` + +`string` + +Server is the llmgw node, the controller or one of the nodes, that +exports Path over NFS to the others: for a site with no storage of +its own. One machine: not highly available, and no faster than its +disk and network. + ## SimConfig ### `bin` @@ -3548,6 +3577,27 @@ given back to llmgw when its last job ends. Partition is the one partition every node is in. Empty is "gpu". +### `shared` + +`SharedStorage` + +Shared is the file system every machine of the Slurm cluster sees at +the same path: home directories, datasets, the jobs' output and +checkpoints. Empty path: none, and a batch job's output stays on the +node it ran on. + +Takes a [SharedStorage](#sharedstorage) block. + +### `users` + +`string` + +Users is who keeps the Linux users that run jobs, the same on every +machine of the Slurm cluster: "llmgw" makes one for each active llmgw +account (its login from the e-mail address, a UID of llmgw's from +200000 up, its SSH keys from the portal); empty leaves users to the +site (a directory through sssd, or by hand). + ## StrataConfig ### `dir` diff --git a/internal/agent/firewall.go b/internal/agent/firewall.go index bc9193b..fc5faf4 100644 --- a/internal/agent/firewall.go +++ b/internal/agent/firewall.go @@ -20,6 +20,10 @@ var slurmPorts = []firewall.Port{ {From: slurm.SrunPortsFrom, To: slurm.SrunPortsTo, What: "slurm: srun, for the steps it starts"}, } +// nfsPort is NFSv4's, on the machine that exports the Slurm cluster's +// shared files. +var nfsPort = firewall.Port{From: slurm.NFSPort, To: slurm.NFSPort, What: "nfs: the Slurm cluster's shared files"} + // applyFirewall keeps llmgw's firewall on this node as the cluster's // settings say: the engines' ports and the node exporter's for the networks // the node reaches its control planes on, and those named, or no table at @@ -69,6 +73,9 @@ func (a *Agent) applyFirewall(m *llmgwv1.Firewall) { if a.slurm != nil && a.slurm.HasPart() { ports = append(ports, slurmPorts...) } + if a.slurm != nil && a.slurm.Serves() { + ports = append(ports, nfsPort) + } changed, err := firewall.Install(ctx, r, firewall.Spec{Ports: ports, Allow: nets}) switch { case err != nil: @@ -79,11 +86,12 @@ func (a *Agent) applyFirewall(m *llmgwv1.Firewall) { } // slurmMessage is the leader's word on this node's part in Slurm. A node -// that gets one has the firewall applied again, with Slurm's ports. +// that gets one, or starts or stops exporting the shared files, has the +// firewall applied again, with Slurm's ports and NFS's. func (a *Agent) slurmMessage(m *llmgwv1.Slurm) { - had := a.slurm.HasPart() + had, served := a.slurm.HasPart(), a.slurm.Serves() a.slurm.Apply(m) - if had || !a.slurm.HasPart() { + if (had || !a.slurm.HasPart()) && served == a.slurm.Serves() { return } a.fwMu.Lock() diff --git a/internal/agent/slurmnode/shared.go b/internal/agent/slurmnode/shared.go new file mode 100644 index 0000000..0c84d89 --- /dev/null +++ b/internal/agent/slurmnode/shared.go @@ -0,0 +1,150 @@ +package slurmnode + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// What a machine says of the shared path. +const ( + SharedMounted = "mounted" + SharedExported = "exported" + SharedMissing = "missing" +) + +// shared keeps the shared path as want says: exported from here, mounted +// from an export, or looked for where the site mounts it. Answers how it +// is. +func (k *Keeper) shared(ctx context.Context, want *llmgwv1.Slurm) (string, error) { + sh := want.GetShared() + path := sh.GetPath() + k.unshare(ctx, path, sh.GetServe(), sh.GetNfs()) + if path == "" { + return "", nil + } + if !strings.HasPrefix(path, "/") || strings.Contains(path, "..") || strings.ContainsAny(path, " \n\t\"'\\$`") { + return SharedMissing, fmt.Errorf("the leader sent a path %q", path) + } + switch { + case sh.GetServe(): + if err := k.host.MkdirAll(path, 0o755, ""); err != nil { + return SharedMissing, err + } + changed, err := k.write(slurm.ExportsPath, []byte(slurm.Exports(path, sh.GetClients())), 0o644, "") + if err != nil { + return SharedMissing, err + } + if !k.active(ctx, "nfs-server") { + k.log.Info("NFS starts on this machine, exporting the Slurm cluster's shared files", "path", path) + if err := k.systemctl(ctx, "enable", "--now", "nfs-server"); err != nil { + return SharedMissing, err + } + changed = true + } + if changed { + if out, err := k.host.Run(ctx, nil, "exportfs", "-ra"); err != nil { + return SharedMissing, fmt.Errorf("exportfs -ra: %v: %s", err, lastLine(out)) + } + } + return SharedExported, nil + case sh.GetNfs() != "": + nfs := sh.GetNfs() + if strings.ContainsAny(nfs, " \n\t\"'\\$`") || !strings.Contains(nfs, ":/") { + return SharedMissing, fmt.Errorf("the leader sent an export %q", nfs) + } + if err := k.host.MkdirAll(path, 0o755, ""); err != nil { + return SharedMissing, err + } + unit := slurm.MountUnitName(path) + changed, err := k.write(filepath.Join(slurm.MountUnitDir, unit), []byte(slurm.MountUnit(nfs, path)), 0o644, "") + if err != nil { + return SharedMissing, err + } + if changed { + k.log.Info("the Slurm cluster's shared files are mounted here", "from", nfs, "at", path) + if err := k.systemctl(ctx, "daemon-reload"); err != nil { + return SharedMissing, err + } + if err := k.systemctl(ctx, "enable", unit); err != nil { + return SharedMissing, err + } + // From another export: a restart mounts the new one, and fails + // while a job still has files open on the old. + if err := k.systemctl(ctx, "restart", unit); err != nil { + return SharedMissing, err + } + } else if !k.active(ctx, unit) { + if err := k.systemctl(ctx, "start", unit); err != nil { + return SharedMissing, err + } + } + } + if _, err := k.host.Run(ctx, nil, "mountpoint", "-q", path); err != nil { + if sh.GetNfs() == "" { + return SharedMissing, fmt.Errorf("%s is not mounted here: the site mounts it (slurm.shared names no export)", path) + } + return SharedMissing, fmt.Errorf("%s is not mounted from %s", path, sh.GetNfs()) + } + return SharedMounted, nil +} + +// unshare takes away the export or the mount llmgw kept here that want no +// longer has: path is where the files are wanted now, serve and nfs how. +func (k *Keeper) unshare(ctx context.Context, path string, serve bool, nfs string) { + if _, err := k.host.ReadFile(slurm.ExportsPath); err == nil && (!serve || path == "") { + k.log.Info("this machine no longer exports the Slurm cluster's shared files") + _ = k.host.Remove(slurm.ExportsPath) + _, _ = k.host.Run(ctx, nil, "exportfs", "-ra") + } + out, err := k.host.Run(ctx, nil, "systemctl", "list-units", "--all", "--plain", "--no-legend", "--type=mount") + if err != nil { + return + } + keep := "" + if path != "" && nfs != "" && !serve { + keep = slurm.MountUnitName(path) + } + for _, l := range strings.Split(string(out), "\n") { + f := strings.Fields(l) + if len(f) == 0 || f[0] == keep { + continue + } + file := filepath.Join(slurm.MountUnitDir, f[0]) + b, err := k.host.ReadFile(file) + if err != nil || !strings.HasPrefix(string(b), "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files.") { + continue + } + k.log.Info("the Slurm cluster's old shared mount is taken away", "unit", f[0]) + if err := k.systemctl(ctx, "disable", "--now", f[0]); err != nil { + k.log.Warn("could not unmount the old shared files", "unit", f[0], "err", err) + continue + } + _ = k.host.Remove(file) + _ = k.systemctl(ctx, "daemon-reload") + } +} + +// sharedPackages are what the shared path needs here, by the command that +// shows each is installed. +func sharedPackages(want *llmgwv1.Slurm, need map[string][2]string) { + switch sh := want.GetShared(); { + case sh.GetPath() == "": + case sh.GetServe(): + need["exportfs"] = [2]string{"nfs-kernel-server", "nfs-utils"} + case sh.GetNfs() != "": + need["mount.nfs"] = [2]string{"nfs-common", "nfs-utils"} + } +} + +// Serves is whether this machine exports the shared files, so NFS's port +// is kept for the cluster's networks. +func (k *Keeper) Serves() bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.want.GetShared().GetServe() && k.want.GetShared().GetPath() != "" +} diff --git a/internal/agent/slurmnode/shared_test.go b/internal/agent/slurmnode/shared_test.go new file mode 100644 index 0000000..3fd286b --- /dev/null +++ b/internal/agent/slurmnode/shared_test.go @@ -0,0 +1,124 @@ +package slurmnode + +import ( + "context" + "slices" + "strings" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +func ayse() *llmgwv1.SlurmUser { + return &llmgwv1.SlurmUser{Login: "ayse", Uid: 200000, Name: "Ayşe Y.", Home: "/shared/home/ayse", + SshKeys: []string{"ssh-ed25519 AAAA ayse@laptop"}} +} + +func TestTheServerExportsAndMakesUsersWithHomes(t *testing.T) { + h := newFake() + h.active["ssh"] = true + inst := &fakeInst{host: h} + k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) + m := nodeMsg(false) + m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Serve: true, Clients: []string{"10.0.0.22", "10.0.0.21"}} + m.ManageUsers, m.MakeHomes, m.Users = true, true, []*llmgwv1.SlurmUser{ayse()} + h.fail["test -d /shared/home/ayse"] = true + st := k.Reconcile(context.Background(), m) + if st.GetError() != "" { + t.Fatal(st.GetError()) + } + if st.GetShared() != SharedExported || st.GetUsers() != 1 { + t.Fatalf("status %v", st) + } + if !slices.Contains(inst.got["deb"], "nfs-kernel-server") { + t.Errorf("packages %v", inst.got) + } + if e := string(h.files[slurm.ExportsPath]); !strings.Contains(e, `"/shared" 10.0.0.21(`) { + t.Errorf("exports:\n%s", e) + } + for _, want := range []string{"systemctl enable --now nfs-server", "exportfs -ra", "groupadd -g 200000 ayse", + "useradd -u 200000 -g 200000 -M -d /shared/home/ayse -s /bin/bash -c Ayşe Y. ayse", + "install -d -m 0700 -o 200000 -g 200000 /shared/home/ayse", "sshd -t", "systemctl reload ssh"} { + if !slices.Contains(h.ran, want) { + t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n")) + } + } + if string(h.files[slurm.AuthorizedKeysDir+"/ayse"]) != "ssh-ed25519 AAAA ayse@laptop\n" || + string(h.files[k.usersMarker()]) != "ayse\n" || h.files[slurm.SSHDropIn] == nil { + t.Errorf("files: keys %q, marker %q", h.files[slurm.AuthorizedKeysDir+"/ayse"], h.files[k.usersMarker()]) + } + + // Again, made: nothing is run again. + h.outputs["getent passwd ayse"] = "ayse:x:200000:200000:Ayşe Y.:/shared/home/ayse:/bin/bash\n" + delete(h.fail, "test -d /shared/home/ayse") + h.ran = nil + k.Reconcile(context.Background(), m) + for _, c := range h.ran { + if strings.HasPrefix(c, "useradd") || strings.HasPrefix(c, "usermod") || strings.HasPrefix(c, "exportfs") || + strings.HasPrefix(c, "install") || c == "systemctl reload ssh" { + t.Errorf("ran %q again", c) + } + } + + // Gone from the accounts: taken away, its home left. + m.Users = nil + k.Reconcile(context.Background(), m) + if !slices.Contains(h.ran, "userdel ayse") || h.files[k.usersMarker()] != nil || h.files[slurm.SSHDropIn] != nil || + h.files[slurm.AuthorizedKeysDir+"/ayse"] != nil { + t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n")) + } + if slices.ContainsFunc(h.ran, func(c string) bool { return strings.HasPrefix(c, "rm") || strings.Contains(c, "userdel -r") }) { + t.Error("a home was removed") + } +} + +func TestAClientMountsAndLeavesOthersUsersAlone(t *testing.T) { + h := newFake() + inst := &fakeInst{host: h} + k := New(h, inst, "/var/lib/llmgw-agent/slurm", nil) + m := nodeMsg(false) + m.Shared = &llmgwv1.SlurmShared{Path: "/shared", Nfs: "10.0.0.11:/shared"} + bob := &llmgwv1.SlurmUser{Login: "bob", Uid: 200001, Home: "/shared/home/bob"} + m.ManageUsers, m.Users = true, []*llmgwv1.SlurmUser{ayse(), bob} + // The site's own ayse, with its own UID. + h.outputs["getent passwd ayse"] = "ayse:x:1001:1001::/home/ayse:/bin/bash\n" + st := k.Reconcile(context.Background(), m) + if st.GetShared() != SharedMounted || st.GetUsers() != 1 { + t.Fatalf("status %v", st) + } + if !strings.Contains(st.GetError(), "ayse is already a user here with UID 1001") { + t.Errorf("error %q", st.GetError()) + } + if u := string(h.files["/etc/systemd/system/shared.mount"]); !strings.Contains(u, "What=10.0.0.11:/shared") { + t.Fatalf("unit:\n%s", u) + } + for _, want := range []string{"systemctl enable shared.mount", "systemctl restart shared.mount", "mountpoint -q /shared", + "useradd -u 200001 -g 200001 -M -d /shared/home/bob -s /bin/bash -c bob"} { + if !slices.Contains(h.ran, want) { + t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n")) + } + } + if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "ayse") && !strings.HasPrefix(c, "getent") }) || + slices.ContainsFunc(h.ran, func(c string) bool { return strings.HasPrefix(c, "install -d") }) { + t.Errorf("touched the site's ayse, or made a home:\n%s", strings.Join(h.ran, "\n")) + } + if string(h.files[k.usersMarker()]) != "bob\n" { + t.Errorf("marker %q", h.files[k.usersMarker()]) + } + + // Not mounted: said. + h.fail["mountpoint -q /shared"] = true + if st := k.Reconcile(context.Background(), m); st.GetShared() != SharedMissing || + !strings.Contains(st.GetError(), "not mounted from 10.0.0.11:/shared") { + t.Errorf("status %v", st) + } + + // No shared path any more: the mount is taken away. + h.outputs["systemctl list-units --all --plain --no-legend --type=mount"] = "shared.mount loaded active mounted llmgw\n" + m.Shared = nil + k.Reconcile(context.Background(), m) + if !slices.Contains(h.ran, "systemctl disable --now shared.mount") || h.files["/etc/systemd/system/shared.mount"] != nil { + t.Errorf("mount kept:\n%s", strings.Join(h.ran, "\n")) + } +} diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index 8f15ae6..6c5176b 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -210,6 +210,31 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl if err := k.munge(ctx, want.GetMungeKey()); err != nil { return fail("munge", err) } + // The shared files, then the users whose homes are on them. Neither + // keeps Slurm from running: what goes wrong is said, and Slurm kept. + var shErr, usErr error + st.Shared, shErr = k.shared(ctx, want) + if shErr != nil { + k.log.Warn("keeping the Slurm cluster's shared files on this node", "err", shErr) + } + if shErr == nil || !want.GetMakeHomes() || want.GetShared().GetPath() == "" { + st.Users, usErr = k.users(ctx, want) + } else { + // Homes made now would be made beside the shared files, not on them. + usErr = errors.New("waiting for the shared files, where the homes are") + } + if usErr != nil { + k.log.Warn("keeping the Slurm users on this node", "err", usErr) + } + defer func() { + switch { + case st.Error != "": + case shErr != nil: + st.Error = "shared files: " + shErr.Error() + case usErr != nil: + st.Error = "users: " + usErr.Error() + } + }() if want.GetController() { // Accounting first: slurmctld reports to it from its first start. // Without it slurmctld still runs jobs, a pass of backfill late @@ -252,6 +277,7 @@ func (k *Keeper) ensurePackages(ctx context.Context, want *llmgwv1.Slurm) error if want.GetNode() { need["slurmd"] = [2]string{"slurmd", "slurm-slurmd"} } + sharedPackages(want, need) pkgs := map[string][]string{} for bin, p := range need { if k.installed[bin] { @@ -518,6 +544,10 @@ func (k *Keeper) stop(ctx context.Context) { } _ = k.host.Remove(DropIn) _ = k.systemctl(ctx, "daemon-reload") + k.unshare(ctx, "", false, "") + if err := k.removeUsers(ctx, k.madeUsers(), nil); err != nil { + k.log.Warn("could not take away every Slurm user llmgw made", "err", err) + } _ = k.host.Remove(k.marker) k.files, k.acctDone, k.clusterAdded = "", "", "" } diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index 2f70c0d..c4b26d8 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -60,7 +60,7 @@ func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...str case "disable": f.active[u] = false } - return nil, nil + return []byte(f.outputs[c]), nil case strings.HasPrefix(c, "scontrol show node"): return []byte("NodeName=cuda01 Arch=x86_64 State=" + f.state + " Partitions=gpu Reason=" + f.reason + " [root@now]\n"), nil case strings.HasPrefix(c, "scontrol update"): @@ -119,7 +119,7 @@ type fakeInst struct { func (i *fakeInst) InstallPackages(_ context.Context, by map[string][]string) error { i.got = by - for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { + for _, b := range []string{"munged", "scontrol", "slurmd", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} { i.host.bins[b] = true } return nil @@ -337,7 +337,7 @@ func TestBadNames(t *testing.T) { func TestAMariaDBThatWasThereKeepsItsSettings(t *testing.T) { h := newFake() - for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb"} { + for _, b := range []string{"munged", "scontrol", "slurmctld", "slurmdbd", "mariadbd", "mariadb", "exportfs", "mount.nfs"} { h.bins[b] = true } k := New(h, &fakeInst{host: h}, "/m", nil) diff --git a/internal/agent/slurmnode/users.go b/internal/agent/slurmnode/users.go new file mode 100644 index 0000000..91fcd18 --- /dev/null +++ b/internal/agent/slurmnode/users.go @@ -0,0 +1,238 @@ +package slurmnode + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "slices" + "strconv" + "strings" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// usersMarker lists, a line each, the logins llmgw made here: the ones it +// may change and take away. A user that was here before is never touched. +func (k *Keeper) usersMarker() string { return k.marker + "-users" } + +// users keeps the Linux users want has, the same login and UID as on every +// other machine, their keys where sshd reads them, and their homes when +// this machine makes them; and takes away those llmgw made that are gone. +// Answers how many it has. A user that cannot be made is said, the others +// are made anyway. +func (k *Keeper) users(ctx context.Context, want *llmgwv1.Slurm) (int32, error) { + made := k.madeUsers() + if !want.GetManageUsers() { + return 0, k.removeUsers(ctx, made, nil) + } + var errs []error + var keep []string + n := int32(0) + for _, u := range want.GetUsers() { + err := k.user(ctx, u, slices.Contains(made, u.GetLogin()), want.GetMakeHomes()) + if errors.Is(err, errNotMade) { + errs = append(errs, err) + continue + } + keep = append(keep, u.GetLogin()) + if err != nil { + errs = append(errs, err) + continue + } + n++ + } + if err := k.keepMade(keep); err != nil { + return n, err + } + if err := k.removeUsers(ctx, made, keep); err != nil { + errs = append(errs, err) + } + if err := k.sshConf(ctx, len(keep) > 0); err != nil { + errs = append(errs, err) + } + return n, errors.Join(errs...) +} + +// errNotMade is a user llmgw could not make here, nor took for its own. +var errNotMade = errors.New("not made") + +type notMade struct{ msg string } + +func (e notMade) Error() string { return e.msg } +func (e notMade) Is(target error) bool { return target == errNotMade } + +// user keeps one user; ours is whether llmgw made it here before. +func (k *Keeper) user(ctx context.Context, u *llmgwv1.SlurmUser, ours, makeHome bool) error { + login, uid, home := u.GetLogin(), u.GetUid(), u.GetHome() + if !slurm.ValidLogin(login) || uid < slurm.MinUID || !filepath.IsAbs(home) || strings.Contains(home, "..") { + return notMade{fmt.Sprintf("the leader sent a user %q, UID %d, home %q", login, uid, home)} + } + id := strconv.FormatUint(uint64(uid), 10) + gecos := strings.Map(func(r rune) rune { + if r == ':' || r == ',' || r == '\n' || r < ' ' { + return -1 + } + return r + }, u.GetName()) + byName, okName := k.passwd(ctx, login) + byID, okID := k.passwd(ctx, id) + switch { + case okName && byName[2] != id: + return notMade{fmt.Sprintf("%s is already a user here with UID %s, not llmgw's %s", login, byName[2], id)} + case !okName && okID: + return notMade{fmt.Sprintf("UID %s is already %s's here, so %s is not made", id, byID[0], login)} + case !okName: + if _, ok := k.group(ctx, login); !ok { + if out, err := k.host.Run(ctx, nil, "groupadd", "-g", id, login); err != nil { + return notMade{fmt.Sprintf("groupadd %s: %v: %s", login, err, lastLine(out))} + } + } + k.log.Info("a Slurm user is made here, as llmgw's accounts say", "login", login, "uid", uid) + if out, err := k.host.Run(ctx, nil, "useradd", "-u", id, "-g", id, "-M", "-d", home, "-s", "/bin/bash", + "-c", gecos, login); err != nil { + return notMade{fmt.Sprintf("useradd %s: %v: %s", login, err, lastLine(out))} + } + case ours && (byName[4] != gecos || byName[5] != home): + if out, err := k.host.Run(ctx, nil, "usermod", "-c", gecos, "-d", home, login); err != nil { + return fmt.Errorf("usermod %s: %v: %s", login, err, lastLine(out)) + } + } + keys := strings.Join(u.GetSshKeys(), "\n") + file := filepath.Join(slurm.AuthorizedKeysDir, login) + if keys == "" { + _ = k.host.Remove(file) + } else if _, err := k.write(file, []byte(keys+"\n"), 0o644, ""); err != nil { + return err + } + if makeHome { + return k.home(ctx, login, id, home) + } + return nil +} + +// home makes a user's home directory, from /etc/skel, when it is not there. +func (k *Keeper) home(ctx context.Context, login, id, home string) error { + if _, err := k.host.Run(ctx, nil, "test", "-d", home); err == nil { + return nil + } + if err := k.host.MkdirAll(filepath.Dir(home), 0o755, ""); err != nil { + return err + } + k.log.Info("a Slurm user's home is made", "login", login, "home", home) + for _, c := range [][]string{ + {"install", "-d", "-m", "0700", "-o", id, "-g", id, home}, + {"cp", "-rT", "/etc/skel", home}, + {"chown", "-R", id + ":" + id, home}, + } { + if out, err := k.host.Run(ctx, nil, c[0], c[1:]...); err != nil { + return fmt.Errorf("making %s's home: %s: %v: %s", login, c[0], err, lastLine(out)) + } + } + return nil +} + +// removeUsers takes away the users llmgw made here that are not in keep. +// Their homes and files stay; their UID is never given to another. +func (k *Keeper) removeUsers(ctx context.Context, made, keep []string) error { + var errs []error + left := slices.Clone(keep) + for _, login := range made { + if slices.Contains(keep, login) { + continue + } + k.log.Info("a Slurm user llmgw made is taken away; its home stays", "login", login) + _ = k.host.Remove(filepath.Join(slurm.AuthorizedKeysDir, login)) + if _, ok := k.passwd(ctx, login); ok { + if out, err := k.host.Run(ctx, nil, "userdel", login); err != nil { + // Still logged in, or running something: tried again. + errs = append(errs, fmt.Errorf("userdel %s: %v: %s", login, err, lastLine(out))) + left = append(left, login) + continue + } + } + } + if !slices.Equal(left, made) { + if err := k.keepMade(left); err != nil { + errs = append(errs, err) + } + } + if len(left) == 0 { + if err := k.sshConf(ctx, false); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (k *Keeper) madeUsers() []string { + b, err := k.host.ReadFile(k.usersMarker()) + if err != nil { + return nil + } + return strings.Fields(string(b)) +} + +func (k *Keeper) keepMade(logins []string) error { + s := slices.Clone(logins) + slices.Sort(s) + s = slices.Compact(s) + if len(s) == 0 { + return k.host.Remove(k.usersMarker()) + } + _, err := k.write(k.usersMarker(), []byte(strings.Join(s, "\n")+"\n"), 0o644, "") + return err +} + +// sshConf has sshd read the users' keys from AuthorizedKeysDir, or no +// longer. +func (k *Keeper) sshConf(ctx context.Context, on bool) error { + if !on { + if _, err := k.host.ReadFile(slurm.SSHDropIn); err != nil { + return nil + } + _ = k.host.Remove(slurm.SSHDropIn) + return k.reloadSSH(ctx) + } + changed, err := k.write(slurm.SSHDropIn, []byte(slurm.SSHConf), 0o644, "") + if err != nil || !changed { + return err + } + if out, err := k.host.Run(ctx, nil, "sshd", "-t"); err != nil { + _ = k.host.Remove(slurm.SSHDropIn) + return fmt.Errorf("sshd refuses llmgw's setting for the users' keys, so it is not kept: %v: %s", err, lastLine(out)) + } + k.log.Info("sshd reads the Slurm users' keys from llmgw too", "dir", slurm.AuthorizedKeysDir) + return k.reloadSSH(ctx) +} + +// reloadSSH has sshd read its settings again, under Debian's name for it +// or Red Hat's; one that is not running reads them when it starts. +func (k *Keeper) reloadSSH(ctx context.Context) error { + for _, u := range []string{"ssh", "sshd"} { + if k.active(ctx, u) { + return k.systemctl(ctx, "reload", u) + } + } + return nil +} + +// passwd is getent's line for a user, by login or UID, in its fields. +func (k *Keeper) passwd(ctx context.Context, key string) ([]string, bool) { + out, err := k.host.Run(ctx, nil, "getent", "passwd", key) + f := strings.Split(strings.TrimSpace(string(out)), ":") + if err != nil || len(f) < 7 { + return nil, false + } + return f, true +} + +func (k *Keeper) group(ctx context.Context, key string) ([]string, bool) { + out, err := k.host.Run(ctx, nil, "getent", "group", key) + f := strings.Split(strings.TrimSpace(string(out)), ":") + if err != nil || len(f) < 3 { + return nil, false + } + return f, true +} diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go index 1ba5abe..e03fdcc 100644 --- a/internal/cli/ctl_slurm.go +++ b/internal/cli/ctl_slurm.go @@ -3,12 +3,16 @@ package cli import ( "encoding/json" "fmt" + "io" + "net/url" + "os" "slices" "strings" "text/tabwriter" "github.com/spf13/cobra" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" ) @@ -27,7 +31,148 @@ func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client), newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true), - newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false)) + newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false), + newSlurmSharedCmd(client), newSlurmUsersCmd(client, asJSON)) + return cmd +} + +func newSlurmSharedCmd(client func() *ctlClient) *cobra.Command { + var path, server, nfs, note string + var off bool + c := &cobra.Command{ + Use: "shared --path DIR (--server NODE | --nfs HOST:/EXPORT) | --off", + Short: "Files every machine of the Slurm cluster sees at one path", + Long: "Home directories, datasets, the jobs' output and checkpoints, at the same path on the controller\n" + + "and every node. Three ways:\n" + + " --nfs HOST:/EXPORT the site's own NFS export, mounted at --path on every machine;\n" + + " --server NODE an llmgw node (the controller or a node) exports --path over NFS to the\n" + + " others, for a site with no storage of its own: one machine, not highly\n" + + " available, no faster than its disk and network;\n" + + " neither the site mounts --path itself (Lustre, BeeGFS, GPFS); llmgw only looks\n" + + " that it is there.\n" + + "The export answers the Slurm machines alone, NFS's port the cluster's networks. A machine\n" + + "that gives up its mount does so when no job holds files on it.", + Example: " llmgw ctl slurm shared --path /shared --server node06\n" + + " llmgw ctl slurm shared --path /shared --nfs filer.site:/export/hpc\n" + + " llmgw ctl slurm shared --path /lustre/hpc\n" + + " llmgw ctl slurm shared --off", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if off == (path != "") { + return fmt.Errorf("give --path, or --off") + } + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if off { + if doc.Node([]string{"slurm", "shared"}) != nil { + doc.Unset([]string{"slurm", "shared"}) + } + return nil + } + v := map[string]string{"path": path} + if server != "" { + v["server"] = server + } + if nfs != "" { + v["nfs"] = nfs + } + b, _ := json.Marshal(v) + return doc.SetYAML([]string{"slurm", "shared"}, string(b)) + }) + }, + } + c.Flags().StringVar(&path, "path", "", "where every machine sees the files, as /shared") + c.Flags().StringVar(&server, "server", "", "the llmgw node that exports them over NFS") + c.Flags().StringVar(&nfs, "nfs", "", "the site's NFS export, HOST:/EXPORT") + c.Flags().BoolVar(&off, "off", false, "no shared files; mounts and exports llmgw made are taken away") + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c +} + +func newSlurmUsersCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + cmd := &cobra.Command{ + Use: "users", + Short: "The Linux users that run jobs, made from llmgw's accounts", + Long: "With slurm.users \"llmgw\" every active account is a Linux user on every machine of the Slurm\n" + + "cluster: its login from its e-mail address, a UID of llmgw's from 200000 up (never given to\n" + + "another, even when the account goes), its home under the shared path. It logs in over SSH\n" + + "with the keys given in the portal (Profile) or here. An account suspended or deleted loses\n" + + "its user; its home and files stay. Off, users are the site's: from its directory (sssd), or\n" + + "made by hand.", + } + list := &cobra.Command{ + Use: "list", + Short: "Every user, with its login, UID and how many keys it has", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var us []slurmsvc.User + if err := client().get(cmd.Context(), "/api/v1/slurm/users", &us); err != nil { + return err + } + w := cmd.OutOrStdout() + if *asJSON { + return json.NewEncoder(w).Encode(us) + } + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "LOGIN\tUID\tACCOUNT\tSSH KEYS") + for _, u := range us { + fmt.Fprintf(tw, "%s\t%d\t%s\t%d\n", u.Login, u.UID, u.Email, len(u.SSHKeys)) + } + return tw.Flush() + }, + } + var note string + toggle := func(use, short, value string) *cobra.Command { + c := &cobra.Command{ + Use: use, + Short: short, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if value == "" { + if doc.Node([]string{"slurm", "users"}) != nil { + doc.Unset([]string{"slurm", "users"}) + } + return nil + } + return doc.SetYAML([]string{"slurm", "users"}, yamlString(value)) + }) + }, + } + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c + } + keys := &cobra.Command{ + Use: "keys ACCOUNT FILE", + Short: "Replace an account's SSH public keys, from a file (- for standard input)", + Long: "FILE holds OpenSSH public keys, one a line, as ~/.ssh/id_ed25519.pub; an empty one takes them all away.", + Example: " llmgw ctl slurm users keys ayse@site.example ayse.pub\n" + + " ssh-add -L | llmgw ctl slurm users keys ayse@site.example -", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + var b []byte + var err error + if args[1] == "-" { + b, err = io.ReadAll(io.LimitReader(cmd.InOrStdin(), 64<<10)) + } else { + b, err = os.ReadFile(args[1]) + } + if err != nil { + return err + } + body, _ := json.Marshal(map[string]string{"keys": string(b)}) + var u slurmsvc.User + if err := client().put(cmd.Context(), "/api/v1/slurm/users/"+url.PathEscape(args[0])+"/ssh-keys", body, &u); err != nil { + return err + } + fmt.Fprintf(cmd.OutOrStdout(), "%s (UID %d) has %d SSH key(s); the machines have them within a minute\n", + u.Login, u.UID, len(u.SSHKeys)) + return nil + }, + } + cmd.AddCommand(list, + toggle("on", "Make a Linux user for every active account, on every machine of the Slurm cluster", "llmgw"), + toggle("off", "Leave users to the site; those llmgw made are taken away, their homes left", ""), + keys) return cmd } @@ -62,14 +207,19 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { default: fmt.Fprintln(w, ": slurmctld not running") } + fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared)) + if st.Users != "" { + fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount) + } tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tNOTE") + fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tSHARED\tNOTE") for _, n := range st.Nodes { - state, jobs, slurmd, note := "-", "-", "-", n.Describe() + state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe() if r := n.Status; r != nil { state = or(r.State, "-") jobs = fmt.Sprint(r.Jobs) slurmd = map[bool]string{true: "running", false: "stopped"}[r.SlurmdRunning] + shared = or(r.Shared, "-") if r.Error != "" { note = r.Error } @@ -77,7 +227,7 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if !n.Connected { note = "agent not connected; " + note } - fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, note) + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, shared, note) } if err := tw.Flush(); err != nil { return err @@ -172,6 +322,19 @@ func newSlurmOffCmd(client func() *ctlClient) *cobra.Command { return c } +// sharedLine says where the shared files are and come from. +func sharedLine(sh config.SharedStorage) string { + switch { + case sh.Path == "": + return "none (`llmgw ctl slurm shared` adds them; a batch job's output stays on its node)" + case sh.Server != "": + return fmt.Sprintf("%s, exported over NFS by %s", sh.Path, sh.Server) + case sh.NFS != "": + return fmt.Sprintf("%s, mounted from %s", sh.Path, sh.NFS) + } + return sh.Path + ", mounted by the site" +} + // newSlurmListCmd adds nodes to slurm.lent, or takes them out. func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cobra.Command { var note string diff --git a/internal/cli/server.go b/internal/cli/server.go index 3e3ca58..7ce7d4d 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -256,6 +256,7 @@ func newServerCmd() *cobra.Command { // The settings in force, for what reads them at each use. var current atomic.Pointer[config.Server] current.Store(&cfg) + rt.SetSlurm(func() config.SlurmSettings { return current.Load().Slurm }) var agentSvc *agentsvc.Service var agentHandler http.Handler var agentPath string @@ -452,6 +453,16 @@ func newServerCmd() *cobra.Command { }, log) // What slurm.conf held, for the leader that comes next. slurmSvc.SetMemory(slurmsvc.StoreMemory(st)) + // slurm.users "llmgw": a Linux user for each active account. + slurmSvc.SetUsers(func(ctx context.Context) ([]slurmsvc.User, error) { + us, err := st.SlurmUsers(ctx) + out := make([]slurmsvc.User, 0, len(us)) + for _, u := range us { + out = append(out, adminapi.SlurmUserView(u)) + } + return out, err + }) + api.SetSlurmUsers(st) agentSvc.SetSlurm(slurmSvc) go slurmSvc.Run(ctx) api.SetSlurm(slurmSvc) diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 22d2f90..4a52eb1 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -127,6 +127,10 @@ type serverInit struct { // slurm runs Slurm beside llmgw: slurmctld on this machine's node, // slurmd on every node, each llmgw's until it is lent. slurm bool + // slurmShared is where Slurm's shared files come from: "here" (this + // machine exports /shared), the site's export as HOST:/EXPORT, or + // "none". slurmUsers is "llmgw" (users from the accounts) or "site". + slurmShared, slurmUsers string // evictAfterIdle and minLifetime are on_demand's; zero leaves its own. evictAfterIdle, minLifetime time.Duration // standbys makes each machine of the replicated store a control plane @@ -263,6 +267,10 @@ func newServerInitCmd() *cobra.Command { "pictures and 3D: the nodes install ComfyUI, and a picture model and a 3D model are added once a card fits one") f.BoolVar(&in.slurm, "slurm", false, "Slurm for training and batch jobs: slurmctld and slurmdbd (with a MariaDB) on this machine, slurmd on every node, each llmgw's until 'llmgw ctl slurm lend'") + f.StringVar(&in.slurmShared, "slurm-shared", "here", + "with --slurm, the jobs' shared files at /shared: here (this machine exports them over NFS), the site's HOST:/EXPORT, or none") + f.StringVar(&in.slurmUsers, "slurm-users", config.UsersLlmgw, + "with --slurm, who makes the Linux users that run jobs: llmgw (one for each account, SSH keys in the portal) or site") f.DurationVar(&in.evictAfterIdle, "evict-after-idle", 0, "how long a model loaded on demand goes unasked-for before another may have its card (0: 2m)") f.DurationVar(&in.minLifetime, "min-lifetime", 0, diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index c28fc9f..cb4c4f5 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -144,6 +144,13 @@ func (in *serverInit) checkChoices() error { if err := (config.SlurmSettings{Enabled: true, Controller: localNodeID()}).Check(); err != nil { return fmt.Errorf("slurm: this machine's hostname: %w", err) } + set := config.SlurmSettings{Enabled: true, Controller: localNodeID(), Shared: slurmShared(in.slurmShared)} + if err := set.Check(); err != nil { + return fmt.Errorf("--slurm-shared %q: here, none, or the site's HOST:/EXPORT: %w", in.slurmShared, err) + } + if u := in.slurmUsers; u != "" && u != config.UsersLlmgw && u != "site" { + return fmt.Errorf("--slurm-users %q: llmgw or site", u) + } } if in.vip != "" { ip, _, err := net.ParseCIDR(in.vip) @@ -329,7 +336,7 @@ func (in *serverInit) tuneConfig(body string) string { "engines:\n comfyui: "+provision.ComfyUIVersion()+"\npresets:\n auto: true\n") } if in.slurm { - extra = append(extra, slurmSection(localNodeID())) + extra = append(extra, slurmSection(localNodeID(), slurmShared(in.slurmShared), in.slurmUsers != "site")) } if in.evictAfterIdle > 0 || in.minLifetime > 0 { b := "# How models that load when asked for take turns on a card, chosen at\n# install.\non_demand:\n" @@ -347,15 +354,43 @@ func (in *serverInit) tuneConfig(body string) string { return body } +// slurmShared is --slurm-shared as settings: "here" is /shared exported +// by this machine's node, "none" nothing, anything else the site's export. +func slurmShared(v string) config.SharedStorage { + switch v { + case "", "none": + return config.SharedStorage{} + case "here": + return config.SharedStorage{Path: "/shared", Server: localNodeID()} + } + return config.SharedStorage{Path: "/shared", NFS: v} +} + // slurmSection is server.yaml's slurm block as an install chooses it: -// slurmctld on this machine's node, slurmd on every node. -func slurmSection(controller string) string { - return "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" + +// slurmctld on this machine's node, slurmd on every node, the shared files +// and the users as chosen. +func slurmSection(controller string, shared config.SharedStorage, users bool) string { + b := "# Slurm beside llmgw, chosen at install: slurmctld, and slurmdbd with a MariaDB\n" + "# for the jobs' records, on this machine; slurmd on\n" + "# every node, those that join later too. Each node is llmgw's, drained in\n" + "# Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm; 'llmgw ctl slurm\n" + "# reclaim NODE' takes it back once its jobs end.\n" + "slurm:\n enabled: true\n controller: " + yamlString(controller) + "\n all_nodes: true\n" + switch { + case shared.Server != "": + b += " # /shared on every Slurm machine, exported over NFS from this one\n" + + " # ('llmgw ctl slurm shared' points it at the site's storage instead).\n" + + " shared:\n path: " + yamlString(shared.Path) + "\n server: " + yamlString(shared.Server) + "\n" + case shared.NFS != "": + b += " # /shared on every Slurm machine, mounted from the site's export.\n" + + " shared:\n path: " + yamlString(shared.Path) + "\n nfs: " + yamlString(shared.NFS) + "\n" + } + if users { + b += " # A Linux user for each llmgw account, the same UID everywhere; SSH keys\n" + + " # in the portal's Profile. Empty leaves users to the site (sssd, a directory).\n" + + " users: llmgw\n" + } + return b } // localNodeID is the node id this machine's agent takes: its hostname. @@ -501,6 +536,12 @@ func (in *serverInit) initFlags() []string { } if in.slurm { add("--slurm") + if in.slurmShared != "" && in.slurmShared != "here" { + add("--slurm-shared", in.slurmShared) + } + if in.slurmUsers == "site" { + add("--slurm-users", "site") + } } if in.evictAfterIdle > 0 { add("--evict-after-idle", durationYAML(in.evictAfterIdle)) diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 4f0b154..53d36b6 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -17,7 +17,7 @@ import ( func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) { in := &serverInit{ language: "tr", signupOpen: true, defaultRPM: 120, defaultTPM: 50000, maxKeyTTL: 30 * 24 * time.Hour, - media: true, slurm: true, evictAfterIdle: time.Minute, minLifetime: 90 * time.Second, + media: true, slurm: true, slurmShared: "here", slurmUsers: "llmgw", evictAfterIdle: time.Minute, minLifetime: 90 * time.Second, judge: judgeSpec{Fail: "open", Categories: []string{"Violent", "PII"}, Replicas: 3, Budget: 3 * time.Second}, } j := in.judgeFor() @@ -43,6 +43,8 @@ func TestEveryChoiceIsInTheConfigurationWrittenAndItLoads(t *testing.T) { t.Errorf("pictures and 3D: presets %v comfyui %q", cfg.Presets.Auto, cfg.Engines["comfyui"]) case !cfg.Slurm.Enabled || !cfg.Slurm.AllNodes || cfg.Slurm.Controller != localNodeID(): t.Errorf("slurm %+v", cfg.Slurm) + case cfg.Slurm.Shared.Path != "/shared" || cfg.Slurm.Shared.Server != localNodeID() || cfg.Slurm.Users != "llmgw": + t.Errorf("slurm's shared files and users %+v", cfg.Slurm) case cfg.OnDemand.EvictAfterIdle != time.Minute || cfg.OnDemand.MinLifetime != 90*time.Second: t.Errorf("on demand %+v", cfg.OnDemand) } diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index 0a49056..be6d153 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -292,9 +292,13 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta "cluster's PostgreSQL). Its port 6819, with Slurm's others, answers the cluster's networks only.\n\n" + "Each node stays llmgw's, drained in Slurm, until 'llmgw ctl slurm lend NODE' gives it to Slurm: " + "its models move off first. 'llmgw ctl slurm reclaim NODE' takes it back once its jobs end. Jobs " + - "are submitted on this machine with sbatch and srun.", + "are submitted on this machine with sbatch and srun.\n\n" + + "Every Slurm machine sees /shared, exported over NFS from this machine (homes, datasets, the jobs' " + + "output); 'llmgw ctl slurm shared' points it at the site's own storage instead. Each llmgw account " + + "is a Linux user there, the same UID on every machine, logging in with the SSH keys given in the " + + "portal; 'llmgw ctl slurm users off' leaves users to the site's directory.", off: "no Slurm; 'llmgw ctl slurm setup' adds it later", - summary: func() string { return "slurmctld here, slurmd on every node; none lent" }, + summary: func() string { return "slurmctld here, slurmd on every node, /shared from here; none lent" }, needs: func() string { if !f.node.on { return "slurmctld runs on this machine's node: turn the node on, or Slurm off" diff --git a/internal/cli/tui_slurm.go b/internal/cli/tui_slurm.go index 6cb845d..ba297b6 100644 --- a/internal/cli/tui_slurm.go +++ b/internal/cli/tui_slurm.go @@ -44,6 +44,16 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { } head += fmt.Sprintf("\n controller %s: %s", st.Controller, ctl) head += "\n " + sDim.Render(fmt.Sprintf("cluster %s, partition %s; sbatch and srun on %s", st.Cluster, st.Partition, st.Controller)) + if st.Shared.Path == "" { + head += "\n shared files: " + sWarn.Render("none") + sDim.Render(" (llmgw ctl slurm shared; output stays on the job's node)") + } else { + head += "\n shared files: " + sharedLine(st.Shared) + } + if st.Users != "" { + head += "\n " + fmt.Sprintf("users: %d from llmgw's accounts", st.UserCount) + sDim.Render(", SSH keys in the portal's Profile") + } else { + head += "\n " + sDim.Render("users: the site's (llmgw ctl slurm users on makes them from llmgw's accounts)") + } head += "\n " + sDim.Render("A node is llmgw's until lent; lent, its replicas move off first.") if len(st.Waiting) > 0 { head += "\n " + sWarn.Render(fmt.Sprintf("not in slurm.conf until they report their hardware: %v", st.Waiting)) @@ -56,13 +66,20 @@ func (m consoleModel) slurmPanel(pw int) (string, string, string) { for _, n := range st.Nodes { rows = append(rows, slurmRow(n)) } - return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "NOTE"}, rows, m.row, pw, m.inPanel), keys + return "Slurm", head + table([]string{"NODE", "WHOSE", "IN SLURM", "JOBS", "SLURMD", "SHARED", "NOTE"}, rows, m.row, pw, m.inPanel), keys } func slurmRow(n slurmsvc.NodeStatus) []string { - state, jobs, slurmd, note := "-", "-", "-", n.Describe() + state, jobs, slurmd, shared, note := "-", "-", "-", "-", n.Describe() if r := n.Status; r != nil { state = or(r.State, "-") + switch r.Shared { + case "": + case "missing": + shared = sBad.Render(r.Shared) + default: + shared = sOK.Render(r.Shared) + } jobs = fmt.Sprint(r.Jobs) slurmd = sBad.Render("stopped") if r.SlurmdRunning { @@ -84,7 +101,7 @@ func slurmRow(n slurmsvc.NodeStatus) []string { case slurmsvc.Lending, slurmsvc.Reclaiming: whose = sWarn.Render(whose) } - return []string{n.Node, whose, state, jobs, slurmd, note} + return []string{n.Node, whose, state, jobs, slurmd, shared, note} } // slurmMenu is what can be done with one of Slurm's nodes. diff --git a/internal/cli/tui_slurm_test.go b/internal/cli/tui_slurm_test.go index 87393ce..2cc0500 100644 --- a/internal/cli/tui_slurm_test.go +++ b/internal/cli/tui_slurm_test.go @@ -11,6 +11,7 @@ import ( tea "github.com/charmbracelet/bubbletea" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" ) @@ -37,16 +38,17 @@ func TestTheConsoleLendsANodeToSlurmFromItsRow(t *testing.T) { m := newConsoleModel(srv.URL, "/nonexistent/server.yaml") m.loaded, m.menu, m.inPanel = true, cSlurm, true m.data.slurm = slurmsvc.Status{Enabled: true, Cluster: "llmgw", Controller: "node06", Partition: "gpu", - Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"}, + Ctld: &slurmsvc.Reported{SlurmctldRunning: true, SlurmdbdRunning: true, Version: "23.11.4"}, + Shared: config.SharedStorage{Path: "/shared", Server: "node06"}, Users: "llmgw", UserCount: 2, Nodes: []slurmsvc.NodeStatus{ {Node: "node05", Phase: slurmsvc.Lent, Connected: true, - Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true}}, + Status: &slurmsvc.Reported{State: "MIXED", Jobs: 2, SlurmdRunning: true, Shared: "mounted"}}, {Node: "cuda01", Phase: slurmsvc.Llmgw, Replicas: 3, Connected: true, Status: &slurmsvc.Reported{State: "IDLE+DRAIN", SlurmdRunning: true}}, }} var tm tea.Model = m view := tm.View() - for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { + for _, want := range []string{"slurmctld 23.11.4 and slurmdbd running", "/shared, exported over NFS by node06", "users: 2", "node05", "Slurm's", "MIXED", "cuda01", "llmgw's"} { if !strings.Contains(view, want) { t.Errorf("the Slurm view has no %q:\n%s", want, view) } diff --git a/internal/config/slurm.go b/internal/config/slurm.go index e1b946e..c2dfb7a 100644 --- a/internal/config/slurm.go +++ b/internal/config/slurm.go @@ -2,8 +2,10 @@ package config import ( "fmt" + "path" "regexp" "slices" + "strings" ) // SlurmSettings is a Slurm cluster run beside llmgw on some of its nodes @@ -38,6 +40,83 @@ type SlurmSettings struct { // Partition is the one partition every node is in. Empty is "gpu". Partition string `koanf:"partition"` + + // Shared is the file system every machine of the Slurm cluster sees at + // the same path: home directories, datasets, the jobs' output and + // checkpoints. Empty path: none, and a batch job's output stays on the + // node it ran on. + Shared SharedStorage `koanf:"shared"` + + // Users is who keeps the Linux users that run jobs, the same on every + // machine of the Slurm cluster: "llmgw" makes one for each active llmgw + // account (its login from the e-mail address, a UID of llmgw's from + // 200000 up, its SSH keys from the portal); empty leaves users to the + // site (a directory through sssd, or by hand). + Users string `koanf:"users"` +} + +// SharedStorage is where the Slurm cluster's shared files come from: the +// site's NFS export (NFS), an NFS export llmgw keeps on one of its nodes +// (Server), or, with neither, a file system the site mounts itself +// (Lustre, BeeGFS, GPFS), which llmgw only looks for. +type SharedStorage struct { + // Path is where every machine sees it, as /shared. + Path string `koanf:"path"` + // NFS is the site's export, host:/path, mounted at Path on every + // machine of the Slurm cluster. + NFS string `koanf:"nfs"` + // Server is the llmgw node, the controller or one of the nodes, that + // exports Path over NFS to the others: for a site with no storage of + // its own. One machine: not highly available, and no faster than its + // disk and network. + Server string `koanf:"server"` +} + +// UsersLlmgw has llmgw make the Slurm cluster's users from its accounts. +const UsersLlmgw = "llmgw" + +// HomeDir is where a user's home directory is: under the shared path when +// there is one, else on each machine's own disk. +func (s SlurmSettings) HomeDir(login string) string { + if s.Shared.Path != "" { + return path.Join(s.Shared.Path, "home", login) + } + return path.Join("/home", login) +} + +// nfsExport is host:/path. +var nfsExport = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:\[\]-]*:/[A-Za-z0-9_./-]*$`) + +// sharedPath is an absolute path of plain names, which a mount unit's name +// is made from. +var sharedPath = regexp.MustCompile(`^(/[A-Za-z0-9_.-]+)+$`) + +func (s SlurmSettings) checkShared() error { + sh := s.Shared + if sh.Path == "" { + if sh.NFS != "" || sh.Server != "" { + return fmt.Errorf("slurm.shared.path is empty: say where every machine sees the shared files, as /shared") + } + return nil + } + if !sharedPath.MatchString(sh.Path) || strings.Contains(sh.Path, "/.") { + return fmt.Errorf("slurm.shared.path %q is not an absolute path of plain names, as /shared", sh.Path) + } + for _, sys := range []string{"/", "/etc", "/usr", "/var", "/var/lib", "/home", "/root", "/boot", "/opt", "/srv", "/tmp", "/proc", "/sys", "/dev", "/run"} { + if path.Clean(sh.Path) == sys { + return fmt.Errorf("slurm.shared.path %s is the system's own directory: take one of its own, as /shared", sh.Path) + } + } + if sh.NFS != "" && sh.Server != "" { + return fmt.Errorf("slurm.shared names both the site's export (nfs) and an llmgw node to export it (server): one of them") + } + if sh.NFS != "" && !nfsExport.MatchString(sh.NFS) { + return fmt.Errorf("slurm.shared.nfs %q is not host:/path", sh.NFS) + } + if sh.Server != "" && sh.Server != s.Controller && !s.AllNodes && !slices.Contains(s.Nodes, sh.Server) { + return fmt.Errorf("slurm.shared.server %q is neither the controller nor one of slurm.nodes", sh.Server) + } + return nil } // slurmName is what Slurm takes as a node, cluster or partition name. @@ -111,5 +190,8 @@ func (s SlurmSettings) Check() error { return fmt.Errorf("slurm.lent[%d] %q is not in slurm.nodes: only a node that runs slurmd can be lent", i, n) } } - return nil + if s.Users != "" && s.Users != UsersLlmgw { + return fmt.Errorf("slurm.users %q: %q has llmgw make them, empty leaves them to the site", s.Users, UsersLlmgw) + } + return s.checkShared() } diff --git a/internal/config/slurm_test.go b/internal/config/slurm_test.go index 78dbeb8..b2175a1 100644 --- a/internal/config/slurm_test.go +++ b/internal/config/slurm_test.go @@ -35,6 +35,49 @@ func TestSlurmCheck(t *testing.T) { } } +func TestSlurmShared(t *testing.T) { + base := SlurmSettings{Enabled: true, Controller: "c", Nodes: []string{"a"}} + for _, sh := range []SharedStorage{{}, {Path: "/shared"}, {Path: "/shared", NFS: "filer.site:/export/hpc"}, + {Path: "/data/hpc", Server: "c"}, {Path: "/shared", Server: "a"}} { + s := base + s.Shared = sh + if err := s.Check(); err != nil { + t.Errorf("%+v refused: %v", sh, err) + } + } + for want, sh := range map[string]SharedStorage{ + "path is empty": {NFS: "f:/x"}, + "plain names": {Path: "shared"}, + "system's own": {Path: "/home"}, + "one of them": {Path: "/shared", NFS: "f:/x", Server: "c"}, + "not host:/path": {Path: "/shared", NFS: "f/x"}, + "neither": {Path: "/shared", Server: "z"}, + } { + s := base + s.Shared = sh + if err := s.Check(); err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%+v: got %v, want %q", sh, err, want) + } + } + s := base + s.Shared = SharedStorage{Path: "/a/../etc"} + if s.Check() == nil { + t.Error("a path with .. taken") + } + s.Shared, s.Users = SharedStorage{}, "ldap" + if s.Check() == nil { + t.Error("slurm.users ldap taken") + } + s.Users, s.Shared = UsersLlmgw, SharedStorage{Path: "/shared"} + if s.Check() != nil || s.HomeDir("ayse") != "/shared/home/ayse" { + t.Errorf("home %s", s.HomeDir("ayse")) + } + s.Shared = SharedStorage{} + if s.HomeDir("ayse") != "/home/ayse" { + t.Errorf("home %s", s.HomeDir("ayse")) + } +} + func TestSlurmAllNodes(t *testing.T) { s := SlurmSettings{Enabled: true, Controller: "c", AllNodes: true, Lent: []string{"joined"}} if err := s.Check(); err != nil { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 583d571..43143cb 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -98,6 +98,8 @@ type API struct { usage Usage quotas Quotas ctl *control.Controller + // slurmUsers are the Linux users made from the accounts; nil without a store. + slurmUsers SlurmUsers } // New builds the API over a controller. diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index d003789..2af4ec5 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -85,6 +85,7 @@ var PathParams = map[string]string{ "projects/{id}": "The project's ID.", "keys/{id}": "The key's ID, as listed (not the key itself).", "users/{id}": "The person's ID or email.", + "users/{user}": "The person's ID or email.", "artifacts/{kind}": "What the job made: peft, a gzipped tar of the PEFT adapter, or gguf, the adapter for llama.cpp.", } @@ -398,6 +399,14 @@ var Routes = []Route{ "(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " + "still running) or unknown (its agent has not said).", Response: slurmsvc.Status{}}, + {Method: "GET", Path: "/api/v1/slurm/users", Summary: "The Linux users made from llmgw's accounts for the Slurm cluster", + Description: "Every active account, with its login, UID and SSH keys; an account is given its login and UID the first " + + "time it is listed. The machines have them when slurm.users is \"llmgw\".", + Response: []slurmsvc.User{}}, + {Method: "PUT", Path: "/api/v1/slurm/users/{user}/ssh-keys", Summary: "Replace an account's SSH public keys", + Description: "{user} is an ID or e-mail. keys is OpenSSH public keys, one a line, without options; empty takes them " + + "all away. The machines have them within a minute.", + Request: SlurmKeysRequest{}, Response: slurmsvc.User{}}, {Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node", Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " + diff --git a/internal/control/adminapi/slurm.go b/internal/control/adminapi/slurm.go index f899acc..1935af6 100644 --- a/internal/control/adminapi/slurm.go +++ b/internal/control/adminapi/slurm.go @@ -1,15 +1,43 @@ package adminapi import ( + "context" + "encoding/json" + "errors" "net/http" + "strings" "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader // keeps it. type SlurmService interface { Status() slurmsvc.Status + UsersChanged() +} + +// SlurmUsers are the Linux users llmgw makes from its accounts. +type SlurmUsers interface { + SlurmUsers(ctx context.Context) ([]store.SlurmUser, error) + SlurmUserByID(ctx context.Context, userID string) (store.SlurmUser, error) + SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error +} + +// SetSlurmUsers enables the Slurm users' endpoints. +func (a *API) SetSlurmUsers(u SlurmUsers) { a.slurmUsers = u } + +// SlurmKeysRequest is a person's SSH public keys, one a line, as +// authorized_keys has them; empty takes them all away. +type SlurmKeysRequest struct { + Keys string `json:"keys"` +} + +// SlurmUserView is the user from a store row. +func SlurmUserView(u store.SlurmUser) slurmsvc.User { + return slurmsvc.User{Login: u.Login, UID: u.UID, Name: u.Name, Email: u.Email, SSHKeys: u.SSHKeys} } // SetSlurm gives the API the Slurm service. @@ -17,6 +45,69 @@ func (a *API) SetSlurm(s SlurmService) { a.slurm = s } func (a *API) registerSlurm(mux Mux) { mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus) + mux.HandleFunc("GET /api/v1/slurm/users", a.slurmUserList) + mux.HandleFunc("PUT /api/v1/slurm/users/{user}/ssh-keys", a.slurmUserKeys) +} + +func (a *API) slurmUserList(w http.ResponseWriter, r *http.Request) { + if a.slurmUsers == nil { + writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without") + return + } + us, err := a.slurmUsers.SlurmUsers(r.Context()) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + out := make([]slurmsvc.User, 0, len(us)) + for _, u := range us { + out = append(out, SlurmUserView(u)) + } + writeJSON(w, http.StatusOK, out) +} + +// slurmUserKeys replaces a person's SSH keys; {user} is an ID or e-mail. +func (a *API) slurmUserKeys(w http.ResponseWriter, r *http.Request) { + if a.slurmUsers == nil { + writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without") + return + } + if !a.tenancyReady(w) { + return + } + var in SlurmKeysRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&in); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + keys, err := slurm.ParseSSHKeys(in.Keys) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + ctx := r.Context() + u, err := a.findUser(ctx, r.PathValue("user")) + if err != nil { + writeError(w, http.StatusNotFound, strings.TrimPrefix(err.Error(), store.ErrNotFound.Error()+": ")) + return + } + if err := a.slurmUsers.SetSlurmSSHKeys(ctx, u.ID, keys); err != nil { + code := http.StatusInternalServerError + if errors.Is(err, store.ErrNotFound) { + code = http.StatusNotFound + } + writeError(w, code, err.Error()) + return + } + if a.slurm != nil { + a.slurm.UsersChanged() + } + su, err := a.slurmUsers.SlurmUserByID(ctx, u.ID) + if err != nil { + writeError(w, http.StatusInternalServerError, err.Error()) + return + } + writeJSON(w, http.StatusOK, SlurmUserView(su)) } func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) { diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 53cabd9..8f29747 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -71,17 +71,21 @@ type NodeStatus struct { // Reported is the agent's SlurmStatus, for the admin API. type Reported struct { - Version string `json:"version,omitempty"` - Hardware string `json:"hardware,omitempty"` - SlurmdRunning bool `json:"slurmd_running"` - SlurmctldRunning bool `json:"slurmctld_running"` - SlurmdbdRunning bool `json:"slurmdbd_running"` - State string `json:"state,omitempty"` - Reason string `json:"reason,omitempty"` - Jobs int `json:"jobs"` - Error string `json:"error,omitempty"` - FilesSHA256 string `json:"files_sha256,omitempty"` - At time.Time `json:"at"` + Version string `json:"version,omitempty"` + Hardware string `json:"hardware,omitempty"` + SlurmdRunning bool `json:"slurmd_running"` + SlurmctldRunning bool `json:"slurmctld_running"` + SlurmdbdRunning bool `json:"slurmdbd_running"` + // Shared is the shared path here: "mounted", "exported", "missing". + Shared string `json:"shared,omitempty"` + // Users is how many of llmgw's users the machine has. + Users int `json:"users"` + State string `json:"state,omitempty"` + Reason string `json:"reason,omitempty"` + Jobs int `json:"jobs"` + Error string `json:"error,omitempty"` + FilesSHA256 string `json:"files_sha256,omitempty"` + At time.Time `json:"at"` } // Status is the whole Slurm cluster, for `ctl slurm status`. @@ -95,6 +99,20 @@ type Status struct { // Waiting are nodes left out of slurm.conf until they report their // hardware. Waiting []string `json:"waiting,omitempty"` + // Shared is where the shared files are and come from; Users who keeps + // the users, and how many llmgw has. + Shared config.SharedStorage `json:"shared"` + Users string `json:"users,omitempty"` + UserCount int `json:"user_count,omitempty"` +} + +// User is a Linux user made from an llmgw account. +type User struct { + Login string `json:"login"` + UID int `json:"uid"` + Name string `json:"name"` + Email string `json:"email"` + SSHKeys []string `json:"ssh_keys,omitempty"` } // Service keeps the Slurm cluster. @@ -127,6 +145,46 @@ type Service struct { // started is the first pass, from which a node neither heard from nor // kept is waited for, for HoldFor. started time.Time + // userList reads the users from the accounts; users is what it last + // said, read at usersAt. Until it has been read once no node is told + // anything of users: an empty list would take every user away. + userList func(ctx context.Context) ([]User, error) + users []User + usersAt time.Time + usersOK bool +} + +// UsersEvery is how often the users are read again without being asked. +var UsersEvery = 30 * time.Second + +// SetUsers has the users read with list, for slurm.users "llmgw". +func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) { + s.mu.Lock() + s.userList = list + s.mu.Unlock() +} + +// UsersChanged has the users read again now: an account or its keys +// changed. +func (s *Service) UsersChanged() { + s.mu.Lock() + s.usersAt = time.Time{} + s.mu.Unlock() + s.Kick() +} + +// readUsers reads the users when they are due. Under s.mu. +func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) { + if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery { + return + } + us, err := s.userList(ctx) + s.usersAt = time.Now() + if err != nil { + s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err) + return + } + s.users, s.usersOK = us, true } // HoldFor is how long a new leader waits for a node it neither has heard @@ -198,6 +256,7 @@ func (s *Service) Kick() { // OnSlurmStatus records what a node's agent says. func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), + Shared: st.GetShared(), Users: int(st.GetUsers()), SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(), Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} s.mu.Lock() @@ -227,6 +286,7 @@ func (s *Service) Tick(ctx context.Context) { s.mu.Lock() defer s.mu.Unlock() s.recall(ctx) + s.readUsers(ctx, set) if s.started.IsZero() { s.started = time.Now() } @@ -477,16 +537,103 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key m.Controller = true m.Files = maps.Clone(files) } + if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) { + return nil + } return m } +// sharedPart puts the shared files in m, false when where they come from +// is not known yet: told nothing rather than no shared files, which would +// unmount them under running jobs. Under s.mu. +func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool { + sh := set.Shared + if sh.Path == "" { + return true + } + m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS} + switch { + case sh.Server == "": + case sh.Server == node: + m.Shared.Serve = true + for _, n := range s.machines(set) { + if n == node { + continue + } + if a := s.addr(set, n); a != "" { + m.Shared.Clients = append(m.Shared.Clients, a) + } + } + slices.Sort(m.Shared.Clients) + default: + a := s.addr(set, sh.Server) + if a == "" { + return false + } + if strings.Contains(a, ":") { + a = "[" + a + "]" + } + m.Shared.Nfs = a + ":" + sh.Path + } + return true +} + +// usersPart puts the users in m, false while they have not been read. +// Under s.mu. +func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool { + if set.Users != config.UsersLlmgw { + return true + } + if !s.usersOK { + return false + } + m.ManageUsers = true + switch { + case set.Shared.Path == "": + m.MakeHomes = true // each machine's own /home + case set.Shared.Server != "": + m.MakeHomes = set.Shared.Server == node + default: + m.MakeHomes = set.Controller == node + } + for _, u := range s.users { + m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up + Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)}) + } + return true +} + +// machines are the controller and the nodes. +func (s *Service) machines(set config.SlurmSettings) []string { + out := slices.Clone(set.Nodes) + if !slices.Contains(out, set.Controller) { + out = append(out, set.Controller) + } + return out +} + +// addr is a machine's management address, from the inventory, else kept +// from before. Under s.mu. +func (s *Service) addr(set config.SlurmSettings, node string) string { + if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" { + return n.MgmtAddr + } + if node == set.Controller { + return s.controllerAddr(set) + } + return s.known.Nodes[node].Addr +} + // Status is the cluster as the service sees it now. func (s *Service) Status() Status { set := s.current() s.mu.Lock() defer s.mu.Unlock() out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller, - Partition: set.PartitionName(), Ctld: s.status[set.Controller]} + Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users} + if set.Users == config.UsersLlmgw { + out.UserCount = len(s.users) + } if !set.Enabled { return out } diff --git a/internal/control/slurmsvc/shared_test.go b/internal/control/slurmsvc/shared_test.go new file mode 100644 index 0000000..3a147f1 --- /dev/null +++ b/internal/control/slurmsvc/shared_test.go @@ -0,0 +1,85 @@ +package slurmsvc + +import ( + "context" + "errors" + "slices" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" +) + +func TestSharedFilesFromTheController(t *testing.T) { + r := newRig() + r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"} + r.tick() + ctl, node := r.agents.last("node06").GetShared(), r.agents.last("cuda01").GetShared() + if !ctl.GetServe() || !slices.Equal(ctl.GetClients(), []string{"10.0.0.147"}) || ctl.GetPath() != "/shared" { + t.Fatalf("server told %v", ctl) + } + if node.GetServe() || node.GetNfs() != "10.0.0.146:/shared" { + t.Fatalf("client told %v", node) + } + // The site's own export: every machine mounts it. + r.set.Shared = config.SharedStorage{Path: "/shared", NFS: "filer:/hpc"} + r.tick() + if r.agents.last("node06").GetShared().GetNfs() != "filer:/hpc" || r.agents.last("cuda01").GetShared().GetServe() { + t.Fatal("the site's export not sent") + } +} + +func TestSharedServerNotKnownTellsNothing(t *testing.T) { + r := newRig() + r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"} + delete(r.inv.nodes, "node06") + r.svc.known.ControllerAddr, r.svc.known.Controller = "10.0.0.146", "node06" + r.tick() + if r.agents.last("cuda01").GetShared().GetNfs() != "10.0.0.146:/shared" { + t.Fatalf("the kept address not used: %v", r.agents.last("cuda01")) + } + r2 := newRig() + r2.set.Nodes = []string{"cuda01", "site2"} + r2.agents.conn = append(r2.agents.conn, "site2") + r2.set.Shared = config.SharedStorage{Path: "/shared", Server: "site2"} + r2.tick() + if r2.agents.last("cuda01") != nil { + t.Fatal("told with no address for the server: its mount would go") + } +} + +func TestUsersSentOnceRead(t *testing.T) { + r := newRig() + r.set.Users = config.UsersLlmgw + r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"} + fail := true + r.svc.SetUsers(func(context.Context) ([]User, error) { + if fail { + return nil, errors.New("store down") + } + return []User{{Login: "ayse", UID: 200000, Name: "Ayşe", SSHKeys: []string{"ssh-ed25519 AAAA"}}}, nil + }) + r.tick() + if len(r.agents.got["cuda01"]) != 0 { + t.Fatal("told before the users were read: every user would go") + } + fail = false + r.svc.UsersChanged() + r.tick() + ctl, node := r.agents.last("node06"), r.agents.last("cuda01") + if !ctl.GetManageUsers() || !ctl.GetMakeHomes() || node.GetMakeHomes() || len(node.GetUsers()) != 1 { + t.Fatalf("controller %v\nnode %v", ctl, node) + } + if u := node.GetUsers()[0]; u.GetHome() != "/shared/home/ayse" || u.GetUid() != 200000 || len(u.GetSshKeys()) != 1 { + t.Fatalf("user %v", u) + } + // The store down later: the last users are kept. + fail = true + r.svc.UsersChanged() + r.tick() + if len(r.agents.last("cuda01").GetUsers()) != 1 { + t.Fatal("users lost when the store could not be read") + } + if st := r.svc.Status(); st.UserCount != 1 || st.Shared.Server != "node06" { + t.Fatalf("status %+v", st) + } +} diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index 77d7366..3cbc98a 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -510,6 +510,11 @@ var Reference = []Block{ {Package: "config", Name: "ServingSettings", Title: "ServingSettings", Doc: "", Fields: []Field{ {Key: "stopped", Type: "bool", Doc: "Stopped takes every model down and answers every request for one with\n503 (system_stopped), leaving the control planes, agents and mirror\nrunning and every model's definition as it is: llmgw stop sets it,\nllmgw start clears it, and the models come back as their catalog\nentries ask. In force on every control plane within seconds.", Nested: ""}, }}, + {Package: "config", Name: "SharedStorage", Title: "SharedStorage", Doc: "", Fields: []Field{ + {Key: "path", Type: "string", Doc: "Path is where every machine sees it, as /shared.", Nested: ""}, + {Key: "nfs", Type: "string", Doc: "NFS is the site's export, host:/path, mounted at Path on every\nmachine of the Slurm cluster.", Nested: ""}, + {Key: "server", Type: "string", Doc: "Server is the llmgw node, the controller or one of the nodes, that\nexports Path over NFS to the others: for a site with no storage of\nits own. One machine: not highly available, and no faster than its\ndisk and network.", Nested: ""}, + }}, {Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{ {Key: "bin", Type: "string", Doc: "Bin is the llmgw binary itself; the engine is one of its subcommands.\nEmpty means whatever started this agent, which is right almost always.", Nested: ""}, }}, @@ -521,6 +526,8 @@ var Reference = []Block{ {Key: "all_nodes", Type: "bool", Doc: "AllNodes runs slurmd on every llmgw node, those that join later too,\nbesides any named in Nodes. Each is llmgw's until it is lent, as any.", Nested: ""}, {Key: "lent", Type: "[]string", Doc: "Lent are the nodes of Nodes Slurm may run jobs on now. A node put here\nis drained of llmgw's replicas first, as `ctl drain` does, and resumed\nin Slurm once none is left; a node taken out is drained in Slurm and\ngiven back to llmgw when its last job ends.", Nested: ""}, {Key: "partition", Type: "string", Doc: "Partition is the one partition every node is in. Empty is \"gpu\".", Nested: ""}, + {Key: "shared", Type: "SharedStorage", Doc: "Shared is the file system every machine of the Slurm cluster sees at\nthe same path: home directories, datasets, the jobs' output and\ncheckpoints. Empty path: none, and a batch job's output stays on the\nnode it ran on.", Nested: "config.SharedStorage"}, + {Key: "users", Type: "string", Doc: "Users is who keeps the Linux users that run jobs, the same on every\nmachine of the Slurm cluster: \"llmgw\" makes one for each active llmgw\naccount (its login from the e-mail address, a UID of llmgw's from\n200000 up, its SSH keys from the portal); empty leaves users to the\nsite (a directory through sssd, or by hand).", Nested: ""}, }}, {Package: "engines", Name: "StrataConfig", Title: "StrataConfig", Doc: "", Fields: []Field{ {Key: "dir", Type: "string", Doc: "Dir is Strata's source directory, where setup.py and serve/ are.", Nested: ""}, diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 3689369..cba291d 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -253,6 +253,9 @@ func (r *Router) Handler() http.Handler { pages.SetPersonalDefaults(r.personalDefaults) if r.store != nil { pages.SetAudit(r.store) + if r.slurmSettings != nil { + pages.SetSlurm(r.store, r.slurmSettings) + } // A forgotten password: a link mailed when the cluster's // settings say how, otherwise asked of an operator. pages.SetPasswordReset(r.store, r.mail, r.mail.BaseURL) diff --git a/internal/router/router.go b/internal/router/router.go index cf60adf..d851fc8 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -208,6 +208,9 @@ type Router struct { // settingsCheck and settingsLocal mount the portal's settings page. settingsCheck func([]byte) error settingsLocal []string + // slurmSettings is the cluster's Slurm settings, for the portal; nil + // offers nothing of Slurm. + slurmSettings func() config.SlurmSettings } // SetClusterView gives the router the data the /admin page renders. @@ -238,6 +241,10 @@ func (r *Router) SetSettingsCheck(check func([]byte) error, local []string) { r.settingsCheck, r.settingsLocal = check, local } +// SetSlurm has the portal offer the Slurm cluster's login and SSH keys as +// settings say. +func (r *Router) SetSlurm(s func() config.SlurmSettings) { r.slurmSettings = s } + // SetStopped refuses, or again serves, every request for a model. func (r *Router) SetStopped(stopped bool) { r.stopped.Store(stopped) } diff --git a/internal/slurm/shared.go b/internal/slurm/shared.go new file mode 100644 index 0000000..1645d32 --- /dev/null +++ b/internal/slurm/shared.go @@ -0,0 +1,127 @@ +package slurm + +import ( + "fmt" + "regexp" + "slices" + "strings" + + "golang.org/x/crypto/ssh" +) + +// Shared storage: one path every machine of the Slurm cluster sees, from +// the site's NFS export, from an llmgw node that exports it, or mounted by +// the site itself. +const ( + // NFSPort is NFSv4's one port, which is all a client needs. + NFSPort = 2049 + // ExportsPath is the export llmgw keeps on the machine that serves. + ExportsPath = "/etc/exports.d/llmgw-slurm.exports" + // MountUnitDir is where the mount unit is written. + MountUnitDir = "/etc/systemd/system" +) + +// Exports is the exports file for path, read-write to clients alone. Root +// on a client is nobody here: a client's root cannot take another's files. +func Exports(path string, clients []string) string { + cs := slices.Clone(clients) + slices.Sort(cs) + cs = slices.Compact(cs) + var b strings.Builder + b.WriteString("# Written by llmgw (BACKLOG P): the Slurm cluster's shared files, to its machines alone.\n") + if len(cs) == 0 { + return b.String() + } + fmt.Fprintf(&b, "%q", path) + for _, c := range cs { + fmt.Fprintf(&b, " %s(rw,sync,no_subtree_check,root_squash)", c) + } + b.WriteString("\n") + return b.String() +} + +// MountUnitName is systemd's name for the mount unit of path, as +// systemd-escape --path makes it. +func MountUnitName(path string) string { + p := strings.Trim(path, "/") + var b strings.Builder + for i := 0; i < len(p); i++ { + c := p[i] + switch { + case c == '/': + b.WriteByte('-') + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '_', c == '.' && i > 0: + b.WriteByte(c) + default: + fmt.Fprintf(&b, `\x%02x`, c) + } + } + return b.String() + ".mount" +} + +// MountUnit mounts the export nfs at path. hard: a job waits for a server +// that is gone rather than reading errors; nconnect spreads a client's +// traffic over several connections. The version is the highest both +// sides speak. +func MountUnit(nfs, path string) string { + return "# Written by llmgw (BACKLOG P): the Slurm cluster's shared files.\n" + + "[Unit]\nDescription=llmgw: the Slurm cluster's shared files\n" + + "After=network-online.target\nWants=network-online.target\n\n" + + "[Mount]\nWhat=" + nfs + "\nWhere=" + path + "\nType=nfs\n" + + "Options=hard,nconnect=4,_netdev\nTimeoutSec=60\n\n" + + "[Install]\nWantedBy=remote-fs.target\n" +} + +// Users: Linux users llmgw makes from its accounts, the same login and UID +// on every machine, logging in with the SSH keys given in the portal. +const ( + // AuthorizedKeysDir holds each user's keys, root's to write, so a + // user's own home (on a file system shared with others) need hold none. + AuthorizedKeysDir = "/etc/llmgw/ssh-keys" + // SSHDropIn has sshd look there too, after the user's own files. + SSHDropIn = "/etc/ssh/sshd_config.d/50-llmgw-slurm.conf" + // MinUID is the lowest UID llmgw gives. + MinUID = 200000 +) + +// SSHConf is SSHDropIn's settings. +const SSHConf = "# Written by llmgw (BACKLOG P): the Slurm users' keys, from llmgw's portal.\n" + + "AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys2 " + AuthorizedKeysDir + "/%u\n" + +// login is what useradd takes, as llmgw gives it. +var login = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,31}$`) + +// ValidLogin is whether a login is one llmgw gives. +func ValidLogin(s string) bool { return login.MatchString(s) } + +// ParseSSHKeys reads OpenSSH public keys, one a line, as authorized_keys +// takes them, without options: what a person pastes into the portal. +// Blank lines and # comments are skipped; each key comes back as its type, +// its base64 and its comment. +func ParseSSHKeys(text string) ([]string, error) { + var out []string + for i, line := range strings.Split(text, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + pk, comment, opts, _, err := ssh.ParseAuthorizedKey([]byte(line)) + if err != nil { + return nil, fmt.Errorf("line %d is not an SSH public key (as ~/.ssh/id_ed25519.pub has it)", i+1) + } + if len(opts) > 0 { + return nil, fmt.Errorf("line %d has options before its key; only the key is taken", i+1) + } + k := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(pk))) + if comment = strings.Join(strings.Fields(comment), " "); comment != "" { + k += " " + comment + } + if !slices.Contains(out, k) { + out = append(out, k) + } + } + if len(out) > 20 { + return nil, fmt.Errorf("%d keys: 20 at most", len(out)) + } + return out, nil +} diff --git a/internal/slurm/shared_test.go b/internal/slurm/shared_test.go new file mode 100644 index 0000000..4849940 --- /dev/null +++ b/internal/slurm/shared_test.go @@ -0,0 +1,42 @@ +package slurm + +import ( + "strings" + "testing" +) + +func TestExportsAndMountUnit(t *testing.T) { + e := Exports("/shared", []string{"10.0.0.22", "10.0.0.21", "10.0.0.22"}) + if !strings.Contains(e, `"/shared" 10.0.0.21(rw,sync,no_subtree_check,root_squash) 10.0.0.22(rw,`) || + strings.Count(e, "10.0.0.22") != 1 { + t.Fatalf("exports:\n%s", e) + } + if strings.Contains(Exports("/shared", nil), "/shared\"") { + t.Fatal("an export to nobody") + } + for path, want := range map[string]string{"/shared": "shared.mount", "/data/hpc": "data-hpc.mount", + "/hpc-data": `hpc\x2ddata.mount`, "/.b": `\x2eb.mount`} { + if got := MountUnitName(path); got != want { + t.Errorf("%s: %s, want %s", path, got, want) + } + } + if u := MountUnit("10.0.0.11:/shared", "/shared"); !strings.Contains(u, "What=10.0.0.11:/shared\nWhere=/shared\nType=nfs") { + t.Fatalf("unit:\n%s", u) + } +} + +func TestParseSSHKeys(t *testing.T) { + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILq3vMdLAKlaDdxwrSgCtP/j/qrF4nzxNBNGuJmK5vxh" + got, err := ParseSSHKeys("# mine\n\n" + key + " ayse@laptop \n" + key + " ayse@laptop\n") + if err != nil || len(got) != 1 || got[0] != key+" ayse@laptop" { + t.Fatalf("%q %v", got, err) + } + for _, bad := range []string{"not a key", `command="rm -rf /" ` + key} { + if _, err := ParseSSHKeys(bad); err == nil { + t.Errorf("%q taken", bad) + } + } + if !ValidLogin("ali_veli") || ValidLogin("9x") || ValidLogin("Ayse") { + t.Error("ValidLogin") + } +} diff --git a/internal/store/copy.go b/internal/store/copy.go index 589e936..fc96479 100644 --- a/internal/store/copy.go +++ b/internal/store/copy.go @@ -29,6 +29,7 @@ var copyOrder = []string{ "user_excluded_licences", // After users, which it points at. "user_directory", + "slurm_users", "preset_state", "api_keys", "key_claims", diff --git a/internal/store/migrations/postgres/0052_slurm_users.sql b/internal/store/migrations/postgres/0052_slurm_users.sql new file mode 100644 index 0000000..929f75d --- /dev/null +++ b/internal/store/migrations/postgres/0052_slurm_users.sql @@ -0,0 +1,12 @@ +-- Ported from the SQLite migration of the same number. +-- The Linux user llmgw makes for an account on every machine of the Slurm +-- cluster, when slurm.users is "llmgw": its login and UID (also its group's +-- GID), given once and never again to another, and the SSH public keys it +-- logs in with. No reference to users: a deleted account's row stays, so its +-- login and UID, still on the files it left, are not handed to someone else. +CREATE TABLE slurm_users ( + user_id TEXT PRIMARY KEY, + login TEXT NOT NULL UNIQUE, + uid INTEGER NOT NULL UNIQUE, + ssh_keys TEXT NOT NULL DEFAULT '' +); diff --git a/internal/store/migrations/sqlite/0052_slurm_users.sql b/internal/store/migrations/sqlite/0052_slurm_users.sql new file mode 100644 index 0000000..bc57477 --- /dev/null +++ b/internal/store/migrations/sqlite/0052_slurm_users.sql @@ -0,0 +1,11 @@ +-- The Linux user llmgw makes for an account on every machine of the Slurm +-- cluster, when slurm.users is "llmgw": its login and UID (also its group's +-- GID), given once and never again to another, and the SSH public keys it +-- logs in with. No reference to users: a deleted account's row stays, so its +-- login and UID, still on the files it left, are not handed to someone else. +CREATE TABLE slurm_users ( + user_id TEXT PRIMARY KEY, + login TEXT NOT NULL UNIQUE, + uid INTEGER NOT NULL UNIQUE, + ssh_keys TEXT NOT NULL DEFAULT '' +); diff --git a/internal/store/slurm_users.go b/internal/store/slurm_users.go new file mode 100644 index 0000000..f7dfc95 --- /dev/null +++ b/internal/store/slurm_users.go @@ -0,0 +1,174 @@ +package store + +import ( + "context" + "database/sql" + "strconv" + "strings" +) + +// SlurmUser is the Linux user llmgw makes for an active account on every +// machine of the Slurm cluster (slurm.users "llmgw"). +type SlurmUser struct { + UserID string + Email string + Name string + Login string + UID int + // SSHKeys are OpenSSH public keys, one a line. + SSHKeys []string +} + +// FirstSlurmUID is the first UID llmgw gives: far above what a +// distribution or a site's own users take, and below the range systemd +// keeps for its containers (524288). +const FirstSlurmUID = 200000 + +// reservedLogins are names an account's login is never given: the +// system's, Slurm's and llmgw's own. +var reservedLogins = map[string]bool{ + "root": true, "daemon": true, "bin": true, "sys": true, "sync": true, "games": true, "man": true, + "lp": true, "mail": true, "news": true, "uucp": true, "proxy": true, "backup": true, "list": true, + "irc": true, "nobody": true, "admin": true, "administrator": true, "slurm": true, "munge": true, + "mysql": true, "mariadb": true, "postgres": true, "llmgw": true, "sshd": true, "ubuntu": true, + "debian": true, "centos": true, "rocky": true, "ec2-user": true, "cloud-user": true, "syslog": true, + "messagebus": true, "polkitd": true, "nfsnobody": true, "operator": true, "ftp": true, "adm": true, + "halt": true, "shutdown": true, "www-data": true, "gnats": true, "user": true, "guest": true, +} + +// loginFrom is an e-mail address's part before the @, as a Linux login: +// lower case, letters, digits, _ and -, a letter first, 24 at most. +func loginFrom(email string) string { + local, _, _ := strings.Cut(strings.ToLower(email), "@") + var b strings.Builder + for _, r := range local { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_', r == '-': + b.WriteRune(r) + case r == '.' || r == '+': + b.WriteByte('_') + } + } + l := strings.Trim(b.String(), "_-") + if l == "" || l[0] < 'a' || l[0] > 'z' { + l = "u" + l + } + if len(l) > 24 { + l = l[:24] + } + return l +} + +// SlurmUsers are the active accounts as Linux users, each given its login +// and UID the first time it is asked for. +func (s *DB) SlurmUsers(ctx context.Context) ([]SlurmUser, error) { + type row struct { + SlurmUser + has bool + } + rows, err := queryAll(ctx, s.db, + `SELECT u.id, u.email, u.name, COALESCE(su.login, ''), COALESCE(su.uid, 0), COALESCE(su.ssh_keys, '') + FROM users u LEFT JOIN slurm_users su ON su.user_id = u.id + WHERE u.state = ? ORDER BY u.created_at`, + []any{string(UserActive)}, func(r *sql.Rows) (row, error) { + var x row + var keys string + err := r.Scan(&x.UserID, &x.Email, &x.Name, &x.Login, &x.UID, &keys) + x.SSHKeys, x.has = splitKeys(keys), x.Login != "" + return x, err + }) + if err != nil { + return nil, err + } + out := make([]SlurmUser, 0, len(rows)) + for _, x := range rows { + if !x.has { + login, uid, err := s.giveSlurmLogin(ctx, x.UserID, x.Email) + if err != nil { + return nil, err + } + x.Login, x.UID = login, uid + } + out = append(out, x.SlurmUser) + } + return out, nil +} + +// SlurmUserByID is one account's Linux user, given its login and UID if it +// has none yet. +func (s *DB) SlurmUserByID(ctx context.Context, userID string) (SlurmUser, error) { + u, err := s.UserByID(ctx, userID) + if err != nil { + return SlurmUser{}, err + } + out := SlurmUser{UserID: u.ID, Email: u.Email, Name: u.Name} + var keys string + err = s.db.QueryRowContext(ctx, `SELECT login, uid, ssh_keys FROM slurm_users WHERE user_id = ?`, userID). + Scan(&out.Login, &out.UID, &keys) + switch { + case err == sql.ErrNoRows: + out.Login, out.UID, err = s.giveSlurmLogin(ctx, u.ID, u.Email) + case err == nil: + out.SSHKeys = splitKeys(keys) + } + return out, err +} + +// SetSlurmSSHKeys replaces an account's SSH public keys, checked by the +// caller. +func (s *DB) SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error { + if _, err := s.SlurmUserByID(ctx, userID); err != nil { + return err + } + _, err := s.db.ExecContext(ctx, `UPDATE slurm_users SET ssh_keys = ? WHERE user_id = ?`, + strings.Join(keys, "\n"), userID) + return err +} + +// giveSlurmLogin gives an account its login, its e-mail's local part with a +// number after it if that is taken, and the next UID. +func (s *DB) giveSlurmLogin(ctx context.Context, userID, email string) (string, int, error) { + var login string + var uid int + err := s.inTx(ctx, func(c conn) error { + // Another pass may have given it meanwhile. + err := c.QueryRowContext(ctx, `SELECT login, uid FROM slurm_users WHERE user_id = ?`, userID).Scan(&login, &uid) + if err == nil { + return nil + } + if err != sql.ErrNoRows { + return err + } + if err := c.QueryRowContext(ctx, `SELECT COALESCE(MAX(uid), 0) FROM slurm_users`).Scan(&uid); err != nil { + return err + } + uid = max(uid+1, FirstSlurmUID) + base := loginFrom(email) + login = base + for n := 2; ; n++ { + if !reservedLogins[login] { + var taken int + if err := c.QueryRowContext(ctx, `SELECT COUNT(*) FROM slurm_users WHERE login = ?`, login).Scan(&taken); err != nil { + return err + } + if taken == 0 { + break + } + } + login = base + strconv.Itoa(n) + } + _, err = c.ExecContext(ctx, `INSERT INTO slurm_users (user_id, login, uid) VALUES (?, ?, ?)`, userID, login, uid) + return err + }) + return login, uid, err +} + +func splitKeys(s string) []string { + var out []string + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + out = append(out, l) + } + } + return out +} diff --git a/internal/store/slurm_users_test.go b/internal/store/slurm_users_test.go new file mode 100644 index 0000000..700b050 --- /dev/null +++ b/internal/store/slurm_users_test.go @@ -0,0 +1,57 @@ +package store + +import ( + "context" + "testing" + "time" +) + +func TestSlurmUsersGetLoginsAndUIDsOnce(t *testing.T) { + ctx := context.Background() + db := openDB(t) + for i, e := range []string{"ayse@a.test", "Ayse@b.test", "root@c.test", "9lives@d.test", "ali.veli@e.test"} { + id := "u" + string(rune('0'+i)) + if err := db.CreateUser(ctx, User{ID: id, Email: e, Name: e, PasswordHash: "x", State: UserActive, + CreatedAt: time.Unix(int64(i), 0)}); err != nil { + t.Fatal(err) + } + } + if err := db.CreateUser(ctx, User{ID: "p", Email: "pending@x.test", PasswordHash: "x", State: UserPending, + CreatedAt: time.Unix(9, 0)}); err != nil { + t.Fatal(err) + } + us, err := db.SlurmUsers(ctx) + if err != nil { + t.Fatal(err) + } + want := []struct { + login string + uid int + }{{"ayse", 200000}, {"ayse2", 200001}, {"root2", 200002}, {"u9lives", 200003}, {"ali_veli", 200004}} + if len(us) != len(want) { + t.Fatalf("%d users: %+v", len(us), us) + } + for i, w := range want { + if us[i].Login != w.login || us[i].UID != w.uid { + t.Errorf("%s: %s %d, want %s %d", us[i].Email, us[i].Login, us[i].UID, w.login, w.uid) + } + } + if err := db.SetSlurmSSHKeys(ctx, "u0", []string{"ssh-ed25519 AAAA a", "ssh-ed25519 BBBB b"}); err != nil { + t.Fatal(err) + } + again, _ := db.SlurmUsers(ctx) + if again[0].UID != 200000 || len(again[0].SSHKeys) != 2 { + t.Fatalf("asked again: %+v", again[0]) + } + // A deleted account's UID is not given again. + if _, err := db.raw.ExecContext(ctx, `DELETE FROM users WHERE id = 'u4'`); err != nil { + t.Fatal(err) + } + if err := db.ApproveUser(ctx, "p", "", "admin", time.Now()); err != nil { + t.Fatal(err) + } + u, err := db.SlurmUserByID(ctx, "p") + if err != nil || u.UID != 200005 || u.Login != "pending" { + t.Fatalf("%+v %v", u, err) + } +} diff --git a/internal/web/account.go b/internal/web/account.go index aca225a..67569ae 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -17,6 +17,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/account" "gitea.mynodes.xyz/ilker/llmgw/internal/audit" "gitea.mynodes.xyz/ilker/llmgw/internal/clientip" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/vscodeext" @@ -151,6 +152,10 @@ type AccountPages struct { resets account.ResetStore mailer Mailer publicURL string + // slurmKeys and slurmSettings offer the Slurm cluster's login; see + // SetSlurm. + slurmKeys SlurmKeys + slurmSettings func() config.SlurmSettings } // ServedModels supplies the names the profile page's example should use. @@ -207,6 +212,7 @@ func (p *AccountPages) Register(mux *http.ServeMux) { mux.HandleFunc("POST /profile/keys/{id}/revoke", p.selfAudited(p.revokeKey)) mux.HandleFunc("POST /profile/password", p.selfAudited(p.changePassword)) mux.HandleFunc("POST /profile/licences", p.selfAudited(p.setLicences)) + mux.HandleFunc("POST /profile/ssh-keys", p.selfAudited(p.setSlurmKeys)) mux.HandleFunc("POST /profile/connect", p.selfAudited(p.connect)) mux.HandleFunc("GET /setup/{tool}", p.setupScript) mux.HandleFunc("GET /setup/vscode/"+vscodeext.FileName, p.vsix) @@ -415,6 +421,9 @@ type profileData struct { Licences *LicenceChoices LicencesSaved bool LicenceError string + // Slurm is the person's login to the Slurm cluster and its SSH keys; + // nil unless llmgw makes the cluster's users. + Slurm *slurmView PasswordChanged bool PasswordError string @@ -558,6 +567,7 @@ func (p *AccountPages) render(w http.ResponseWriter, r *http.Request, d.Keys = keys p.fillPlaces(r.Context(), langOf(r), user, &d) p.fillLicences(r.Context(), user.ID, &d) + p.fillSlurm(r.Context(), user, &d) render(w, r, "profile", d) } diff --git a/internal/web/locales/en/account.json b/internal/web/locales/en/account.json index ad007f6..0985fe1 100644 --- a/internal/web/locales/en/account.json +++ b/internal/web/locales/en/account.json @@ -188,5 +188,13 @@ "reset.bad": "This link does not work: it was used already, it expired, or a newer one was sent.", "reset.err.mismatch": "the two passwords are not the same", "reset.mail.subject": "Set a new password for llmgw", - "reset.mail.body": "Hello %s,\n\nA link to set a new password for your llmgw account was asked for:\n\n%s\n\nIt works once, for %d minutes. If you did not ask for it, ignore this message: your password stays as it is.\n" + "reset.mail.body": "Hello %s,\n\nA link to set a new password for your llmgw account was asked for:\n\n%s\n\nIt works once, for %d minutes. If you did not ask for it, ignore this message: your password stays as it is.\n", + "profile.slurm": "Slurm: login and SSH keys", + "profile.slurm.saved": "Your keys are kept; every machine of the Slurm cluster has them within a minute.", + "profile.slurm.intro": "You are %s (UID %d) on every machine of the Slurm cluster, your home %s. Log in to the controller with one of the keys below and submit jobs there.", + "profile.slurm.keys": "SSH public keys, one a line (as ~/.ssh/id_ed25519.pub)", + "profile.slurm.save": "Save keys", + "profile.slurm.note": "Only the public key: the private one never leaves your computer. An empty box takes every key away.", + "profile.slurm.err_keys": "Not kept: %s.", + "profile.slurm.placeholder": "ssh-ed25519 AAAA… you@laptop" } diff --git a/internal/web/locales/tr/account.json b/internal/web/locales/tr/account.json index 73e44b8..a5217a8 100644 --- a/internal/web/locales/tr/account.json +++ b/internal/web/locales/tr/account.json @@ -188,5 +188,13 @@ "reset.bad": "Bu bağlantı çalışmıyor: daha önce kullanılmış, süresi dolmuş ya da daha yenisi gönderilmiş.", "reset.err.mismatch": "iki şifre aynı değil", "reset.mail.subject": "llmgw için yeni şifre belirleyin", - "reset.mail.body": "Merhaba %s,\n\nllmgw hesabınız için yeni şifre belirleme bağlantısı istendi:\n\n%s\n\nBir kez ve %d dakika boyunca geçerli. Siz istemediyseniz bu mesajı dikkate almayın: şifreniz değişmez.\n" + "reset.mail.body": "Merhaba %s,\n\nllmgw hesabınız için yeni şifre belirleme bağlantısı istendi:\n\n%s\n\nBir kez ve %d dakika boyunca geçerli. Siz istemediyseniz bu mesajı dikkate almayın: şifreniz değişmez.\n", + "profile.slurm": "Slurm: giriş ve SSH anahtarları", + "profile.slurm.saved": "Anahtarlarınız kaydedildi; Slurm cluster'ının her makinesine bir dakika içinde ulaşır.", + "profile.slurm.intro": "Slurm cluster'ının her makinesinde %s (UID %d) olarak varsınız, ev dizininiz %s. Aşağıdaki anahtarlardan biriyle kontrolcüye bağlanıp işlerinizi oradan gönderin.", + "profile.slurm.keys": "SSH açık anahtarları, satır başına bir tane (~/.ssh/id_ed25519.pub gibi)", + "profile.slurm.save": "Anahtarları kaydet", + "profile.slurm.note": "Yalnızca açık anahtar: gizli anahtar bilgisayarınızdan hiç çıkmaz. Kutuyu boşaltmak bütün anahtarları kaldırır.", + "profile.slurm.err_keys": "Kaydedilmedi: %s.", + "profile.slurm.placeholder": "ssh-ed25519 AAAA… siz@dizüstü" } diff --git a/internal/web/slurmkeys.go b/internal/web/slurmkeys.go new file mode 100644 index 0000000..9de773f --- /dev/null +++ b/internal/web/slurmkeys.go @@ -0,0 +1,98 @@ +package web + +import ( + "context" + "net/http" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// SlurmKeys keeps the SSH keys a person logs in to the Slurm cluster with. +type SlurmKeys interface { + SlurmUserByID(ctx context.Context, userID string) (store.SlurmUser, error) + SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error +} + +// SetSlurm has the profile page offer the Slurm cluster's login and SSH +// keys while settings say llmgw makes its users. +func (p *AccountPages) SetSlurm(keys SlurmKeys, settings func() config.SlurmSettings) { + p.slurmKeys, p.slurmSettings = keys, settings +} + +// slurmView is the profile's Slurm part. +type slurmView struct { + Login string + UID int + Home string + // Host is where jobs are submitted: the controller. + Host string + Keys string + Saved bool + Error string +} + +// slurmOn is the settings, and whether the page offers the part. +func (p *AccountPages) slurmOn() (config.SlurmSettings, bool) { + if p.slurmKeys == nil || p.slurmSettings == nil { + return config.SlurmSettings{}, false + } + s := p.slurmSettings() + return s, s.Enabled && s.Users == config.UsersLlmgw +} + +// fillSlurm puts the person's Slurm login in d, keeping what d says of a +// save. +func (p *AccountPages) fillSlurm(ctx context.Context, user store.User, d *profileData) { + set, on := p.slurmOn() + if !on || !user.CanSignIn() { + return + } + u, err := p.slurmKeys.SlurmUserByID(ctx, user.ID) + if err != nil { + p.log.Error("could not read a person's Slurm login", "user", user.ID, "err", err) + return + } + v := &slurmView{Login: u.Login, UID: u.UID, Home: set.HomeDir(u.Login), Host: set.Controller, + Keys: strings.Join(u.SSHKeys, "\n")} + if d.Slurm != nil { + v.Saved, v.Error = d.Slurm.Saved, d.Slurm.Error + if v.Error != "" { + v.Keys = d.Slurm.Keys + } + } + d.Slurm = v +} + +func (p *AccountPages) setSlurmKeys(w http.ResponseWriter, r *http.Request) { + noStore(w) + sess, user, ok := p.session(r) + if !ok { + http.Redirect(w, r, "/login", http.StatusSeeOther) + return + } + if _, on := p.slurmOn(); !on { + http.NotFound(w, r) + return + } + if !p.accounts.CheckCSRF(sess, r.FormValue("csrf")) { + badForm(w, r) + return + } + l := langOf(r) + text := r.FormValue("keys") + keys, err := slurm.ParseSSHKeys(text) + if err != nil { + p.render(w, r, sess, user, profileData{Slurm: &slurmView{Keys: text, Error: l.T("profile.slurm.err_keys", err.Error())}}) + return + } + if err := p.slurmKeys.SetSlurmSSHKeys(r.Context(), user.ID, keys); err != nil { + p.log.Error("could not keep a person's SSH keys", "user", user.ID, "err", err) + p.render(w, r, sess, user, profileData{Slurm: &slurmView{Keys: text, Error: l.T("profile.err.try_again")}}) + return + } + p.log.Info("a user changed their SSH keys for Slurm", "user", user.ID, "keys", len(keys)) + p.render(w, r, sess, user, profileData{Slurm: &slurmView{Saved: true}}) +} diff --git a/internal/web/slurmkeys_test.go b/internal/web/slurmkeys_test.go new file mode 100644 index 0000000..c0b6d82 --- /dev/null +++ b/internal/web/slurmkeys_test.go @@ -0,0 +1,74 @@ +package web + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +func TestTheProfileKeepsAPersonsSSHKeysForSlurm(t *testing.T) { + ctx := t.Context() + st, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer st.Close() + ayse := store.User{ID: "usr-a", Email: "ayse@llmt.test", Name: "Ayşe", State: store.UserActive, CreatedAt: time.Now()} + if err := st.CreateUser(ctx, ayse); err != nil { + t.Fatal(err) + } + set := config.SlurmSettings{Enabled: true, Controller: "llmt-1", Users: config.UsersLlmgw, + Shared: config.SharedStorage{Path: "/shared", Server: "llmt-1"}} + p := NewAccountPages(fakeAccounts{user: ayse}, st, nil, "v0", slog.New(slog.DiscardHandler)) + p.SetSlurm(st, func() config.SlurmSettings { return set }) + mux := http.NewServeMux() + p.Register(mux) + do := func(form url.Values) (int, string) { + req := httptest.NewRequest(http.MethodGet, "/profile", nil) + if form != nil { + form.Set("csrf", "tok") + req = httptest.NewRequest(http.MethodPost, "/profile/ssh-keys", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + req.AddCookie(&http.Cookie{Name: SessionCookie, Value: "good"}) + req.AddCookie(&http.Cookie{Name: csrfCookie, Value: "tok"}) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + return rec.Code, rec.Body.String() + } + _, page := do(nil) + for _, want := range []string{`action="/profile/ssh-keys"`, "ssh ayse@llmt-1", "UID 200000", "/shared/home/ayse"} { + if !strings.Contains(page, want) { + t.Errorf("the page lacks %q", want) + } + } + key := "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILq3vMdLAKlaDdxwrSgCtP/j/qrF4nzxNBNGuJmK5vxh ayse@laptop" + _, saved := do(url.Values{"keys": {key + "\n"}}) + u, _ := st.SlurmUserByID(ctx, ayse.ID) + if len(u.SSHKeys) != 1 || u.SSHKeys[0] != key || !strings.Contains(saved, "within a minute") { + t.Fatalf("kept %v", u.SSHKeys) + } + _, bad := do(url.Values{"keys": {"not a key"}}) + if !strings.Contains(bad, "Not kept") || !strings.Contains(bad, "not a key") { + t.Error("a bad key was not refused, or the box lost what was typed") + } + if u, _ := st.SlurmUserByID(ctx, ayse.ID); len(u.SSHKeys) != 1 { + t.Error("a refused form changed the keys") + } + // Off: neither the part nor the form. + set.Users = "" + if _, page := do(nil); strings.Contains(page, "ssh-keys") { + t.Error("offered with slurm.users off") + } + if code, _ := do(url.Values{"keys": {key}}); code != http.StatusNotFound { + t.Errorf("posted with slurm.users off: %d", code) + } +} diff --git a/internal/web/templates/account.html b/internal/web/templates/account.html index 11114b7..f1d79ba 100644 --- a/internal/web/templates/account.html +++ b/internal/web/templates/account.html @@ -232,6 +232,21 @@ document.addEventListener("click", (e) => {

{{t "profile.licences.note"}}

{{end}} +{{with .Slurm}} +

{{t "profile.slurm"}}

+{{if .Saved}}

{{t "profile.slurm.saved"}}

{{end}} +{{if .Error}}

{{.Error}}

{{end}} +{{if .Login}}

{{t "profile.slurm.intro" .Login .UID .Home}}

+
ssh {{.Login}}@{{.Host}}
+sbatch job.sh
{{end}} +
+ + + +
+

{{t "profile.slurm.note"}}

+{{end}} +

{{t "profile.password"}}

{{if .PasswordChanged}}

{{t "profile.password_changed"}}

{{end}} {{if .PasswordError}}

{{.PasswordError}}

{{end}}