Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

api/gen/llmgw/v1/agent.pb.go

@@ -5317,9 +5317,14 @@ type Slurm struct {
 	MakeHomes bool `protobuf:"varint,11,opt,name=make_homes,json=makeHomes,proto3" json:"make_homes,omitempty"`
 	// containers has the machine install pyxis and enroot, so jobs may run
 	// in containers (srun --container-image), and set them up.
-	Containers    bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"`
-	unknownFields protoimpl.UnknownFields
-	sizeCache     protoimpl.SizeCache
+	Containers bool `protobuf:"varint,12,opt,name=containers,proto3" json:"containers,omitempty"`
+	// manage_accounts has the controller keep Slurm's accounts as accounts
+	// says: one for each organization, team and project, nested, the users
+	// on each project's. Only those llmgw made are changed or taken away.
+	ManageAccounts bool            `protobuf:"varint,13,opt,name=manage_accounts,json=manageAccounts,proto3" json:"manage_accounts,omitempty"`
+	Accounts       []*SlurmAccount `protobuf:"bytes,14,rep,name=accounts,proto3" json:"accounts,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
 }
 
 func (x *Slurm) Reset() {
@@ -5436,6 +5441,199 @@ func (x *Slurm) GetContainers() bool {
 	return false
 }
 
+func (x *Slurm) GetManageAccounts() bool {
+	if x != nil {
+		return x.ManageAccounts
+	}
+	return false
+}
+
+func (x *Slurm) GetAccounts() []*SlurmAccount {
+	if x != nil {
+		return x.Accounts
+	}
+	return nil
+}
+
+// SlurmAccount is an account in Slurm's accounting, parents before their
+// children.
+type SlurmAccount struct {
+	state         protoimpl.MessageState `protogen:"open.v1"`
+	Name          string                 `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Parent        string                 `protobuf:"bytes,2,opt,name=parent,proto3" json:"parent,omitempty"`
+	Users         []string               `protobuf:"bytes,3,rep,name=users,proto3" json:"users,omitempty"`
+	unknownFields protoimpl.UnknownFields
+	sizeCache     protoimpl.SizeCache
+}
+
+func (x *SlurmAccount) Reset() {
+	*x = SlurmAccount{}
+	mi := &file_llmgw_v1_agent_proto_msgTypes[49]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SlurmAccount) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SlurmAccount) ProtoMessage() {}
+
+func (x *SlurmAccount) ProtoReflect() protoreflect.Message {
+	mi := &file_llmgw_v1_agent_proto_msgTypes[49]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SlurmAccount.ProtoReflect.Descriptor instead.
+func (*SlurmAccount) Descriptor() ([]byte, []int) {
+	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{49}
+}
+
+func (x *SlurmAccount) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *SlurmAccount) GetParent() string {
+	if x != nil {
+		return x.Parent
+	}
+	return ""
+}
+
+func (x *SlurmAccount) GetUsers() []string {
+	if x != nil {
+		return x.Users
+	}
+	return nil
+}
+
+// SlurmJob is a job that ended, as sacct has it, for llmgw's accounting.
+type SlurmJob struct {
+	state          protoimpl.MessageState `protogen:"open.v1"`
+	Id             string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	User           string                 `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"`
+	Account        string                 `protobuf:"bytes,3,opt,name=account,proto3" json:"account,omitempty"`
+	Partition      string                 `protobuf:"bytes,4,opt,name=partition,proto3" json:"partition,omitempty"`
+	State          string                 `protobuf:"bytes,5,opt,name=state,proto3" json:"state,omitempty"`
+	StartUnix      int64                  `protobuf:"varint,6,opt,name=start_unix,json=startUnix,proto3" json:"start_unix,omitempty"`
+	EndUnix        int64                  `protobuf:"varint,7,opt,name=end_unix,json=endUnix,proto3" json:"end_unix,omitempty"`
+	ElapsedSeconds int64                  `protobuf:"varint,8,opt,name=elapsed_seconds,json=elapsedSeconds,proto3" json:"elapsed_seconds,omitempty"`
+	Cpus           int32                  `protobuf:"varint,9,opt,name=cpus,proto3" json:"cpus,omitempty"`
+	Gpus           int32                  `protobuf:"varint,10,opt,name=gpus,proto3" json:"gpus,omitempty"`
+	unknownFields  protoimpl.UnknownFields
+	sizeCache      protoimpl.SizeCache
+}
+
+func (x *SlurmJob) Reset() {
+	*x = SlurmJob{}
+	mi := &file_llmgw_v1_agent_proto_msgTypes[50]
+	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+	ms.StoreMessageInfo(mi)
+}
+
+func (x *SlurmJob) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*SlurmJob) ProtoMessage() {}
+
+func (x *SlurmJob) ProtoReflect() protoreflect.Message {
+	mi := &file_llmgw_v1_agent_proto_msgTypes[50]
+	if x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use SlurmJob.ProtoReflect.Descriptor instead.
+func (*SlurmJob) Descriptor() ([]byte, []int) {
+	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{50}
+}
+
+func (x *SlurmJob) GetId() string {
+	if x != nil {
+		return x.Id
+	}
+	return ""
+}
+
+func (x *SlurmJob) GetUser() string {
+	if x != nil {
+		return x.User
+	}
+	return ""
+}
+
+func (x *SlurmJob) GetAccount() string {
+	if x != nil {
+		return x.Account
+	}
+	return ""
+}
+
+func (x *SlurmJob) GetPartition() string {
+	if x != nil {
+		return x.Partition
+	}
+	return ""
+}
+
+func (x *SlurmJob) GetState() string {
+	if x != nil {
+		return x.State
+	}
+	return ""
+}
+
+func (x *SlurmJob) GetStartUnix() int64 {
+	if x != nil {
+		return x.StartUnix
+	}
+	return 0
+}
+
+func (x *SlurmJob) GetEndUnix() int64 {
+	if x != nil {
+		return x.EndUnix
+	}
+	return 0
+}
+
+func (x *SlurmJob) GetElapsedSeconds() int64 {
+	if x != nil {
+		return x.ElapsedSeconds
+	}
+	return 0
+}
+
+func (x *SlurmJob) GetCpus() int32 {
+	if x != nil {
+		return x.Cpus
+	}
+	return 0
+}
+
+func (x *SlurmJob) GetGpus() int32 {
+	if x != nil {
+		return x.Gpus
+	}
+	return 0
+}
+
 // SlurmShared is where the Slurm cluster's shared files come from.
 type SlurmShared struct {
 	state protoimpl.MessageState `protogen:"open.v1"`
@@ -5454,7 +5652,7 @@ type SlurmShared struct {
 
 func (x *SlurmShared) Reset() {
 	*x = SlurmShared{}
-	mi := &file_llmgw_v1_agent_proto_msgTypes[49]
+	mi := &file_llmgw_v1_agent_proto_msgTypes[51]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -5466,7 +5664,7 @@ func (x *SlurmShared) String() string {
 func (*SlurmShared) ProtoMessage() {}
 
 func (x *SlurmShared) ProtoReflect() protoreflect.Message {
-	mi := &file_llmgw_v1_agent_proto_msgTypes[49]
+	mi := &file_llmgw_v1_agent_proto_msgTypes[51]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -5479,7 +5677,7 @@ func (x *SlurmShared) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use SlurmShared.ProtoReflect.Descriptor instead.
 func (*SlurmShared) Descriptor() ([]byte, []int) {
-	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{49}
+	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{51}
 }
 
 func (x *SlurmShared) GetPath() string {
@@ -5527,7 +5725,7 @@ type SlurmUser struct {
 
 func (x *SlurmUser) Reset() {
 	*x = SlurmUser{}
-	mi := &file_llmgw_v1_agent_proto_msgTypes[50]
+	mi := &file_llmgw_v1_agent_proto_msgTypes[52]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -5539,7 +5737,7 @@ func (x *SlurmUser) String() string {
 func (*SlurmUser) ProtoMessage() {}
 
 func (x *SlurmUser) ProtoReflect() protoreflect.Message {
-	mi := &file_llmgw_v1_agent_proto_msgTypes[50]
+	mi := &file_llmgw_v1_agent_proto_msgTypes[52]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -5552,7 +5750,7 @@ func (x *SlurmUser) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use SlurmUser.ProtoReflect.Descriptor instead.
 func (*SlurmUser) Descriptor() ([]byte, []int) {
-	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{50}
+	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{52}
 }
 
 func (x *SlurmUser) GetLogin() string {
@@ -5623,14 +5821,17 @@ type SlurmStatus struct {
 	Users int32 `protobuf:"varint,12,opt,name=users,proto3" json:"users,omitempty"`
 	// containers is what this machine has for jobs in containers, as
 	// "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
-	Containers    string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"`
+	Containers string `protobuf:"bytes,13,opt,name=containers,proto3" json:"containers,omitempty"`
+	// ended are, on the controller, the jobs that ended in the last day; the
+	// leader records each once.
+	Ended         []*SlurmJob `protobuf:"bytes,14,rep,name=ended,proto3" json:"ended,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
 func (x *SlurmStatus) Reset() {
 	*x = SlurmStatus{}
-	mi := &file_llmgw_v1_agent_proto_msgTypes[51]
+	mi := &file_llmgw_v1_agent_proto_msgTypes[53]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
@@ -5642,7 +5843,7 @@ func (x *SlurmStatus) String() string {
 func (*SlurmStatus) ProtoMessage() {}
 
 func (x *SlurmStatus) ProtoReflect() protoreflect.Message {
-	mi := &file_llmgw_v1_agent_proto_msgTypes[51]
+	mi := &file_llmgw_v1_agent_proto_msgTypes[53]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -5655,7 +5856,7 @@ func (x *SlurmStatus) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use SlurmStatus.ProtoReflect.Descriptor instead.
 func (*SlurmStatus) Descriptor() ([]byte, []int) {
-	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{51}
+	return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{53}
 }
 
 func (x *SlurmStatus) GetVersion() string {
@@ -5749,6 +5950,13 @@ func (x *SlurmStatus) GetContainers() string {
 	return ""
 }
 
+func (x *SlurmStatus) GetEnded() []*SlurmJob {
+	if x != nil {
+		return x.Ended
+	}
+	return nil
+}
+
 var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
 
 const file_llmgw_v1_agent_proto_rawDesc = "" +
@@ -6254,7 +6462,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" +
 	"\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" +
 	"\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" +
-	"\x04last\x18\x05 \x01(\bR\x04last\"\xc9\x03\n" +
+	"\x04last\x18\x05 \x01(\bR\x04last\"\xa6\x04\n" +
 	"\x05Slurm\x12\x1e\n" +
 	"\n" +
 	"controller\x18\x01 \x01(\bR\n" +
@@ -6274,11 +6482,30 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"make_homes\x18\v \x01(\bR\tmakeHomes\x12\x1e\n" +
 	"\n" +
 	"containers\x18\f \x01(\bR\n" +
-	"containers\x1a8\n" +
+	"containers\x12'\n" +
+	"\x0fmanage_accounts\x18\r \x01(\bR\x0emanageAccounts\x122\n" +
+	"\baccounts\x18\x0e \x03(\v2\x16.llmgw.v1.SlurmAccountR\baccounts\x1a8\n" +
 	"\n" +
 	"FilesEntry\x12\x10\n" +
 	"\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" +
-	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"c\n" +
+	"\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"P\n" +
+	"\fSlurmAccount\x12\x12\n" +
+	"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
+	"\x06parent\x18\x02 \x01(\tR\x06parent\x12\x14\n" +
+	"\x05users\x18\x03 \x03(\tR\x05users\"\x87\x02\n" +
+	"\bSlurmJob\x12\x0e\n" +
+	"\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" +
+	"\x04user\x18\x02 \x01(\tR\x04user\x12\x18\n" +
+	"\aaccount\x18\x03 \x01(\tR\aaccount\x12\x1c\n" +
+	"\tpartition\x18\x04 \x01(\tR\tpartition\x12\x14\n" +
+	"\x05state\x18\x05 \x01(\tR\x05state\x12\x1d\n" +
+	"\n" +
+	"start_unix\x18\x06 \x01(\x03R\tstartUnix\x12\x19\n" +
+	"\bend_unix\x18\a \x01(\x03R\aendUnix\x12'\n" +
+	"\x0felapsed_seconds\x18\b \x01(\x03R\x0eelapsedSeconds\x12\x12\n" +
+	"\x04cpus\x18\t \x01(\x05R\x04cpus\x12\x12\n" +
+	"\x04gpus\x18\n" +
+	" \x01(\x05R\x04gpus\"c\n" +
 	"\vSlurmShared\x12\x12\n" +
 	"\x04path\x18\x01 \x01(\tR\x04path\x12\x10\n" +
 	"\x03nfs\x18\x02 \x01(\tR\x03nfs\x12\x14\n" +
@@ -6289,7 +6516,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x03uid\x18\x02 \x01(\rR\x03uid\x12\x12\n" +
 	"\x04name\x18\x03 \x01(\tR\x04name\x12\x12\n" +
 	"\x04home\x18\x04 \x01(\tR\x04home\x12\x19\n" +
-	"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\x8b\x03\n" +
+	"\bssh_keys\x18\x05 \x03(\tR\asshKeys\"\xb5\x03\n" +
 	"\vSlurmStatus\x12\x18\n" +
 	"\aversion\x18\x01 \x01(\tR\aversion\x12\x1a\n" +
 	"\bhardware\x18\x02 \x01(\tR\bhardware\x12%\n" +
@@ -6306,7 +6533,8 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
 	"\x05users\x18\f \x01(\x05R\x05users\x12\x1e\n" +
 	"\n" +
 	"containers\x18\r \x01(\tR\n" +
-	"containers2O\n" +
+	"containers\x12(\n" +
+	"\x05ended\x18\x0e \x03(\v2\x12.llmgw.v1.SlurmJobR\x05ended2O\n" +
 	"\fAgentService\x12?\n" +
 	"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
 	"\fcom.llmgw.v1B\n" +
@@ -6325,7 +6553,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
 }
 
 var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
-var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 61)
+var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 63)
 var file_llmgw_v1_agent_proto_goTypes = []any{
 	(ShardStatus_State)(0),    // 0: llmgw.v1.ShardStatus.State
 	(*AgentMessage)(nil),      // 1: llmgw.v1.AgentMessage
@@ -6377,18 +6605,20 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
 	(*TrainStatus)(nil),       // 47: llmgw.v1.TrainStatus
 	(*TrainArtifact)(nil),     // 48: llmgw.v1.TrainArtifact
 	(*Slurm)(nil),             // 49: llmgw.v1.Slurm
-	(*SlurmShared)(nil),       // 50: llmgw.v1.SlurmShared
-	(*SlurmUser)(nil),         // 51: llmgw.v1.SlurmUser
-	(*SlurmStatus)(nil),       // 52: llmgw.v1.SlurmStatus
-	nil,                       // 53: llmgw.v1.ControlMessage.TraceEntry
-	nil,                       // 54: llmgw.v1.Register.LabelsEntry
-	nil,                       // 55: llmgw.v1.Register.EngineVersionsEntry
-	nil,                       // 56: llmgw.v1.GPU.NicPathsEntry
-	nil,                       // 57: llmgw.v1.IBPort.CountersEntry
-	nil,                       // 58: llmgw.v1.Heartbeat.EngineVersionsEntry
-	nil,                       // 59: llmgw.v1.EngineWant.OfflineSha256Entry
-	nil,                       // 60: llmgw.v1.EngineProfile.EnvEntry
-	nil,                       // 61: llmgw.v1.Slurm.FilesEntry
+	(*SlurmAccount)(nil),      // 50: llmgw.v1.SlurmAccount
+	(*SlurmJob)(nil),          // 51: llmgw.v1.SlurmJob
+	(*SlurmShared)(nil),       // 52: llmgw.v1.SlurmShared
+	(*SlurmUser)(nil),         // 53: llmgw.v1.SlurmUser
+	(*SlurmStatus)(nil),       // 54: llmgw.v1.SlurmStatus
+	nil,                       // 55: llmgw.v1.ControlMessage.TraceEntry
+	nil,                       // 56: llmgw.v1.Register.LabelsEntry
+	nil,                       // 57: llmgw.v1.Register.EngineVersionsEntry
+	nil,                       // 58: llmgw.v1.GPU.NicPathsEntry
+	nil,                       // 59: llmgw.v1.IBPort.CountersEntry
+	nil,                       // 60: llmgw.v1.Heartbeat.EngineVersionsEntry
+	nil,                       // 61: llmgw.v1.EngineWant.OfflineSha256Entry
+	nil,                       // 62: llmgw.v1.EngineProfile.EnvEntry
+	nil,                       // 63: llmgw.v1.Slurm.FilesEntry
 }
 var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	8,  // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
@@ -6401,7 +6631,7 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	2,  // 7: llmgw.v1.AgentMessage.kv_events:type_name -> llmgw.v1.KVEvents
 	47, // 8: llmgw.v1.AgentMessage.train_status:type_name -> llmgw.v1.TrainStatus
 	48, // 9: llmgw.v1.AgentMessage.train_artifact:type_name -> llmgw.v1.TrainArtifact
-	52, // 10: llmgw.v1.AgentMessage.slurm_status:type_name -> llmgw.v1.SlurmStatus
+	54, // 10: llmgw.v1.AgentMessage.slurm_status:type_name -> llmgw.v1.SlurmStatus
 	30, // 11: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
 	33, // 12: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
 	34, // 13: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
@@ -6419,11 +6649,11 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	45, // 25: llmgw.v1.ControlMessage.train_start:type_name -> llmgw.v1.TrainStart
 	46, // 26: llmgw.v1.ControlMessage.train_cancel:type_name -> llmgw.v1.TrainCancel
 	49, // 27: llmgw.v1.ControlMessage.slurm:type_name -> llmgw.v1.Slurm
-	53, // 28: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry
-	54, // 29: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
+	55, // 28: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry
+	56, // 29: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
 	12, // 30: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
 	14, // 31: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
-	55, // 32: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
+	57, // 32: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
 	11, // 33: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
 	15, // 34: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
 	10, // 35: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
@@ -6431,15 +6661,15 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	21, // 37: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
 	0,  // 38: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
 	27, // 39: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute
-	56, // 40: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
+	58, // 40: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
 	13, // 41: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice
 	18, // 42: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
 	17, // 43: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
-	57, // 44: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
+	59, // 44: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
 	20, // 45: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
 	10, // 46: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
 	15, // 47: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
-	58, // 48: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
+	60, // 48: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
 	26, // 49: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
 	16, // 50: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
 	19, // 51: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
@@ -6447,23 +6677,25 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
 	22, // 53: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
 	21, // 54: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
 	25, // 55: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
-	59, // 56: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry
+	61, // 56: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry
 	0,  // 57: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
 	29, // 58: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
 	27, // 59: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute
 	31, // 60: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
-	60, // 61: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
+	62, // 61: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
 	44, // 62: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
-	61, // 63: llmgw.v1.Slurm.files:type_name -> llmgw.v1.Slurm.FilesEntry
-	50, // 64: llmgw.v1.Slurm.shared:type_name -> llmgw.v1.SlurmShared
-	51, // 65: llmgw.v1.Slurm.users:type_name -> llmgw.v1.SlurmUser
-	1,  // 66: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
-	3,  // 67: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
-	67, // [67:68] is the sub-list for method output_type
-	66, // [66:67] is the sub-list for method input_type
-	66, // [66:66] is the sub-list for extension type_name
-	66, // [66:66] is the sub-list for extension extendee
-	0,  // [0:66] is the sub-list for field type_name
+	63, // 63: llmgw.v1.Slurm.files:type_name -> llmgw.v1.Slurm.FilesEntry
+	52, // 64: llmgw.v1.Slurm.shared:type_name -> llmgw.v1.SlurmShared
+	53, // 65: llmgw.v1.Slurm.users:type_name -> llmgw.v1.SlurmUser
+	50, // 66: llmgw.v1.Slurm.accounts:type_name -> llmgw.v1.SlurmAccount
+	51, // 67: llmgw.v1.SlurmStatus.ended:type_name -> llmgw.v1.SlurmJob
+	1,  // 68: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
+	3,  // 69: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
+	69, // [69:70] is the sub-list for method output_type
+	68, // [68:69] is the sub-list for method input_type
+	68, // [68:68] is the sub-list for extension type_name
+	68, // [68:68] is the sub-list for extension extendee
+	0,  // [0:68] is the sub-list for field type_name
 }
 
 func init() { file_llmgw_v1_agent_proto_init() }
@@ -6509,7 +6741,7 @@ func file_llmgw_v1_agent_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
 			NumEnums:      1,
-			NumMessages:   61,
+			NumMessages:   63,
 			NumExtensions: 0,
 			NumServices:   1,
 		},

api/proto/llmgw/v1/agent.proto

@@ -1023,6 +1023,33 @@ message Slurm {
   // containers has the machine install pyxis and enroot, so jobs may run
   // in containers (srun --container-image), and set them up.
   bool containers = 12;
+  // manage_accounts has the controller keep Slurm's accounts as accounts
+  // says: one for each organization, team and project, nested, the users
+  // on each project's. Only those llmgw made are changed or taken away.
+  bool manage_accounts = 13;
+  repeated SlurmAccount accounts = 14;
+}
+
+// SlurmAccount is an account in Slurm's accounting, parents before their
+// children.
+message SlurmAccount {
+  string name = 1;
+  string parent = 2;
+  repeated string users = 3;
+}
+
+// SlurmJob is a job that ended, as sacct has it, for llmgw's accounting.
+message SlurmJob {
+  string id = 1;
+  string user = 2;
+  string account = 3;
+  string partition = 4;
+  string state = 5;
+  int64 start_unix = 6;
+  int64 end_unix = 7;
+  int64 elapsed_seconds = 8;
+  int32 cpus = 9;
+  int32 gpus = 10;
 }
 
 // SlurmShared is where the Slurm cluster's shared files come from.
@@ -1083,4 +1110,7 @@ message SlurmStatus {
   // containers is what this machine has for jobs in containers, as
   // "pyxis 0.24.0, enroot 4.2.1"; empty while it has not got both.
   string containers = 13;
+  // ended are, on the controller, the jobs that ended in the last day; the
+  // leader records each once.
+  repeated SlurmJob ended = 14;
 }