package slurmsvc import ( "context" "encoding/json" "errors" "slices" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Memory keeps the nodes slurm.conf last held, and the controller's // address, where a leader that takes over finds them. Without it a new // leader's first slurm.conf has none of the nodes it has not heard from // yet, and the controller, restarted on it, drops them and the jobs // running there. type Memory interface { Recall(ctx context.Context) (Kept, error) Keep(ctx context.Context, k Kept) error } // Kept is what Memory holds. type Kept struct { Controller string `json:"controller,omitempty"` ControllerAddr string `json:"controller_addr,omitempty"` Nodes map[string]slurm.Node `json:"nodes,omitempty"` } func (k Kept) equal(o Kept) bool { if k.Controller != o.Controller || k.ControllerAddr != o.ControllerAddr || len(k.Nodes) != len(o.Nodes) { return false } for name, n := range k.Nodes { m, ok := o.Nodes[name] if !ok || n.Addr != m.Addr || n.Hardware != m.Hardware || !slices.Equal(n.GPUs, m.GPUs) { return false } } return true } // KV is the store's cluster state. type KV interface { PutClusterState(ctx context.Context, key, value string) (bool, error) ClusterState(ctx context.Context, key string) (string, error) } // keptKey is the cluster state key it is kept under. const keptKey = "slurm-nodes" // StoreMemory keeps it in the store's cluster state. func StoreMemory(kv KV) Memory { return kvMemory{kv} } type kvMemory struct{ kv KV } func (m kvMemory) Recall(ctx context.Context) (Kept, error) { var k Kept v, err := m.kv.ClusterState(ctx, keptKey) if errors.Is(err, store.ErrNotFound) { return k, nil } if err != nil { return k, err } err = json.Unmarshal([]byte(v), &k) return k, err } func (m kvMemory) Keep(ctx context.Context, k Kept) error { b, err := json.Marshal(k) if err != nil { return err } _, err = m.kv.PutClusterState(ctx, keptKey, string(b)) return err }