Ilker ManapandClaude Opus 5.5
141a1f46c5
CI / test (push) Successful in 10m50s
Tried on the PoC: with a job on a lent node, the leader restarted and its first slurm.conf had no node in it (none had reported to it yet), so slurmctld dropped the node and the job ended in NODE_FAIL. The leader keeps slurm.conf's nodes and the controller's address in the store's cluster state (slurm-nodes); the next one starts from them and writes the same files until the nodes report again. With the store unread, the controller gets no files that leave a node out. A node is told nothing, rather than that it has no part, until its controller's address is known: no part stops slurmd, and its jobs with it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
78 lines
2.0 KiB
Go
78 lines
2.0 KiB
Go
package slurmsvc
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Memory keeps the nodes slurm.conf last held, and the controller's
|
|
// address, where a leader that takes over finds them. Without it a new
|
|
// leader's first slurm.conf has none of the nodes it has not heard from
|
|
// yet, and the controller, restarted on it, drops them and the jobs
|
|
// running there.
|
|
type Memory interface {
|
|
Recall(ctx context.Context) (Kept, error)
|
|
Keep(ctx context.Context, k Kept) error
|
|
}
|
|
|
|
// Kept is what Memory holds.
|
|
type Kept struct {
|
|
Controller string `json:"controller,omitempty"`
|
|
ControllerAddr string `json:"controller_addr,omitempty"`
|
|
Nodes map[string]slurm.Node `json:"nodes,omitempty"`
|
|
}
|
|
|
|
func (k Kept) equal(o Kept) bool {
|
|
if k.Controller != o.Controller || k.ControllerAddr != o.ControllerAddr || len(k.Nodes) != len(o.Nodes) {
|
|
return false
|
|
}
|
|
for name, n := range k.Nodes {
|
|
m, ok := o.Nodes[name]
|
|
if !ok || n.Addr != m.Addr || n.Hardware != m.Hardware || !slices.Equal(n.GPUs, m.GPUs) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// KV is the store's cluster state.
|
|
type KV interface {
|
|
PutClusterState(ctx context.Context, key, value string) (bool, error)
|
|
ClusterState(ctx context.Context, key string) (string, error)
|
|
}
|
|
|
|
// keptKey is the cluster state key it is kept under.
|
|
const keptKey = "slurm-nodes"
|
|
|
|
// StoreMemory keeps it in the store's cluster state.
|
|
func StoreMemory(kv KV) Memory { return kvMemory{kv} }
|
|
|
|
type kvMemory struct{ kv KV }
|
|
|
|
func (m kvMemory) Recall(ctx context.Context) (Kept, error) {
|
|
var k Kept
|
|
v, err := m.kv.ClusterState(ctx, keptKey)
|
|
if errors.Is(err, store.ErrNotFound) {
|
|
return k, nil
|
|
}
|
|
if err != nil {
|
|
return k, err
|
|
}
|
|
err = json.Unmarshal([]byte(v), &k)
|
|
return k, err
|
|
}
|
|
|
|
func (m kvMemory) Keep(ctx context.Context, k Kept) error {
|
|
b, err := json.Marshal(k)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = m.kv.PutClusterState(ctx, keptKey, string(b))
|
|
return err
|
|
}
|