CPU replicas start on half the machine's CPUs, not all of them: two busy llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece, where at 4 threads they keep 5.5. llamacpp.threads sets it per node. Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with admission, schedule and upstream spans and traceparent sent to the engine. Security: onboarding no longer puts the cluster root in the system trust store and removes it where it was; the mirror's mutual-TLS API can be limited to named certificates (--api-client); init.log no longer keeps the administrator's password; a warning when auth is off on a public listen; the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and the remaining engine gaps are documented. reconciler.go and agent.go are split by topic, the status mapping of each failure is tested, and the plan's checklist is brought up to date. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
114 lines
3.2 KiB
Go
114 lines
3.2 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"time"
|
|
)
|
|
|
|
// retire finishes restarting replicas.
|
|
//
|
|
// A retiring replica keeps serving until its replacement is ready, and is then
|
|
// drained like any other. When no replacement could be placed at all —
|
|
// nowhere else has the memory, which on a small CPU fleet is the usual case —
|
|
// it is stopped after DrainGrace so its replacement can take its place: a
|
|
// restart in place, with the model unserved while it loads if it had no other
|
|
// replica. Said in the log, since that is the one case with an outage.
|
|
func (r *Reconciler) retire(ctx context.Context, ids []string, readyLive, live, desired int) {
|
|
for _, id := range ids {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
var since time.Time
|
|
if ok {
|
|
since = rep.RetiringAt
|
|
}
|
|
r.mu.Unlock()
|
|
if !ok {
|
|
continue
|
|
}
|
|
switch {
|
|
case readyLive >= desired:
|
|
r.drainForRestart(ctx, id)
|
|
case live < desired && r.now().Sub(since) > r.cfg.DrainGrace:
|
|
r.log.Warn("restarting a replica in place: no room for its replacement beside it",
|
|
"replica", id, "after", r.cfg.DrainGrace)
|
|
r.drainForRestart(ctx, id)
|
|
r.stopReplica(ctx, id)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
|
|
r.mu.Lock()
|
|
rep, ok := r.replicas[id]
|
|
var shards []Shard
|
|
if ok {
|
|
rep.State, rep.DrainingAt, rep.RetiringAt = Draining, r.now(), time.Time{}
|
|
shards = slices.Clone(rep.Shards)
|
|
}
|
|
r.mu.Unlock()
|
|
for _, s := range shards {
|
|
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
|
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// nextRestartLocked pops the next queued replica of a model that is still
|
|
// ready to be restarted. Called with mu held.
|
|
func (r *Reconciler) nextRestartLocked(model string) (string, bool) {
|
|
q := r.restartQueue[model]
|
|
for len(q) > 0 {
|
|
id := q[0]
|
|
q = q[1:]
|
|
if rep, ok := r.replicas[id]; ok && rep.State == Ready && rep.RetiringAt.IsZero() {
|
|
r.restartQueue[model] = q
|
|
return id, true
|
|
}
|
|
}
|
|
delete(r.restartQueue, model)
|
|
return "", false
|
|
}
|
|
|
|
// ErrNoSuchReplica means no ready replica has that id.
|
|
var ErrNoSuchReplica = errors.New("no ready replica with that id")
|
|
|
|
// Restart queues replicas to be replaced, each by a new one started beside it
|
|
// before it is stopped — to pick up an edited profile, an engine key, a new
|
|
// engine version. Replicas of one model are replaced one at a time.
|
|
func (r *Reconciler) Restart(ids ...string) error {
|
|
r.mu.Lock()
|
|
for _, id := range ids {
|
|
rep, ok := r.replicas[id]
|
|
if !ok || rep.State != Ready {
|
|
r.mu.Unlock()
|
|
return fmt.Errorf("%w: %s", ErrNoSuchReplica, id)
|
|
}
|
|
if !slices.Contains(r.restartQueue[rep.Model], id) && rep.RetiringAt.IsZero() {
|
|
r.restartQueue[rep.Model] = append(r.restartQueue[rep.Model], id)
|
|
}
|
|
}
|
|
r.mu.Unlock()
|
|
r.Notify()
|
|
return nil
|
|
}
|
|
|
|
// RestartState says where a replica's restart stands: "queued", "replacing",
|
|
// or "" when none was asked for.
|
|
func (r *Reconciler) RestartState(id string) string {
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
rep, ok := r.replicas[id]
|
|
switch {
|
|
case !ok:
|
|
return ""
|
|
case !rep.RetiringAt.IsZero():
|
|
return "replacing"
|
|
case slices.Contains(r.restartQueue[rep.Model], id):
|
|
return "queued"
|
|
}
|
|
return ""
|
|
}
|