server init: the firewall and the judge before the starter model, and a model that fits nowhere said within two minutes
A starter model with no room failed init after ten minutes and left the firewall and the judge uninstalled. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ddb0472f9b
commit
07af03aa93
4 files changed
+79
-16
No files matched your search
@@ -572,14 +572,40 @@ func (in *serverInit) ensureServed(ctx context.Context, name, source, version st
|
||||
return in.waitServing(ctx, name)
|
||||
}
|
||||
|
||||
// unplacedGrace is how long init lets placement say a model fits nowhere
|
||||
// before believing it: long enough for a node still installing its engine.
|
||||
var unplacedGrace = 90 * time.Second
|
||||
|
||||
func (in *serverInit) waitServing(ctx context.Context, name string) error {
|
||||
client := in.adminAPI()
|
||||
last := ""
|
||||
var unplaced time.Time
|
||||
return waitFor(ctx, 10*time.Minute, name+" to start", func() (bool, error) {
|
||||
var reps []adminapi.ReplicaView
|
||||
if err := client.get(ctx, "/api/v1/replicas?model="+name, &reps); err != nil {
|
||||
return false, nil
|
||||
}
|
||||
// Nowhere to put it is not waited out for ten minutes: once
|
||||
// placement has said why for unplacedGrace, that is the answer.
|
||||
if len(reps) == 0 {
|
||||
var ex adminapi.ExplainView
|
||||
if err := client.get(ctx, "/api/v1/explain-placement?model="+name, &ex); err == nil &&
|
||||
!ex.Placed && len(ex.Rejections) > 0 {
|
||||
if unplaced.IsZero() {
|
||||
unplaced = time.Now()
|
||||
}
|
||||
if time.Since(unplaced) > unplacedGrace {
|
||||
var why []string
|
||||
for _, r := range ex.Rejections {
|
||||
why = append(why, r.Node+": "+r.Reason)
|
||||
}
|
||||
return false, fmt.Errorf("%s has nowhere to run (%s); add a node with room, "+
|
||||
"or free some ('llmgw ctl explain-placement %s')", name, strings.Join(why, "; "), name)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
unplaced = time.Time{}
|
||||
for _, r := range reps {
|
||||
if r.State == "ready" {
|
||||
in.kept("%s is serving (profile %s)", name, r.Profile)
|
||||
|
||||
Reference in new issue
Block a user