How long an on-demand model holds a card is a live setting

On one card shared by flux and sf3d, a mesh asked for after a picture
waited out the picture model's five minutes of protection and two of idle.
on_demand.evict_after_idle and on_demand.min_lifetime set those for the
cluster, in force on the leader as they change; unset keeps 2m and 5m.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-14 08:58:38 +02:00
1 parent a9a2360f44
commit 1e7086e13e
13 files changed
+139 -7

No files matched your search

+4
View File
@@ -264,6 +264,7 @@ func newServerCmd() *cobra.Command {
agentSvc = svc
// Stopped by the operators (llmgw stop): nothing placed.
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
ctl.Reconciler().SetEvictionTimes(cfg.OnDemand.EvictAfterIdle, cfg.OnDemand.MinLifetime)
// Whether nodes keep llmgw's firewall, as the settings say now.
svc.SetFirewall(func() *llmgwv1.Firewall {
c := current.Load()
@@ -778,6 +779,9 @@ func newServerCmd() *cobra.Command {
fwPub.Kick()
}
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "on_demand") }) && admin != nil {
admin.Reconciler().SetEvictionTimes(next.OnDemand.EvictAfterIdle, next.OnDemand.MinLifetime)
}
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "agents") }) && agentSvc != nil {
// A moment apart per node, so off this goroutine.
go agentSvc.BroadcastAgentSettings()