How long an on-demand model holds a card is a live setting
On one card shared by flux and sf3d, a mesh asked for after a picture waited out the picture model's five minutes of protection and two of idle. on_demand.evict_after_idle and on_demand.min_lifetime set those for the cluster, in force on the leader as they change; unset keeps 2m and 5m. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a9a2360f44
commit
1e7086e13e
13 files changed
+139
-7
No files matched your search
@@ -245,6 +245,8 @@ var liveSettings = []string{
|
||||
"agents",
|
||||
// Read by the leader's presets pass each time it runs.
|
||||
"presets",
|
||||
// Handed to the reconciler as they change.
|
||||
"on_demand",
|
||||
// Read by server init, node add and backups, not by the server.
|
||||
"backup_recipients", "backup_copy_to",
|
||||
}
|
||||
|
||||
@@ -264,6 +264,7 @@ func newServerCmd() *cobra.Command {
|
||||
agentSvc = svc
|
||||
// Stopped by the operators (llmgw stop): nothing placed.
|
||||
ctl.Reconciler().SetStopped(cfg.Serving.Stopped)
|
||||
ctl.Reconciler().SetEvictionTimes(cfg.OnDemand.EvictAfterIdle, cfg.OnDemand.MinLifetime)
|
||||
// Whether nodes keep llmgw's firewall, as the settings say now.
|
||||
svc.SetFirewall(func() *llmgwv1.Firewall {
|
||||
c := current.Load()
|
||||
@@ -778,6 +779,9 @@ func newServerCmd() *cobra.Command {
|
||||
fwPub.Kick()
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "on_demand") }) && admin != nil {
|
||||
admin.Reconciler().SetEvictionTimes(next.OnDemand.EvictAfterIdle, next.OnDemand.MinLifetime)
|
||||
}
|
||||
if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "agents") }) && agentSvc != nil {
|
||||
// A moment apart per node, so off this goroutine.
|
||||
go agentSvc.BroadcastAgentSettings()
|
||||
|
||||
Reference in new issue
Block a user