Every setting of a served model from the mirror page, and its replicas per profile
CI / test (push) Successful in 10m51s
CI / test (push) Successful in 10m51s
The row of a served model leads to its card, which now changes its context and longest answer, its cards, its replicas on each profile with a profile on CPUs to add and its speed beside, stops, starts and restarts it, and removes it with its files or without. Stopped on the row stops the model rather than setting its scaling to nothing. The console gains Replicas per profile, and ctl models shows the counts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e23732fbbb
commit
1a3f385be4
15 files changed
+721
-20
No files matched your search
@@ -2,6 +2,8 @@ package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -48,10 +50,29 @@ type PurgeView struct {
|
||||
// purgeModel removes a model from the catalog and its files from the mirror
|
||||
// and the nodes.
|
||||
func (a *API) purgeModel(w http.ResponseWriter, r *http.Request, name string) {
|
||||
out, err := a.Purge(r.Context(), name, actorOf(r))
|
||||
switch {
|
||||
case errors.Is(err, errNoSuchModel):
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
case err != nil:
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
default:
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
}
|
||||
|
||||
// errNoSuchModel is a model the catalog does not have.
|
||||
var errNoSuchModel = errors.New("no such model in the catalog")
|
||||
|
||||
// Purge removes a model from the catalog and its files from the mirror and
|
||||
// the nodes, waiting a while for its replicas to stop before the nodes'.
|
||||
func (a *API) Purge(ctx context.Context, name, by string) (PurgeView, error) {
|
||||
if a.models == nil {
|
||||
return PurgeView{}, errors.New("this control plane has no store")
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "no model "+name+" in the catalog")
|
||||
return
|
||||
return PurgeView{}, fmt.Errorf("%w: %s", errNoSuchModel, name)
|
||||
}
|
||||
name = m.Name
|
||||
others := []catalog.Model{}
|
||||
@@ -60,37 +81,35 @@ func (a *API) purgeModel(w http.ResponseWriter, r *http.Request, name string) {
|
||||
others = append(others, o)
|
||||
}
|
||||
}
|
||||
if err := a.models.Remove(r.Context(), name); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
if err := a.models.Remove(ctx, name); err != nil {
|
||||
return PurgeView{}, err
|
||||
}
|
||||
a.presetDeclined(r.Context(), name, actorOf(r))
|
||||
a.presetDeclined(ctx, name, by)
|
||||
out := PurgeView{Removed: name}
|
||||
|
||||
sources := purgeSources(m)
|
||||
if a.purgeMirror == nil {
|
||||
out.Note = "this control plane has no mirror; its files there, if any, are left"
|
||||
} else {
|
||||
out.Mirror, out.Kept, out.Note = purgeFromMirror(r.Context(), a.purgeMirror, sources, others)
|
||||
out.Mirror, out.Kept, out.Note = purgeFromMirror(ctx, a.purgeMirror, sources, others)
|
||||
}
|
||||
|
||||
// The nodes' copies, once nothing runs on them.
|
||||
deadline := time.Now().Add(purgeWait)
|
||||
for a.runs(name) && time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return out, ctx.Err()
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
if a.runs(name) {
|
||||
out.Note = strings.TrimSpace(out.Note + " Its replicas had not stopped yet: its copies on the nodes are " +
|
||||
"left, and llmgw ctl weights --collect deletes them.")
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
return
|
||||
return out, nil
|
||||
}
|
||||
out.Nodes = a.purgeFromNodes(r.Context(), sources, others)
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
out.Nodes = a.purgeFromNodes(ctx, sources, others)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// runs says whether any replica of a model is still on a node.
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
// What the portal changes of a served model beside what the catalog holds:
|
||||
// the same as the API's routes, called in the process.
|
||||
|
||||
// SetModelContext changes a served model's context and longest answer.
|
||||
func (a *API) SetModelContext(ctx context.Context, name string, req modelserve.ContextRequest, by string) (modelserve.ContextChange, error) {
|
||||
if a.serve == nil {
|
||||
return modelserve.ContextChange{}, errors.New("this control plane cannot change a model's context")
|
||||
}
|
||||
return a.serve.SetContext(ctx, name, req, by)
|
||||
}
|
||||
|
||||
// SetModelGPUs spreads a served model over more cards a replica, or fewer.
|
||||
func (a *API) SetModelGPUs(ctx context.Context, name string, req modelserve.GPUsRequest, by string) (modelserve.GPUsChange, error) {
|
||||
if a.serve == nil {
|
||||
return modelserve.GPUsChange{}, errors.New("this control plane cannot change a model's cards")
|
||||
}
|
||||
return a.serve.SetGPUs(ctx, name, req, by)
|
||||
}
|
||||
|
||||
// RestartModel starts each of a model's ready replicas again, one at a time,
|
||||
// and says how many.
|
||||
func (a *API) RestartModel(name string) (int, error) {
|
||||
var ids []string
|
||||
for _, rep := range a.ctl.Reconciler().ReadyReplicas() {
|
||||
if rep.Model == name {
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return 0, errors.New("none of its replicas is ready to restart")
|
||||
}
|
||||
return len(ids), a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...)
|
||||
}
|
||||
|
||||
// RemoveModel takes a model out of the catalog, with its files on the mirror
|
||||
// and the nodes when purge.
|
||||
func (a *API) RemoveModel(ctx context.Context, name string, purge bool, by string) (PurgeView, error) {
|
||||
if purge {
|
||||
return a.Purge(ctx, name, by)
|
||||
}
|
||||
if a.models == nil {
|
||||
return PurgeView{}, errors.New("this control plane has no store")
|
||||
}
|
||||
if err := a.models.Remove(ctx, name); err != nil {
|
||||
return PurgeView{}, err
|
||||
}
|
||||
a.presetDeclined(ctx, name, by)
|
||||
return PurgeView{Removed: name}, nil
|
||||
}
|
||||
Reference in new issue
Block a user