Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
@@ -126,6 +127,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
|
||||
mux.HandleFunc("GET /api/v1/plan", a.planModel)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -1544,6 +1546,56 @@ func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision})
|
||||
}
|
||||
|
||||
// PoolNodes is the fleet's ready nodes as a plan sees them: their cards,
|
||||
// which of them are free, the fabric they share, and their memory.
|
||||
func (a *API) PoolNodes() []modelplan.PoolNode {
|
||||
var out []modelplan.PoolNode
|
||||
for _, n := range a.Nodes() {
|
||||
if n.State != "ready" {
|
||||
continue
|
||||
}
|
||||
p := modelplan.PoolNode{ID: n.ID, GPUs: n.GPUTotal, FreeGPUs: n.GPUFree, HostMemGB: n.MemoryGB,
|
||||
HostFreeGB: n.MemoryGB}
|
||||
if len(n.Classes) > 0 {
|
||||
p.Class = n.Classes[0]
|
||||
p.GPUMemGB = modelplan.MemFromClass(p.Class)
|
||||
}
|
||||
if len(n.Fabrics) > 0 {
|
||||
p.Fabric = n.Fabrics[0]
|
||||
}
|
||||
if l := n.Load; l != nil && l.MemTotal > 0 {
|
||||
p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30)
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *API) planModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented, "planning needs a mirror and its registry, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
q := r.URL.Query()
|
||||
req := modelserve.PlanRequest{Repo: q.Get("repo"), Format: q.Get("format")}
|
||||
if c := q.Get("context"); c != "" {
|
||||
n, err := strconv.Atoi(c)
|
||||
if err != nil || n < 0 {
|
||||
writeError(w, http.StatusBadRequest, "context is a whole number of tokens")
|
||||
return
|
||||
}
|
||||
req.Context = n
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
defer cancel()
|
||||
p, err := a.serve.Plan(ctx, req)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, p)
|
||||
}
|
||||
|
||||
// actorOf is who a change is recorded as made by: what the caller says, since
|
||||
// the admin API answers only this machine and trusts whoever reaches it. Kept
|
||||
// to a short printable name, so a claim cannot pass for more than one — no
|
||||
|
||||
Reference in new issue
Block a user