Serve and update models in the background from the console, and follow them under Jobs

The console starts a serve or update job over the admin API and stays,
instead of closing to follow the download. Jobs lists them with their
progress and steps; the sidebar counts those running. ctl model serve and
update take --detach; ctl jobs lists jobs and follows one.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:27:02 +02:00
1 parent d311a3ca4a
commit e12996f7dd
12 files changed
+280 -34

No files matched your search

+9
View File
@@ -76,6 +76,15 @@ is not an update.
A model from `server.yaml` is reported but not changed: its source is edited
there.
### Serving and updating run in the background
Serving a model or moving one to new weights from the console starts a job
on the control plane and leaves the console free at once, instead of
closing it to follow a download that can take an hour. A new **Jobs** view
shows each job's progress and steps, and the sidebar how many are running.
`llmgw ctl model serve --detach` and `llmgw ctl model update --detach` do the
same; `llmgw ctl jobs` lists the jobs and `llmgw ctl jobs ID` follows one.
### The file that suits the hardware, and how to spread it
`llmgw ctl model plan REPO` now plans the files the repository actually has
+12 -3
View File
@@ -45,9 +45,12 @@ with its reason, word for word.
Anything that cannot be undone — removing, draining, deleting, revoking —
asks first, and only **Y** goes ahead.
Tasks that take a while and print as they go — serving a model, upgrading
the cluster, a backup, adding a node — close the screen and run in the
terminal, so their output is there to read; Enter brings the console back.
Serving a model and moving one to new weights are jobs on the control plane:
started from here, they run there in the background whatever this screen
does, and the console is free at once. **Jobs** follows them. Upgrading the
cluster, a backup and adding a node change this machine or another over ssh;
those close the screen and run in the terminal, so their output is there to
read, and Enter brings the console back.
## Watch
@@ -119,6 +122,12 @@ their checksums and where each was fetched from, the source a model
definition names for it, or remove it — with a warning when a model still
uses it.
**Jobs** lists the models being served or updated since the control plane
started: what each is doing now — the download's progress, a replica
starting — and whether it ended done, failed, or waiting for room on busy
cards. The entry shows how many are running. Enter on one shows its steps.
`llmgw ctl jobs` is the same, and `llmgw ctl jobs ID` follows one.
**Logs** shows a node's agent log, newest at the bottom, errors and warnings
coloured.
+1
View File
@@ -142,6 +142,7 @@ func newCtlCmd() *cobra.Command {
newRestartCmd(client),
newRetryCmd(client),
newUpgradeCmd(client),
newJobsCmd(client, &asJSON),
newEngineCheckCmd(),
newEngineBuildCmd(client),
newToolCheckCmd(),
+56
View File
@@ -15,6 +15,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
var (
req modelserve.Request
onDemand bool
detach bool
)
c := &cobra.Command{
Use: "serve OWNER/REPO",
@@ -49,6 +50,9 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
return err
}
if detach {
return detached(cmd, job)
}
return followJob(cmd, client, job, "is served")
},
}
@@ -64,6 +68,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
f.IntVar(&req.TP, "tp", 0, "the older name of --gpus")
_ = f.MarkHidden("tp")
f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy")
f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
@@ -110,3 +115,54 @@ func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job,
}
}
}
// detached says where a job started without waiting for it can be followed.
func detached(cmd *cobra.Command, job modelserve.Job) error {
fmt.Fprintf(cmd.OutOrStdout(), "started %s on the control plane\nfollow it: llmgw ctl jobs %s\n", job.ID, job.ID)
return nil
}
func newJobsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
Use: "jobs [JOB]",
Short: "Models being served or updated, or one followed until it ends",
Long: "Serving a model and moving one to new weights run as jobs on the control\n" +
"plane, which carry on whether or not anything watches them; they are\n" +
"kept until it restarts. With a job's id, its steps are shown as they\n" +
"happen until it ends.",
Args: cobra.MaximumNArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 1 {
var job modelserve.Job
if err := client().get(cmd.Context(), "/api/v1/jobs/"+args[0], &job); err != nil {
return err
}
return followJob(cmd, client, job, "is done")
}
var jobs []modelserve.Job
if err := client().get(cmd.Context(), "/api/v1/jobs", &jobs); err != nil {
return err
}
if *asJSON {
return emitJSON(jobs)
}
if len(jobs) == 0 {
fmt.Fprintln(cmd.OutOrStdout(), "no jobs since the control plane started")
return nil
}
w := tabTo(cmd.OutOrStdout())
fmt.Fprintln(w, "JOB\tMODEL\tSTATE\tSTARTED\tNOW")
for _, j := range jobs {
state := j.State
for _, st := range j.Steps {
if st.Name == "placement" && st.State == modelserve.Skipped {
state = "waiting for room"
}
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", j.ID, or(j.Model, j.Request.Repo), state,
ago(j.Started, time.Now()), truncate(jobNow(j), 80))
}
return w.Flush()
},
}
}
+7 -1
View File
@@ -82,7 +82,8 @@ func updateCell(u modelserve.UpdateInfo) string {
}
func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
return &cobra.Command{
var detach bool
c := &cobra.Command{
Use: "update MODEL",
Short: "Move a model to the weights its repository publishes now",
Long: "The new files are fetched into the mirror beside the ones in use and\n" +
@@ -97,9 +98,14 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil {
return err
}
if detach {
return detached(cmd, job)
}
return followJob(cmd, client, job, "runs the new weights")
},
}
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
return c
}
func newModelRollbackCmd(client func() *ctlClient) *cobra.Command {
+4 -2
View File
@@ -712,7 +712,6 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
}
return nil
}
server := "--server=" + m.client.base
var items []menuItem
if u := md.Update; u != nil && u.Available {
items = append(items, menuItem{fmt.Sprintf("Update to %s (%s)…", catalog.ShortRevision(u.Latest), bytesShort(u.Bytes)),
@@ -722,7 +721,10 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
"They are fetched into the mirror beside the ones in use and checked, then the\n"+
"replicas are replaced one at a time. The old files stay, to go back to.",
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")),
yes: func(m *consoleModel) tea.Cmd { return m.runAfter("ctl", server, "model", "update", md.Name) }}
yes: func(m *consoleModel) tea.Cmd {
return m.call("Update "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/update",
map[string]string{}, startedSays)
}}
return nil
}})
}
+35 -6
View File
@@ -9,6 +9,7 @@ import (
"strings"
"sync"
"testing"
"time"
tea "github.com/charmbracelet/bubbletea"
@@ -312,11 +313,18 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
}
}
}
var started modelserve.Request
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo {
_ = json.NewEncoder(w).Encode(plan)
return
}
if r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/serve" {
_ = json.NewDecoder(r.Body).Decode(&started)
w.WriteHeader(http.StatusAccepted)
_ = json.NewEncoder(w).Encode(modelserve.Job{ID: "serve-1", State: modelserve.Running})
return
}
_, _ = w.Write([]byte("[]"))
}))
defer admin.Close()
@@ -351,12 +359,17 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
t.Fatalf("form: %+v", f)
}
f.focus = len(f.fields)
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
act := tm.(consoleModel).action
got := strings.Join(act.targets, " ")
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") ||
!strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") {
t.Fatalf("would run %q %q", act.kind, got)
tm = pressRun(tm, "enter")
// Started on the control plane; the console stays.
if act := tm.(consoleModel).action; act.kind != "" {
t.Fatalf("the console left to run %v", act)
}
if started.Repo != plan.Repo || started.File != "Qwen2.5-3B-Instruct-Q8_0.gguf" || started.GPUClass != "6000-24g" ||
started.GPUsPerNode != 1 || !started.Fallbacks || started.Name != "qwen2.5-3b-instruct" {
t.Fatalf("started %+v", started)
}
if v := tm.View(); !strings.Contains(v, "serve-1") {
t.Errorf("the start was not shown:\n%s", v)
}
}
@@ -463,3 +476,19 @@ func TestEveryScreenShowsTheVersionTopRight(t *testing.T) {
t.Errorf("the manual's top line: %q", first)
}
}
func TestJobsAreFollowedFromTheirOwnView(t *testing.T) {
m, _ := consoleWith(t, cJobs)
m.width = 160
m.data.jobs = []modelserve.Job{{ID: "serve-9", Model: "phi-4", State: modelserve.Running, Started: time.Now(),
Steps: []modelserve.Step{{Name: "registry", State: modelserve.Done, Detail: "phi-4-Q8_0.gguf"},
{Name: "mirror", State: modelserve.Running, Detail: "fetching: 3.1 of 14.6 GB (21%)"}}}}
v := m.View()
if !strings.Contains(v, "Jobs (1)") || !strings.Contains(v, "fetching: 3.1 of 14.6 GB") {
t.Fatalf("%s", v)
}
tm := press(m, "enter")
if v := tm.View(); !strings.Contains(v, "serve-9") || !strings.Contains(v, "phi-4-Q8_0.gguf") {
t.Errorf("steps not shown:\n%s", v)
}
}
+15 -1
View File
@@ -20,6 +20,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/docgen"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
@@ -39,6 +40,7 @@ const (
cUsage
cQuotas
cMirror
cJobs
cLogs
cAccounts
cKeys
@@ -55,7 +57,7 @@ const (
cCount
)
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Logs", "Accounts", "API keys", "Organizations", "Manual",
var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual",
"server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster",
"Restart all replicas", "Back up now", "Leave"}
@@ -71,6 +73,7 @@ type consoleData struct {
nodes []adminapi.NodeView
models []adminapi.ModelView
replicas []adminapi.ReplicaView
jobs []modelserve.Job
err error
}
@@ -221,6 +224,7 @@ func (m consoleModel) fetch() tea.Cmd {
_ = c.get(ctx, "/api/v1/nodes", &d.nodes)
_ = c.get(ctx, "/api/v1/models", &d.models)
_ = c.get(ctx, "/api/v1/replicas", &d.replicas)
_ = c.get(ctx, "/api/v1/jobs", &d.jobs)
return consoleDataMsg(d)
}
}
@@ -448,6 +452,9 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.menuD = m.quotaMenu(m.quotas[m.row])
case cMirror:
m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row])
case cJobs:
m.dialog = "error"
m.dialogTitle, m.dialogText = jobDialog(m.data.jobs[m.row])
case cServe:
return m, m.fetchRepo(m.hf.models[m.row].ID)
case cAccounts:
@@ -831,6 +838,8 @@ func (m consoleModel) rowCount() int {
return len(m.quotas)
case cMirror:
return len(m.mirrorGroups())
case cJobs:
return len(m.data.jobs)
case cServe:
return len(m.hf.models)
case cAccounts:
@@ -871,6 +880,9 @@ func (m consoleModel) View() string {
if i == cSettings {
label = filepathBase(m.file)
}
if n := runningJobs(m.data.jobs); i == cJobs && n > 0 {
label += fmt.Sprintf(" (%d)", n)
}
if i == cSettings && m.doc != nil && m.doc.Changed > 0 {
label += fmt.Sprintf(" (%d)", m.doc.Changed)
}
@@ -1060,6 +1072,8 @@ func (m consoleModel) panel() (title, body, keys string) {
return "Manual", d.list(pw, m.inPanel), keys
case cMirror:
return m.mirrorPanel(pw)
case cJobs:
return m.jobsPanel(pw)
case cServe:
return m.hfPanel(pw)
case cSettings:
+16 -21
View File
@@ -3,6 +3,7 @@ package cli
import (
"context"
"fmt"
"net/http"
"net/url"
"slices"
"strconv"
@@ -174,7 +175,6 @@ func (m consoleModel) repoMenu(p modelserve.Plan) *menuDialog {
}
func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact) {
server := "--server=" + m.client.base
var opts []modelplan.Option
for _, o := range p.Options {
if o.Artifact == a.Name {
@@ -228,33 +228,26 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
if c, err := strconv.Atoi(v[2]); err != nil || c <= 0 {
return nil, fmt.Errorf("the context length is a whole number")
}
args := []string{"ctl", server, "model", "serve", p.Repo, "--name=" + v[0], "--replicas=" + v[1],
"--context-length=" + v[2]}
if a.Format == "safetensors" {
args = append(args, "--format=safetensors")
} else {
args = append(args, "--file="+a.Name)
ctxLen, _ := strconv.Atoi(v[2])
req := modelserve.Request{Repo: p.Repo, Name: v[0], Replicas: n, ContextLength: ctxLen,
Description: v[3], Format: a.Format}
if a.Format == "gguf" {
req.File = a.Name
}
o := opts[slices.Index(where, v[4])]
if o.CPU {
args = append(args, "--cpu")
req.CPU = true
} else {
args = append(args, "--gpu-class="+o.Class, "--gpus="+strconv.Itoa(o.GPUsPerNode),
"--nodes="+strconv.Itoa(max(o.Nodes, 1)))
}
if strings.HasPrefix(v[5], "the next") && len(opts) > 1 {
args = append(args, "--fallbacks")
}
if n == 0 {
args = append(args, "--on-demand")
}
if v[3] != "" {
args = append(args, "--description="+v[3])
req.GPUClass, req.GPUsPerNode, req.Nodes = o.Class, o.GPUsPerNode, max(o.Nodes, 1)
}
req.Fallbacks = strings.HasPrefix(v[5], "the next") && len(opts) > 1
if proj >= 0 && !strings.HasPrefix(v[proj], "none") {
args = append(args, "--projector="+v[proj])
req.Projector = v[proj]
}
return m.runAfter(args...), nil
// Started on the control plane, which runs it whatever happens
// to this screen: the console is free again at once, and Jobs
// shows how far it has got.
return m.call("Serve "+p.Repo, http.MethodPost, "/api/v1/models/serve", req, startedSays), nil
}})
}
@@ -319,6 +312,8 @@ func ageShort(t time.Time) string {
}
d := time.Since(t)
switch {
case d < time.Minute:
return "just now"
case d < time.Hour:
return fmt.Sprintf("%dm ago", int(d.Minutes()))
case d < 48*time.Hour:
+102
View File
@@ -0,0 +1,102 @@
package cli
import (
"encoding/json"
"fmt"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
)
// Jobs: serving a model or moving one to new weights is a job on the control
// plane — a download that can take an hour, then a replica starting. Started
// from the console, it runs there whatever this screen does; the console is
// free at once, and this view follows it.
// startedSays is the result of starting a job.
func startedSays(raw json.RawMessage) string {
var j modelserve.Job
if json.Unmarshal(raw, &j) != nil || j.ID == "" {
return "Started."
}
return fmt.Sprintf("Started %s. It runs on the control plane — leaving this screen does not stop it.\n"+
"Jobs, under Watch, shows how far it has got.", j.ID)
}
// runningJobs counts the jobs still under way.
func runningJobs(jobs []modelserve.Job) int {
n := 0
for _, j := range jobs {
if j.State == modelserve.Running {
n++
}
}
return n
}
func jobState(j modelserve.Job) string {
switch j.State {
case modelserve.Running:
return sWarn.Render("running")
case modelserve.Failed:
return sBad.Render("failed")
}
for _, st := range j.Steps {
if st.Name == "placement" && st.State == modelserve.Skipped {
return sWarn.Render("waiting for room")
}
}
return sOK.Render("done")
}
// jobNow is the step a job is at, in a line.
func jobNow(j modelserve.Job) string {
if j.State == modelserve.Failed {
return j.Error
}
if len(j.Steps) == 0 {
return ""
}
st := j.Steps[len(j.Steps)-1]
return st.Name + ": " + st.Detail
}
func (m consoleModel) jobsPanel(pw int) (string, string, string) {
keys := "→ or Enter open · ↑↓ choose · Q leave"
if m.inPanel {
keys = "↑↓ row · Enter its steps · ← back · Q leave"
}
head := "\n " + sDim.Render("Models being served or updated, on the control plane; kept until it restarts.") + "\n\n"
var rows [][]string
for _, j := range m.data.jobs {
kind := j.Kind
if kind == "" {
kind = "serve"
}
what := j.Model
if what == "" {
what = j.Request.Repo
}
rows = append(rows, []string{ageShort(j.Started), kind, what, jobState(j), jobNow(j)})
}
if len(rows) == 0 {
return "Jobs", head + " " + sDim.Render("None since the control plane started. Serve a model, under Do, starts one."), keys
}
return "Jobs", head + table([]string{"STARTED", "KIND", "MODEL", "STATE", "NOW"}, rows, m.row, pw, m.inPanel), keys
}
// jobDialog is a job's steps, each as it stands.
func jobDialog(j modelserve.Job) (string, string) {
var b strings.Builder
mark := map[string]string{modelserve.Running: sWarn.Render(".."), modelserve.Done: sOK.Render("ok"),
modelserve.Failed: sBad.Render("!!"), modelserve.Skipped: sDim.Render("--")}
for _, st := range j.Steps {
fmt.Fprintf(&b, "%s %-11s %s\n", mark[st.State], st.Name, wrap(st.Detail, 80, strings.Repeat(" ", 15)))
}
if j.Reply != "" {
fmt.Fprintf(&b, "\n%s said %q at %.1f tokens/s\n", j.Model, j.Reply, j.TokensPerSecond)
}
fmt.Fprintf(&b, "\n%s, started %s\n\n%s", j.ID, j.Started.Format(time.DateTime), sDim.Render("any key"))
return j.ID, b.String()
}
+7
View File
@@ -122,6 +122,13 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/restart", a.restart)
mux.HandleFunc("POST /api/v1/models/serve", a.serveModel)
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
mux.HandleFunc("GET /api/v1/jobs", func(w http.ResponseWriter, _ *http.Request) {
out := []modelserve.Job{}
if a.serve != nil {
out = a.serve.Jobs()
}
writeJSON(w, http.StatusOK, out)
})
mux.HandleFunc("GET /api/v1/updates", a.listUpdates)
mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates)
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
+16
View File
@@ -240,6 +240,22 @@ func (s *Server) Start(req Request) (Job, error) {
return s.snapshot(j), nil
}
// Jobs is every job since the control plane started, the newest first.
func (s *Server) Jobs() []Job {
s.mu.Lock()
all := make([]*Job, 0, len(s.jobs))
for _, j := range s.jobs {
all = append(all, j)
}
s.mu.Unlock()
out := make([]Job, 0, len(all))
for _, j := range all {
out = append(out, s.snapshot(j))
}
slices.SortFunc(out, func(a, b Job) int { return b.Started.Compare(a.Started) })
return out
}
// Get returns a job as it stands.
func (s *Server) Get(id string) (Job, bool) {
s.mu.Lock()