From e12996f7ddcf101f47940f644eb46dc836c2987e Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 4 Oct 2026 01:27:02 +0200 Subject: [PATCH] Serve and update models in the background from the console, and follow them under Jobs The console starts a serve or update job over the admin API and stays, instead of closing to follow the download. Jobs lists them with their progress and steps; the sidebar counts those running. ctl model serve and update take --detach; ctl jobs lists jobs and follows one. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 9 ++ docs/console.md | 15 +++- internal/cli/ctl.go | 1 + internal/cli/model_serve.go | 56 ++++++++++++ internal/cli/model_update.go | 8 +- internal/cli/tui_actions.go | 6 +- internal/cli/tui_actions_test.go | 41 +++++++-- internal/cli/tui_console.go | 16 +++- internal/cli/tui_hf.go | 37 ++++---- internal/cli/tui_jobs.go | 102 ++++++++++++++++++++++ internal/control/adminapi/api.go | 7 ++ internal/control/modelserve/modelserve.go | 16 ++++ 12 files changed, 280 insertions(+), 34 deletions(-) create mode 100644 internal/cli/tui_jobs.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 231f199a..07dbef25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,15 @@ is not an update. A model from `server.yaml` is reported but not changed: its source is edited there. +### Serving and updating run in the background + +Serving a model or moving one to new weights from the console starts a job +on the control plane and leaves the console free at once, instead of +closing it to follow a download that can take an hour. A new **Jobs** view +shows each job's progress and steps, and the sidebar how many are running. +`llmgw ctl model serve --detach` and `llmgw ctl model update --detach` do the +same; `llmgw ctl jobs` lists the jobs and `llmgw ctl jobs ID` follows one. + ### The file that suits the hardware, and how to spread it `llmgw ctl model plan REPO` now plans the files the repository actually has diff --git a/docs/console.md b/docs/console.md index 85d77a72..8cc5a2fe 100644 --- a/docs/console.md +++ b/docs/console.md @@ -45,9 +45,12 @@ with its reason, word for word. Anything that cannot be undone — removing, draining, deleting, revoking — asks first, and only **Y** goes ahead. -Tasks that take a while and print as they go — serving a model, upgrading -the cluster, a backup, adding a node — close the screen and run in the -terminal, so their output is there to read; Enter brings the console back. +Serving a model and moving one to new weights are jobs on the control plane: +started from here, they run there in the background whatever this screen +does, and the console is free at once. **Jobs** follows them. Upgrading the +cluster, a backup and adding a node change this machine or another over ssh; +those close the screen and run in the terminal, so their output is there to +read, and Enter brings the console back. ## Watch @@ -119,6 +122,12 @@ their checksums and where each was fetched from, the source a model definition names for it, or remove it — with a warning when a model still uses it. +**Jobs** lists the models being served or updated since the control plane +started: what each is doing now — the download's progress, a replica +starting — and whether it ended done, failed, or waiting for room on busy +cards. The entry shows how many are running. Enter on one shows its steps. +`llmgw ctl jobs` is the same, and `llmgw ctl jobs ID` follows one. + **Logs** shows a node's agent log, newest at the bottom, errors and warnings coloured. diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 50061e76..fc41efb3 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -142,6 +142,7 @@ func newCtlCmd() *cobra.Command { newRestartCmd(client), newRetryCmd(client), newUpgradeCmd(client), + newJobsCmd(client, &asJSON), newEngineCheckCmd(), newEngineBuildCmd(client), newToolCheckCmd(), diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go index 0bba050a..a8e2105a 100644 --- a/internal/cli/model_serve.go +++ b/internal/cli/model_serve.go @@ -15,6 +15,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { var ( req modelserve.Request onDemand bool + detach bool ) c := &cobra.Command{ Use: "serve OWNER/REPO", @@ -49,6 +50,9 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { return err } + if detach { + return detached(cmd, job) + } return followJob(cmd, client, job, "is served") }, } @@ -64,6 +68,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command { f.IntVar(&req.TP, "tp", 0, "the older name of --gpus") _ = f.MarkHidden("tp") f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy") + f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it") f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)") f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") @@ -110,3 +115,54 @@ func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job, } } } + +// detached says where a job started without waiting for it can be followed. +func detached(cmd *cobra.Command, job modelserve.Job) error { + fmt.Fprintf(cmd.OutOrStdout(), "started %s on the control plane\nfollow it: llmgw ctl jobs %s\n", job.ID, job.ID) + return nil +} + +func newJobsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "jobs [JOB]", + Short: "Models being served or updated, or one followed until it ends", + Long: "Serving a model and moving one to new weights run as jobs on the control\n" + + "plane, which carry on whether or not anything watches them; they are\n" + + "kept until it restarts. With a job's id, its steps are shown as they\n" + + "happen until it ends.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 1 { + var job modelserve.Job + if err := client().get(cmd.Context(), "/api/v1/jobs/"+args[0], &job); err != nil { + return err + } + return followJob(cmd, client, job, "is done") + } + var jobs []modelserve.Job + if err := client().get(cmd.Context(), "/api/v1/jobs", &jobs); err != nil { + return err + } + if *asJSON { + return emitJSON(jobs) + } + if len(jobs) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "no jobs since the control plane started") + return nil + } + w := tabTo(cmd.OutOrStdout()) + fmt.Fprintln(w, "JOB\tMODEL\tSTATE\tSTARTED\tNOW") + for _, j := range jobs { + state := j.State + for _, st := range j.Steps { + if st.Name == "placement" && st.State == modelserve.Skipped { + state = "waiting for room" + } + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", j.ID, or(j.Model, j.Request.Repo), state, + ago(j.Started, time.Now()), truncate(jobNow(j), 80)) + } + return w.Flush() + }, + } +} diff --git a/internal/cli/model_update.go b/internal/cli/model_update.go index fe985ddd..1795956b 100644 --- a/internal/cli/model_update.go +++ b/internal/cli/model_update.go @@ -82,7 +82,8 @@ func updateCell(u modelserve.UpdateInfo) string { } func newModelUpdateCmd(client func() *ctlClient) *cobra.Command { - return &cobra.Command{ + var detach bool + c := &cobra.Command{ Use: "update MODEL", Short: "Move a model to the weights its repository publishes now", Long: "The new files are fetched into the mirror beside the ones in use and\n" + @@ -97,9 +98,14 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command { if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil { return err } + if detach { + return detached(cmd, job) + } return followJob(cmd, client, job, "runs the new weights") }, } + c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it") + return c } func newModelRollbackCmd(client func() *ctlClient) *cobra.Command { diff --git a/internal/cli/tui_actions.go b/internal/cli/tui_actions.go index f6b9491f..1f2613cc 100644 --- a/internal/cli/tui_actions.go +++ b/internal/cli/tui_actions.go @@ -712,7 +712,6 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { } return nil } - server := "--server=" + m.client.base var items []menuItem if u := md.Update; u != nil && u.Available { items = append(items, menuItem{fmt.Sprintf("Update to %s (%s)…", catalog.ShortRevision(u.Latest), bytesShort(u.Bytes)), @@ -722,7 +721,10 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem { "They are fetched into the mirror beside the ones in use and checked, then the\n"+ "replicas are replaced one at a time. The old files stay, to go back to.", u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")), - yes: func(m *consoleModel) tea.Cmd { return m.runAfter("ctl", server, "model", "update", md.Name) }} + yes: func(m *consoleModel) tea.Cmd { + return m.call("Update "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/update", + map[string]string{}, startedSays) + }} return nil }}) } diff --git a/internal/cli/tui_actions_test.go b/internal/cli/tui_actions_test.go index 794ef3b4..38d7ce65 100644 --- a/internal/cli/tui_actions_test.go +++ b/internal/cli/tui_actions_test.go @@ -9,6 +9,7 @@ import ( "strings" "sync" "testing" + "time" tea "github.com/charmbracelet/bubbletea" @@ -312,11 +313,18 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { } } } + var started modelserve.Request admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo { _ = json.NewEncoder(w).Encode(plan) return } + if r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/serve" { + _ = json.NewDecoder(r.Body).Decode(&started) + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(modelserve.Job{ID: "serve-1", State: modelserve.Running}) + return + } _, _ = w.Write([]byte("[]")) })) defer admin.Close() @@ -351,12 +359,17 @@ func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) { t.Fatalf("form: %+v", f) } f.focus = len(f.fields) - tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter}) - act := tm.(consoleModel).action - got := strings.Join(act.targets, " ") - if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") || - !strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") { - t.Fatalf("would run %q %q", act.kind, got) + tm = pressRun(tm, "enter") + // Started on the control plane; the console stays. + if act := tm.(consoleModel).action; act.kind != "" { + t.Fatalf("the console left to run %v", act) + } + if started.Repo != plan.Repo || started.File != "Qwen2.5-3B-Instruct-Q8_0.gguf" || started.GPUClass != "6000-24g" || + started.GPUsPerNode != 1 || !started.Fallbacks || started.Name != "qwen2.5-3b-instruct" { + t.Fatalf("started %+v", started) + } + if v := tm.View(); !strings.Contains(v, "serve-1") { + t.Errorf("the start was not shown:\n%s", v) } } @@ -463,3 +476,19 @@ func TestEveryScreenShowsTheVersionTopRight(t *testing.T) { t.Errorf("the manual's top line: %q", first) } } + +func TestJobsAreFollowedFromTheirOwnView(t *testing.T) { + m, _ := consoleWith(t, cJobs) + m.width = 160 + m.data.jobs = []modelserve.Job{{ID: "serve-9", Model: "phi-4", State: modelserve.Running, Started: time.Now(), + Steps: []modelserve.Step{{Name: "registry", State: modelserve.Done, Detail: "phi-4-Q8_0.gguf"}, + {Name: "mirror", State: modelserve.Running, Detail: "fetching: 3.1 of 14.6 GB (21%)"}}}} + v := m.View() + if !strings.Contains(v, "Jobs (1)") || !strings.Contains(v, "fetching: 3.1 of 14.6 GB") { + t.Fatalf("%s", v) + } + tm := press(m, "enter") + if v := tm.View(); !strings.Contains(v, "serve-9") || !strings.Contains(v, "phi-4-Q8_0.gguf") { + t.Errorf("steps not shown:\n%s", v) + } +} diff --git a/internal/cli/tui_console.go b/internal/cli/tui_console.go index 80591718..06ef9a36 100644 --- a/internal/cli/tui_console.go +++ b/internal/cli/tui_console.go @@ -20,6 +20,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/docgen" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/settings" @@ -39,6 +40,7 @@ const ( cUsage cQuotas cMirror + cJobs cLogs cAccounts cKeys @@ -55,7 +57,7 @@ const ( cCount ) -var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Logs", "Accounts", "API keys", "Organizations", "Manual", +var consoleLabels = [cCount]string{"Overview", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Manual", "server.yaml", "Add a node", "Replicate the store", "Serve a model", "Upgrade the cluster", "Restart all replicas", "Back up now", "Leave"} @@ -71,6 +73,7 @@ type consoleData struct { nodes []adminapi.NodeView models []adminapi.ModelView replicas []adminapi.ReplicaView + jobs []modelserve.Job err error } @@ -221,6 +224,7 @@ func (m consoleModel) fetch() tea.Cmd { _ = c.get(ctx, "/api/v1/nodes", &d.nodes) _ = c.get(ctx, "/api/v1/models", &d.models) _ = c.get(ctx, "/api/v1/replicas", &d.replicas) + _ = c.get(ctx, "/api/v1/jobs", &d.jobs) return consoleDataMsg(d) } } @@ -448,6 +452,9 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.menuD = m.quotaMenu(m.quotas[m.row]) case cMirror: m.menuD = m.mirrorMenu(m.mirrorGroups()[m.row]) + case cJobs: + m.dialog = "error" + m.dialogTitle, m.dialogText = jobDialog(m.data.jobs[m.row]) case cServe: return m, m.fetchRepo(m.hf.models[m.row].ID) case cAccounts: @@ -831,6 +838,8 @@ func (m consoleModel) rowCount() int { return len(m.quotas) case cMirror: return len(m.mirrorGroups()) + case cJobs: + return len(m.data.jobs) case cServe: return len(m.hf.models) case cAccounts: @@ -871,6 +880,9 @@ func (m consoleModel) View() string { if i == cSettings { label = filepathBase(m.file) } + if n := runningJobs(m.data.jobs); i == cJobs && n > 0 { + label += fmt.Sprintf(" (%d)", n) + } if i == cSettings && m.doc != nil && m.doc.Changed > 0 { label += fmt.Sprintf(" (%d)", m.doc.Changed) } @@ -1060,6 +1072,8 @@ func (m consoleModel) panel() (title, body, keys string) { return "Manual", d.list(pw, m.inPanel), keys case cMirror: return m.mirrorPanel(pw) + case cJobs: + return m.jobsPanel(pw) case cServe: return m.hfPanel(pw) case cSettings: diff --git a/internal/cli/tui_hf.go b/internal/cli/tui_hf.go index a555365b..2b0abccf 100644 --- a/internal/cli/tui_hf.go +++ b/internal/cli/tui_hf.go @@ -3,6 +3,7 @@ package cli import ( "context" "fmt" + "net/http" "net/url" "slices" "strconv" @@ -174,7 +175,6 @@ func (m consoleModel) repoMenu(p modelserve.Plan) *menuDialog { } func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact) { - server := "--server=" + m.client.base var opts []modelplan.Option for _, o := range p.Options { if o.Artifact == a.Name { @@ -228,33 +228,26 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact if c, err := strconv.Atoi(v[2]); err != nil || c <= 0 { return nil, fmt.Errorf("the context length is a whole number") } - args := []string{"ctl", server, "model", "serve", p.Repo, "--name=" + v[0], "--replicas=" + v[1], - "--context-length=" + v[2]} - if a.Format == "safetensors" { - args = append(args, "--format=safetensors") - } else { - args = append(args, "--file="+a.Name) + ctxLen, _ := strconv.Atoi(v[2]) + req := modelserve.Request{Repo: p.Repo, Name: v[0], Replicas: n, ContextLength: ctxLen, + Description: v[3], Format: a.Format} + if a.Format == "gguf" { + req.File = a.Name } o := opts[slices.Index(where, v[4])] if o.CPU { - args = append(args, "--cpu") + req.CPU = true } else { - args = append(args, "--gpu-class="+o.Class, "--gpus="+strconv.Itoa(o.GPUsPerNode), - "--nodes="+strconv.Itoa(max(o.Nodes, 1))) - } - if strings.HasPrefix(v[5], "the next") && len(opts) > 1 { - args = append(args, "--fallbacks") - } - if n == 0 { - args = append(args, "--on-demand") - } - if v[3] != "" { - args = append(args, "--description="+v[3]) + req.GPUClass, req.GPUsPerNode, req.Nodes = o.Class, o.GPUsPerNode, max(o.Nodes, 1) } + req.Fallbacks = strings.HasPrefix(v[5], "the next") && len(opts) > 1 if proj >= 0 && !strings.HasPrefix(v[proj], "none") { - args = append(args, "--projector="+v[proj]) + req.Projector = v[proj] } - return m.runAfter(args...), nil + // Started on the control plane, which runs it whatever happens + // to this screen: the console is free again at once, and Jobs + // shows how far it has got. + return m.call("Serve "+p.Repo, http.MethodPost, "/api/v1/models/serve", req, startedSays), nil }}) } @@ -319,6 +312,8 @@ func ageShort(t time.Time) string { } d := time.Since(t) switch { + case d < time.Minute: + return "just now" case d < time.Hour: return fmt.Sprintf("%dm ago", int(d.Minutes())) case d < 48*time.Hour: diff --git a/internal/cli/tui_jobs.go b/internal/cli/tui_jobs.go new file mode 100644 index 00000000..702a6d4e --- /dev/null +++ b/internal/cli/tui_jobs.go @@ -0,0 +1,102 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" +) + +// Jobs: serving a model or moving one to new weights is a job on the control +// plane — a download that can take an hour, then a replica starting. Started +// from the console, it runs there whatever this screen does; the console is +// free at once, and this view follows it. + +// startedSays is the result of starting a job. +func startedSays(raw json.RawMessage) string { + var j modelserve.Job + if json.Unmarshal(raw, &j) != nil || j.ID == "" { + return "Started." + } + return fmt.Sprintf("Started %s. It runs on the control plane — leaving this screen does not stop it.\n"+ + "Jobs, under Watch, shows how far it has got.", j.ID) +} + +// runningJobs counts the jobs still under way. +func runningJobs(jobs []modelserve.Job) int { + n := 0 + for _, j := range jobs { + if j.State == modelserve.Running { + n++ + } + } + return n +} + +func jobState(j modelserve.Job) string { + switch j.State { + case modelserve.Running: + return sWarn.Render("running") + case modelserve.Failed: + return sBad.Render("failed") + } + for _, st := range j.Steps { + if st.Name == "placement" && st.State == modelserve.Skipped { + return sWarn.Render("waiting for room") + } + } + return sOK.Render("done") +} + +// jobNow is the step a job is at, in a line. +func jobNow(j modelserve.Job) string { + if j.State == modelserve.Failed { + return j.Error + } + if len(j.Steps) == 0 { + return "" + } + st := j.Steps[len(j.Steps)-1] + return st.Name + ": " + st.Detail +} + +func (m consoleModel) jobsPanel(pw int) (string, string, string) { + keys := "→ or Enter open · ↑↓ choose · Q leave" + if m.inPanel { + keys = "↑↓ row · Enter its steps · ← back · Q leave" + } + head := "\n " + sDim.Render("Models being served or updated, on the control plane; kept until it restarts.") + "\n\n" + var rows [][]string + for _, j := range m.data.jobs { + kind := j.Kind + if kind == "" { + kind = "serve" + } + what := j.Model + if what == "" { + what = j.Request.Repo + } + rows = append(rows, []string{ageShort(j.Started), kind, what, jobState(j), jobNow(j)}) + } + if len(rows) == 0 { + return "Jobs", head + " " + sDim.Render("None since the control plane started. Serve a model, under Do, starts one."), keys + } + return "Jobs", head + table([]string{"STARTED", "KIND", "MODEL", "STATE", "NOW"}, rows, m.row, pw, m.inPanel), keys +} + +// jobDialog is a job's steps, each as it stands. +func jobDialog(j modelserve.Job) (string, string) { + var b strings.Builder + mark := map[string]string{modelserve.Running: sWarn.Render(".."), modelserve.Done: sOK.Render("ok"), + modelserve.Failed: sBad.Render("!!"), modelserve.Skipped: sDim.Render("--")} + for _, st := range j.Steps { + fmt.Fprintf(&b, "%s %-11s %s\n", mark[st.State], st.Name, wrap(st.Detail, 80, strings.Repeat(" ", 15))) + } + if j.Reply != "" { + fmt.Fprintf(&b, "\n%s said %q at %.1f tokens/s\n", j.Model, j.Reply, j.TokensPerSecond) + } + fmt.Fprintf(&b, "\n%s, started %s\n\n%s", j.ID, j.Started.Format(time.DateTime), sDim.Render("any key")) + return j.ID, b.String() +} diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 6ad85dd1..ec37921a 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -122,6 +122,13 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/restart", a.restart) mux.HandleFunc("POST /api/v1/models/serve", a.serveModel) mux.HandleFunc("GET /api/v1/jobs/{id}", a.job) + mux.HandleFunc("GET /api/v1/jobs", func(w http.ResponseWriter, _ *http.Request) { + out := []modelserve.Job{} + if a.serve != nil { + out = a.serve.Jobs() + } + writeJSON(w, http.StatusOK, out) + }) mux.HandleFunc("GET /api/v1/updates", a.listUpdates) mux.HandleFunc("POST /api/v1/updates/check", a.checkUpdates) mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel) diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index 4a59c34c..9a148930 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -240,6 +240,22 @@ func (s *Server) Start(req Request) (Job, error) { return s.snapshot(j), nil } +// Jobs is every job since the control plane started, the newest first. +func (s *Server) Jobs() []Job { + s.mu.Lock() + all := make([]*Job, 0, len(s.jobs)) + for _, j := range s.jobs { + all = append(all, j) + } + s.mu.Unlock() + out := make([]Job, 0, len(all)) + for _, j := range all { + out = append(out, s.snapshot(j)) + } + slices.SortFunc(out, func(a, b Job) int { return b.Started.Compare(a.Started) }) + return out +} + // Get returns a job as it stands. func (s *Server) Get(id string) (Job, bool) { s.mu.Lock()