Serve and update models in the background from the console, and follow them under Jobs
The console starts a serve or update job over the admin API and stays, instead of closing to follow the download. Jobs lists them with their progress and steps; the sidebar counts those running. ctl model serve and update take --detach; ctl jobs lists jobs and follows one. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d311a3ca4a
commit
e12996f7dd
12 files changed
+280
-34
No files matched your search
@@ -15,6 +15,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
req modelserve.Request
|
||||
onDemand bool
|
||||
detach bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "serve OWNER/REPO",
|
||||
@@ -49,6 +50,9 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
if detach {
|
||||
return detached(cmd, job)
|
||||
}
|
||||
return followJob(cmd, client, job, "is served")
|
||||
},
|
||||
}
|
||||
@@ -64,6 +68,7 @@ func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
f.IntVar(&req.TP, "tp", 0, "the older name of --gpus")
|
||||
_ = f.MarkHidden("tp")
|
||||
f.BoolVar(&req.Fallbacks, "fallbacks", false, "also add the next best ways to run it, used when the first is busy")
|
||||
f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
@@ -110,3 +115,54 @@ func followJob(cmd *cobra.Command, client func() *ctlClient, job modelserve.Job,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detached says where a job started without waiting for it can be followed.
|
||||
func detached(cmd *cobra.Command, job modelserve.Job) error {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "started %s on the control plane\nfollow it: llmgw ctl jobs %s\n", job.ID, job.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func newJobsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "jobs [JOB]",
|
||||
Short: "Models being served or updated, or one followed until it ends",
|
||||
Long: "Serving a model and moving one to new weights run as jobs on the control\n" +
|
||||
"plane, which carry on whether or not anything watches them; they are\n" +
|
||||
"kept until it restarts. With a job's id, its steps are shown as they\n" +
|
||||
"happen until it ends.",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if len(args) == 1 {
|
||||
var job modelserve.Job
|
||||
if err := client().get(cmd.Context(), "/api/v1/jobs/"+args[0], &job); err != nil {
|
||||
return err
|
||||
}
|
||||
return followJob(cmd, client, job, "is done")
|
||||
}
|
||||
var jobs []modelserve.Job
|
||||
if err := client().get(cmd.Context(), "/api/v1/jobs", &jobs); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(jobs)
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "no jobs since the control plane started")
|
||||
return nil
|
||||
}
|
||||
w := tabTo(cmd.OutOrStdout())
|
||||
fmt.Fprintln(w, "JOB\tMODEL\tSTATE\tSTARTED\tNOW")
|
||||
for _, j := range jobs {
|
||||
state := j.State
|
||||
for _, st := range j.Steps {
|
||||
if st.Name == "placement" && st.State == modelserve.Skipped {
|
||||
state = "waiting for room"
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", j.ID, or(j.Model, j.Request.Repo), state,
|
||||
ago(j.Started, time.Now()), truncate(jobNow(j), 80))
|
||||
}
|
||||
return w.Flush()
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user