llmgw/internal/cli/model_canary.go

package cli

import (
	"encoding/json"
	"fmt"
	"slices"
	"text/tabwriter"

	"github.com/spf13/cobra"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
)

func newModelCanaryCmd(client func() *ctlClient) *cobra.Command {
	var (
		cn            catalog.Canary
		abort, detach bool
	)
	c := &cobra.Command{
		Use:   "canary MODEL",
		Short: "Try a model's newer weights on a share of its requests before moving to them",
		Long: "The newer files of the model's repository are fetched and run beside it\n" +
			"as MODEL-canary, one replica, and given --percent of MODEL's requests. A\n" +
			"session (x-session-id, or the user field) stays on one side; the\n" +
			"x-llmgw-variant header (canary or stable) pins a request to one. Clients\n" +
			"cannot name the canary itself.\n\n" +
			"The two sides are compared as they serve. Once the canary has served\n" +
			"--min-requests, it is taken out by itself if its error rate is\n" +
			"--max-error-increase above the model's, or its time to the first token\n" +
			"(or, unstreamed, to the answer) --max-ttft-ratio times the model's, and\n" +
			"canary_rolled_back goes to the notification webhooks.\n\n" +
			"Run again with --percent to change its share, --abort to take it out;\n" +
			"`llmgw ctl model promote MODEL` moves the model to it.",
		Example: "  llmgw ctl model canary qwen-7b --percent 10\n" +
			"  llmgw ctl model canary qwen-7b --percent 50\n" +
			"  llmgw ctl model canary qwen-7b --abort",
		Args: cobra.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			name := args[0]
			var out map[string]string
			if abort {
				if err := client().delete(cmd.Context(), "/api/v1/models/"+name+"/canary", &out); err != nil {
					return err
				}
				fmt.Fprintln(cmd.OutOrStdout(), out["result"])
				return nil
			}
			var have []modelserve.CanaryView
			if err := client().get(cmd.Context(), "/api/v1/canaries", &have); err != nil {
				return err
			}
			if slices.ContainsFunc(have, func(v modelserve.CanaryView) bool { return v.Model == name }) {
				if !cmd.Flags().Changed("percent") {
					return fmt.Errorf("%s already has a canary: give --percent to change its share, or --abort", name)
				}
				body, _ := json.Marshal(map[string]int{"percent": cn.Percent})
				if err := client().put(cmd.Context(), "/api/v1/models/"+name+"/canary", body, &out); err != nil {
					return err
				}
				fmt.Fprintln(cmd.OutOrStdout(), out["result"])
				return nil
			}
			body, _ := json.Marshal(cn)
			var job modelserve.Job
			if err := client().post(cmd.Context(), "/api/v1/models/"+name+"/canary", body, &job); err != nil {
				return err
			}
			if detach {
				return detached(cmd, job)
			}
			return followJob(cmd, client, job, "has a canary")
		},
	}
	f := c.Flags()
	f.IntVar(&cn.Percent, "percent", 10, "share of the model's requests the canary is given, 0 to 100")
	f.IntVar(&cn.MinRequests, "min-requests", 50, "requests the canary serves before it is judged")
	f.Float64Var(&cn.MaxErrorIncrease, "max-error-increase", 0.05, "error rate above the model's that rolls it back (0.05 is five points)")
	f.Float64Var(&cn.MaxTTFTRatio, "max-ttft-ratio", 1.5, "times the model's time to the first token that rolls it back")
	f.BoolVar(&abort, "abort", false, "take the canary out; the model serves all its requests again")
	f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
	return c
}

func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
	var (
		detach bool
		gate   gateFlags
	)
	c := &cobra.Command{
		Use:   "promote MODEL",
		Short: "Move a model to its canary's weights, and take the canary out",
		Long: "The update the canary was a trial of: the catalog names the canary's\n" +
			"files, the model's replicas are replaced one at a time, and the canary\n" +
			"is removed. `llmgw ctl model rollback MODEL` goes back.\n\n" +
			"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
			"gate`), the canary is asked it first, and below its score nothing\n" +
			"moves: the canary stays, with its share, until it is taken out.",
		Example: "  llmgw ctl model promote qwen-7b\n" +
			"  llmgw ctl model promote qwen-7b --eval ./support.yaml --min-score 0.9 --judge llama-70b",
		Args: cobra.ExactArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			body, err := gate.body()
			if err != nil {
				return err
			}
			var job modelserve.Job
			if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", body, &job); err != nil {
				return err
			}
			if detach {
				return detached(cmd, job)
			}
			return followJob(cmd, client, job, "runs the canary's weights")
		},
	}
	c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
	gate.add(c)
	return c
}

func newModelCanariesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
	return &cobra.Command{
		Use:   "canaries",
		Short: "Models trying a canary, and how each side has served since it began",
		Args:  cobra.NoArgs,
		RunE: func(cmd *cobra.Command, _ []string) error {
			var vs []modelserve.CanaryView
			if err := client().get(cmd.Context(), "/api/v1/canaries", &vs); err != nil {
				return err
			}
			if *asJSON {
				return json.NewEncoder(cmd.OutOrStdout()).Encode(vs)
			}
			if len(vs) == 0 {
				fmt.Fprintln(cmd.OutOrStdout(), "No model has a canary.")
				return nil
			}
			w := tabwriter.NewWriter(cmd.OutOrStdout(), 0, 0, 2, ' ', 0)
			fmt.Fprintln(w, "MODEL\tCANARY\tSHARE\tREQUESTS\tERRORS\tFIRST TOKEN\tANSWER")
			for _, v := range vs {
				side := func(label string, req int64, errRate, ttft, lat float64) {
					fmt.Fprintf(w, "%s\t%s\t\t%d\t%.1f%%\t%s\t%s\n", "", label, req, 100*errRate, meanMs(ttft), meanMs(lat))
				}
				fmt.Fprintf(w, "%s\t%s\t%d%%\t\t\t\t\n", v.Model, v.Canary.Model, v.Canary.Percent)
				if s := v.Serving; s != nil {
					side("  canary", s.Candidate.Requests, s.Candidate.ErrorRate(), s.Candidate.TTFTMs, s.Candidate.LatencyMs)
					side("  stable", s.Stable.Requests, s.Stable.ErrorRate(), s.Stable.TTFTMs, s.Stable.LatencyMs)
				}
			}
			return w.Flush()
		},
	}
}

func meanMs(v float64) string {
	if v <= 0 {
		return "-"
	}
	return fmt.Sprintf("%.0f ms", v)
}