Merge features-c2: the docs portal and its links in English and Turkish
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
e8a62b23b7
201 files changed
+26276
-630
No files matched your search
+19
-3
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
)
|
||||
|
||||
@@ -64,6 +65,21 @@ func newAgentCmd() *cobra.Command {
|
||||
}
|
||||
cfg.Version = version.Version
|
||||
|
||||
// Before the agent, so its tracer is the configured one.
|
||||
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
|
||||
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
|
||||
Service: "llmgw-agent", Version: version.Version, Instance: cfg.NodeID,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
// Bounded: a collector that is gone must not hold up a shutdown.
|
||||
sctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = shutdownTracing(sctx)
|
||||
}()
|
||||
|
||||
httpClient, err := agent.Client(cfg.TLS, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -159,13 +175,13 @@ func printInventory(inv discovery.Inventory) {
|
||||
if !g.Health.OK() {
|
||||
health = g.Health.Problems[0]
|
||||
}
|
||||
fmt.Fprintf(w, " %d\t%s\t%s\t%d MiB\t%s\t%s\n",
|
||||
g.Index, g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||
fmt.Fprintf(w, " %s\t%s\t%s\t%d MiB\t%s\t%s\n",
|
||||
discovery.DeviceLabel(g.Index), g.Class(), g.Name, g.MemoryTotalMB, g.ComputeCapability, health)
|
||||
}
|
||||
fmt.Fprintf(w, " classes: %v\n", inv.Classes())
|
||||
for _, g := range inv.GPUs {
|
||||
if len(g.NVLinkPeers) > 0 {
|
||||
fmt.Fprintf(w, " GPU %d NVLink peers: %v\n", g.Index, g.NVLinkPeers)
|
||||
fmt.Fprintf(w, " GPU %s NVLink peers: %v\n", discovery.DeviceLabel(g.Index), g.NVLinkPeers)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newAnswerCacheCmd(open opener) *cobra.Command {
|
||||
var (
|
||||
project string
|
||||
ttl time.Duration
|
||||
anyAnswer bool
|
||||
off bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "cache",
|
||||
Short: "Reuse a project's answers: the same request is answered again without the model",
|
||||
Long: "With this on, a request of the project that is the same as one answered within\n" +
|
||||
"--ttl, to the same model, gets that answer again, from the gateway's memory,\n" +
|
||||
"rather than asking the model. It is answered with x-llmgw-cache: hit, recorded\n" +
|
||||
"as a request with no tokens, and costs nothing against a budget.\n\n" +
|
||||
"Only answers that would come out the same are reused: a chat or completion\n" +
|
||||
"asked at temperature 0 or with a seed, and embeddings, rerank and score.\n" +
|
||||
"--any reuses every answer, sampled or not. A client sending\n" +
|
||||
"Cache-Control: no-cache or x-llmgw-cache: no is always asked afresh, and so is\n" +
|
||||
"a request continuing a stored conversation or under a guardrails no_store\n" +
|
||||
"policy. The guardrails judge a reused answer as they judge any other.\n\n" +
|
||||
"Off for every project until this is run. Without --ttl or --off, says how\n" +
|
||||
"it stands. The answers are held in memory, router.cache bounds how much.",
|
||||
Example: " llmgw keys cache --project Acme/ml/train --ttl 1h\n" +
|
||||
" llmgw keys cache --project Acme/ml/eval --ttl 24h --any\n" +
|
||||
" llmgw keys cache --project Acme/ml/train --off",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
switch {
|
||||
case off && (ttl != 0 || anyAnswer):
|
||||
return fmt.Errorf("--off, or --ttl with --any: not both")
|
||||
case ttl < 0:
|
||||
return fmt.Errorf("--ttl is how long an answer is reused, like 1h")
|
||||
case ttl > 0 && ttl < time.Second:
|
||||
return fmt.Errorf("--ttl is at least a second")
|
||||
case anyAnswer && ttl == 0:
|
||||
return fmt.Errorf("--any goes with --ttl: say how long answers are reused")
|
||||
}
|
||||
s, err := open(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
p, err := resolveProject(cmd.Context(), s, project)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
if err := s.SetAnswerCache(cmd.Context(), p.ID, ttl, anyAnswer); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
c, err := s.AnswerCacheOf(cmd.Context(), p.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch {
|
||||
case c.TTL == 0:
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses no answers.\n", p.Path)
|
||||
case c.Any:
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses every answer for %s, sampled or not.\n", p.Path, c.TTL)
|
||||
default:
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s reuses answers for %s: those asked at temperature 0 or with a seed, "+
|
||||
"and embeddings, rerank and score.\n", p.Path, c.TTL)
|
||||
}
|
||||
if off || ttl > 0 {
|
||||
fmt.Fprintln(cmd.OutOrStdout(), "Gateways see the change within 30 seconds.")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
|
||||
cmd.Flags().DurationVar(&ttl, "ttl", 0, "how long an answer is reused, like 1h")
|
||||
cmd.Flags().BoolVar(&anyAnswer, "any", false, "reuse sampled answers too, not only reproducible ones")
|
||||
cmd.Flags().BoolVar(&off, "off", false, "stop reusing answers for this project")
|
||||
return cmd
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAProjectIsToldToReuseItsAnswersAndToStop(t *testing.T) {
|
||||
db := filepath.Join(t.TempDir(), "s.db")
|
||||
mustRun(t, db, "org", "add", "org", "Acme")
|
||||
mustRun(t, db, "org", "add", "team", "ml", "--in", "Acme")
|
||||
mustRun(t, db, "org", "add", "project", "train", "--in", "Acme/ml")
|
||||
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train"); !strings.Contains(out, "reuses no answers") {
|
||||
t.Errorf("before: %s", out)
|
||||
}
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "Acme/ml/train", "--ttl", "1h"); !strings.Contains(out, "for 1h0m0s: those asked at temperature 0") {
|
||||
t.Errorf("on: %s", out)
|
||||
}
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--ttl", "24h", "--any"); !strings.Contains(out, "every answer for 24h0m0s") {
|
||||
t.Errorf("any: %s", out)
|
||||
}
|
||||
for _, bad := range [][]string{{"--off", "--ttl", "1h"}, {"--any"}, {"--ttl", "10ms"}} {
|
||||
if _, err := run(t, db, append([]string{"keys", "cache", "--project", "train"}, bad...)...); err == nil {
|
||||
t.Errorf("%v was accepted", bad)
|
||||
}
|
||||
}
|
||||
if out := mustRun(t, db, "keys", "cache", "--project", "train", "--off"); !strings.Contains(out, "reuses no answers") {
|
||||
t.Errorf("off: %s", out)
|
||||
}
|
||||
}
|
||||
@@ -106,7 +106,7 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&target, "target", "http://127.0.0.1:8080", "base URL: a replica, or the gateway")
|
||||
f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage")
|
||||
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp")
|
||||
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp | trtllm (whose are at /prometheus/metrics)")
|
||||
f.StringVar(&key, "key", "", "API key, when the gateway requires one")
|
||||
f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)")
|
||||
f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level")
|
||||
|
||||
+25
-2
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
@@ -147,6 +148,7 @@ func newCtlCmd() *cobra.Command {
|
||||
// cert works on files rather than the admin API, so --server does not
|
||||
// apply to it; it lives here because this is where operators look.
|
||||
newBenchCmd(client, &asJSON),
|
||||
newEvalCmd(client, &asJSON),
|
||||
newStatusCmd(client, &asJSON),
|
||||
newModelCmd(client, &asJSON),
|
||||
newScaleCmd(client),
|
||||
@@ -212,7 +214,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
// at when a model will not place and the hardware looks
|
||||
// fine.
|
||||
join(n.Engines),
|
||||
join(n.Classes), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
@@ -410,7 +412,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
for _, r := range reps {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
|
||||
// "[0/mig2]" for a MIG instance, not the 1002 it is
|
||||
// numbered by.
|
||||
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||||
}
|
||||
// The state says stale rather than a column of its own: most
|
||||
// replicas are not, a column would be empty for all of them,
|
||||
@@ -540,6 +544,25 @@ func emitJSON(v any) error {
|
||||
return enc.Encode(v)
|
||||
}
|
||||
|
||||
// classesColumn is a node's GPU classes, with how many of each when it has
|
||||
// more than one or MIG instances among them: a card cut into two 3g.40gb
|
||||
// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
|
||||
// the GPUS column's 3 alone would read as three of something.
|
||||
func classesColumn(n adminapi.NodeView) string {
|
||||
counted := len(n.ClassGPUs) > 1
|
||||
for _, c := range n.ClassGPUs {
|
||||
counted = counted || c.MIG
|
||||
}
|
||||
if !counted {
|
||||
return join(n.Classes)
|
||||
}
|
||||
parts := make([]string, len(n.ClassGPUs))
|
||||
for i, c := range n.ClassGPUs {
|
||||
parts[i] = fmt.Sprintf("%s×%d", c.Class, c.Total)
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
func join(v []string) string {
|
||||
if len(v) == 0 {
|
||||
return "-"
|
||||
|
||||
@@ -0,0 +1,408 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
llmgw "gitea.mynodes.xyz/ilker/llmgw"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router"
|
||||
)
|
||||
|
||||
// newEvalService is the control plane's runner of quality suites: through
|
||||
// the router as its own requests, with the suites of control.eval_dir and
|
||||
// those the binary carries, kept in the store when there is one.
|
||||
func newEvalService(cfg config.Server, rt *router.Router, admin *control.Controller, log *slog.Logger) *eval.Service {
|
||||
builtin, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
|
||||
if err != nil {
|
||||
builtin = nil
|
||||
}
|
||||
d := eval.Deps{Call: rt.Replay, Dir: cfg.Control.EvalDir, Builtin: builtin, Log: log,
|
||||
Origin: func(name string) (string, string) {
|
||||
if m, ok := admin.Catalog().Resolve(name); ok && m.Origin != nil {
|
||||
return m.Origin.Repo, m.Origin.Revision
|
||||
}
|
||||
return "", ""
|
||||
},
|
||||
CanaryOf: func(name string) string { return admin.Catalog().CanaryOf(name) },
|
||||
}
|
||||
// Only a store that is there: a nil one in the interface would look
|
||||
// like one.
|
||||
if st := rt.Store(); st != nil {
|
||||
d.Store = st
|
||||
}
|
||||
return eval.NewService(d)
|
||||
}
|
||||
|
||||
// suiteRequest is a suite named on the command line: a file here, sent
|
||||
// whole, or else the name of one on the control plane.
|
||||
func suiteRequest(arg string) (eval.Request, error) {
|
||||
if st, err := os.Stat(arg); err == nil && !st.IsDir() {
|
||||
b, err := os.ReadFile(arg)
|
||||
if err != nil {
|
||||
return eval.Request{}, err
|
||||
}
|
||||
// Read here first, so a mistake in it is said before anything is sent.
|
||||
if _, err := eval.Parse(filepath.Base(arg), b); err != nil {
|
||||
return eval.Request{}, err
|
||||
}
|
||||
return eval.Request{Content: string(b), File: filepath.Base(arg)}, nil
|
||||
}
|
||||
if strings.ContainsAny(arg, `/\`) {
|
||||
return eval.Request{}, fmt.Errorf("%s: no such file here", arg)
|
||||
}
|
||||
return eval.Request{Suite: arg}, nil
|
||||
}
|
||||
|
||||
func newEvalCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var (
|
||||
suite, judge, run string
|
||||
history, suites, detach bool
|
||||
concurrency, limit int
|
||||
minScore float64
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "eval MODEL --suite FILE|NAME",
|
||||
Short: "Ask a model a quality suite and score its answers",
|
||||
Long: "A suite is a set of cases — a prompt or a conversation, and how its answer\n" +
|
||||
"is scored: exact, contains, regex, a number within a tolerance, JSON that\n" +
|
||||
"satisfies a schema, or a rubric another model on the cluster grades — with\n" +
|
||||
"a weight each and a score, 0 to 1, the suite is passed at (docs/install.md,\n" +
|
||||
"\"Quality suites\").\n\n" +
|
||||
"It runs on the control plane, which asks the model through the router as\n" +
|
||||
"its own requests, as a replay does: past admission, quotas, accounting and\n" +
|
||||
"the guardrails, so the score is the weights' and not a tenant's limits'.\n" +
|
||||
"--suite is a file here, sent whole, or the name of a suite in the control\n" +
|
||||
"plane's control.eval_dir (/etc/llmgw/eval) or built in: arithmetic,\n" +
|
||||
"json-format. Every run is kept, with the revision of the weights that\n" +
|
||||
"answered; --history lists them.\n\n" +
|
||||
"It exits non-zero when the model scores below the suite's threshold, or\n" +
|
||||
"--min-score.",
|
||||
Example: " llmgw ctl eval qwen-7b --suite arithmetic\n" +
|
||||
" llmgw ctl eval qwen-7b --suite ./support.yaml --judge llama-70b --concurrency 8\n" +
|
||||
" llmgw ctl eval --history qwen-7b\n" +
|
||||
" llmgw ctl eval --run eval-20261004-101500-1a2b3c4d\n" +
|
||||
" llmgw ctl eval --suites",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
out := cmd.OutOrStdout()
|
||||
switch {
|
||||
case suites:
|
||||
var list []eval.SuiteInfo
|
||||
if err := client().get(cmd.Context(), "/api/v1/eval/suites", &list); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(list)
|
||||
}
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "SUITE\tCASES\tPASS AT\tJUDGE\tFROM\tABOUT")
|
||||
for _, s := range list {
|
||||
if s.Error != "" {
|
||||
fmt.Fprintf(w, "%s\t-\t-\t-\t%s\tcannot be run: %s\n", s.Name, s.From, truncate(s.Error, 70))
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%d\t%.2f\t%s\t%s\t%s\n", s.Name, s.Cases, s.Threshold, yesNo(s.Judge), s.From,
|
||||
truncate(s.Description, 60))
|
||||
}
|
||||
return w.Flush()
|
||||
case run != "":
|
||||
var l eval.Live
|
||||
if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(run), &l); err != nil {
|
||||
return err
|
||||
}
|
||||
if l.State == eval.Running && !*asJSON {
|
||||
return followEval(cmd, client, l, false)
|
||||
}
|
||||
return showEval(cmd, l, *asJSON)
|
||||
case history:
|
||||
q := url.Values{}
|
||||
if len(args) == 1 {
|
||||
q.Set("model", args[0])
|
||||
}
|
||||
if limit > 0 {
|
||||
q.Set("limit", strconv.Itoa(limit))
|
||||
}
|
||||
var runs []eval.Live
|
||||
if err := client().get(cmd.Context(), "/api/v1/eval/runs?"+q.Encode(), &runs); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(runs)
|
||||
}
|
||||
return printEvalHistory(out, runs)
|
||||
}
|
||||
if len(args) != 1 {
|
||||
return errors.New("name the model to ask: llmgw ctl eval MODEL --suite FILE|NAME")
|
||||
}
|
||||
if suite == "" {
|
||||
return errors.New("--suite is required: a file, or a suite's name on the control plane (`llmgw ctl eval --suites`)")
|
||||
}
|
||||
req, err := suiteRequest(suite)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Concurrency, req.Judge, req.MinScore = concurrency, judge, minScore
|
||||
body, _ := json.Marshal(req)
|
||||
var l eval.Live
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+url.PathEscape(args[0])+"/eval", body, &l); err != nil {
|
||||
return err
|
||||
}
|
||||
if detach {
|
||||
fmt.Fprintf(out, "started %s on the control plane\nfollow it: llmgw ctl eval --run %s\n", l.ID, l.ID)
|
||||
return nil
|
||||
}
|
||||
return followEval(cmd, client, l, *asJSON)
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&suite, "suite", "", "the suite: a file here, or a suite's name on the control plane")
|
||||
f.IntVar(&concurrency, "concurrency", 4, "cases asked at once")
|
||||
f.StringVar(&judge, "judge", "", "the model that grades rubric cases, instead of the suite's judge")
|
||||
f.Float64Var(&minScore, "min-score", 0, "the score to pass at, 0 to 1, instead of the suite's threshold")
|
||||
f.BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane")
|
||||
f.BoolVar(&history, "history", false, "list the runs kept, of MODEL or of every model")
|
||||
f.IntVar(&limit, "limit", 20, "with --history, how many runs")
|
||||
f.StringVar(&run, "run", "", "show one run, by its id, with each case")
|
||||
f.BoolVar(&suites, "suites", false, "list the suites the control plane can run by name")
|
||||
return c
|
||||
}
|
||||
|
||||
func yesNo(b bool) string {
|
||||
if b {
|
||||
return "yes"
|
||||
}
|
||||
return "-"
|
||||
}
|
||||
|
||||
// followEval waits for a run, saying how far it has got, then shows it.
|
||||
func followEval(cmd *cobra.Command, client func() *ctlClient, l eval.Live, asJSON bool) error {
|
||||
out := cmd.OutOrStdout()
|
||||
if !asJSON {
|
||||
fmt.Fprintf(out, "run %s: %s asked %s, %d cases\n", l.ID, evalTarget(l), l.Suite, l.Cases)
|
||||
}
|
||||
shown := -1
|
||||
for l.State == eval.Running {
|
||||
if !asJSON && l.Done != shown {
|
||||
fmt.Fprintf(out, " %d of %d answered\n", l.Done, l.Cases)
|
||||
shown = l.Done
|
||||
}
|
||||
select {
|
||||
case <-cmd.Context().Done():
|
||||
return cmd.Context().Err()
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
if err := client().get(cmd.Context(), "/api/v1/eval/runs/"+url.PathEscape(l.ID), &l); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !asJSON {
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
return showEval(cmd, l, asJSON)
|
||||
}
|
||||
|
||||
func evalTarget(l eval.Live) string {
|
||||
if l.Target != "" {
|
||||
return l.Target
|
||||
}
|
||||
return l.Model
|
||||
}
|
||||
|
||||
// showEval prints a finished run's cases and score, and fails below it.
|
||||
func showEval(cmd *cobra.Command, l eval.Live, asJSON bool) error {
|
||||
out := cmd.OutOrStdout()
|
||||
if asJSON {
|
||||
if err := emitJSON(l); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if l.State == eval.Failed {
|
||||
return fmt.Errorf("the run could not be scored: %s", l.Error)
|
||||
}
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "CASE\tKIND\tWEIGHT\tSCORE\tTIME\tANSWER\tWHY")
|
||||
for _, r := range l.Results {
|
||||
score := fmt.Sprintf("%.2f", r.Score)
|
||||
if !r.Passed {
|
||||
score += " ✗"
|
||||
}
|
||||
why := r.Detail
|
||||
if r.Error != "" {
|
||||
why = "error: " + r.Error
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%g\t%s\t%s\t%s\t%s\n", r.ID, r.Kind, r.Weight, score,
|
||||
(time.Duration(r.TookMs) * time.Millisecond).Round(10*time.Millisecond), dash(truncate(oneLine(r.Answer), 30)),
|
||||
truncate(why, 70))
|
||||
}
|
||||
if err := w.Flush(); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
fmt.Fprintln(out, evalSummary(l))
|
||||
if l.Error != "" {
|
||||
fmt.Fprintln(out, l.Error)
|
||||
}
|
||||
}
|
||||
if !l.Passed {
|
||||
return fmt.Errorf("%s scored %.2f on %s, below %.2f", evalTarget(l), l.Score, l.Suite, l.Threshold)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// evalSummary is a run in a sentence.
|
||||
func evalSummary(l eval.Live) string {
|
||||
verdict := "passed"
|
||||
if !l.Passed {
|
||||
verdict = "failed"
|
||||
}
|
||||
s := fmt.Sprintf("%s scored %.2f on %s: %d of %d cases passed, %.2f asked: %s", evalTarget(l), l.Score, l.Suite,
|
||||
l.Cases-l.Below, l.Cases, l.Threshold, verdict)
|
||||
if l.Revision != "" {
|
||||
s += fmt.Sprintf(" (weights %s at %s)", dash(l.Repo), catalog.ShortRevision(l.Revision))
|
||||
}
|
||||
return s + "."
|
||||
}
|
||||
|
||||
func printEvalHistory(out io.Writer, runs []eval.Live) error {
|
||||
if len(runs) == 0 {
|
||||
fmt.Fprintln(out, "No runs kept: `llmgw ctl eval MODEL --suite NAME` makes one.")
|
||||
return nil
|
||||
}
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "WHEN\tRUN\tMODEL\tWEIGHTS\tSUITE\tSCORE\tPASS AT\tRESULT\tFOR")
|
||||
for _, r := range runs {
|
||||
model := r.Model
|
||||
if r.Target != "" {
|
||||
model += " (" + r.Target + ")"
|
||||
}
|
||||
result := "passed"
|
||||
switch {
|
||||
case r.State == eval.Running:
|
||||
result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
|
||||
case r.State == eval.Failed:
|
||||
result = "could not run"
|
||||
case !r.Passed:
|
||||
result = "failed"
|
||||
}
|
||||
score := fmt.Sprintf("%.2f", r.Score)
|
||||
if r.State == eval.Running {
|
||||
score = "-"
|
||||
}
|
||||
forWhat := r.Job
|
||||
if forWhat == "" {
|
||||
forWhat = r.By
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%.2f\t%s\t%s\n", ago(r.At, time.Now()), r.ID, model,
|
||||
dash(catalog.ShortRevision(r.Revision)), r.Suite, score, r.Threshold, result, dash(forWhat))
|
||||
}
|
||||
return w.Flush()
|
||||
}
|
||||
|
||||
// gateFlags are how an update or a promotion is held to a quality suite.
|
||||
type gateFlags struct {
|
||||
suite, judge string
|
||||
minScore float64
|
||||
skip bool
|
||||
}
|
||||
|
||||
func (g *gateFlags) add(c *cobra.Command) {
|
||||
f := c.Flags()
|
||||
f.StringVar(&g.suite, "eval", "", "a quality suite the new weights must pass first: a file here, or a suite's name on the control plane")
|
||||
f.Float64Var(&g.minScore, "min-score", 0, "with --eval, the score to pass at, 0 to 1; the suite's threshold otherwise")
|
||||
f.StringVar(&g.judge, "judge", "", "with --eval, the model that grades rubric cases")
|
||||
f.BoolVar(&g.skip, "skip-eval", false, "move without the model's own quality suite (`llmgw ctl model gate`)")
|
||||
}
|
||||
|
||||
// body is the change, as the API takes it.
|
||||
func (g gateFlags) body() ([]byte, error) {
|
||||
var ch modelserve.Change
|
||||
if g.suite != "" {
|
||||
if g.skip {
|
||||
return nil, errors.New("--eval and --skip-eval together: give one")
|
||||
}
|
||||
r, err := suiteRequest(g.suite)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.MinScore, r.Judge = g.minScore, g.judge
|
||||
ch.Eval = &r
|
||||
} else if g.minScore != 0 || g.judge != "" {
|
||||
return nil, errors.New("--min-score and --judge go with --eval")
|
||||
}
|
||||
ch.SkipEval = g.skip
|
||||
return json.Marshal(ch)
|
||||
}
|
||||
|
||||
func newModelGateCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
g adminapi.EvalGateRequest
|
||||
off bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "gate MODEL --eval SUITE [--min-score N]",
|
||||
Short: "Hold a model's updates and promotions to a quality suite",
|
||||
Long: "From then on `llmgw ctl model update` and `model promote` (and the\n" +
|
||||
"console's) ask the new weights the suite before they replace the old,\n" +
|
||||
"and stop below its score: an update runs them first as a canary given\n" +
|
||||
"none of the model's requests, and takes it out again when it fails. The\n" +
|
||||
"suite is one on the control plane, by name: in control.eval_dir, or\n" +
|
||||
"built in. --skip-eval on an update moves without it, once; --off takes it\n" +
|
||||
"away.",
|
||||
Example: " llmgw ctl model gate qwen-7b --eval arithmetic --min-score 0.8\n" +
|
||||
" llmgw ctl model gate qwen-7b --off",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
path := "/api/v1/models/" + url.PathEscape(args[0]) + "/eval-gate"
|
||||
var out struct {
|
||||
Model string `json:"model"`
|
||||
Eval *catalog.EvalGate `json:"eval"`
|
||||
}
|
||||
if off {
|
||||
if err := client().delete(cmd.Context(), path, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s's updates are no longer held to a quality suite\n", args[0])
|
||||
return nil
|
||||
}
|
||||
if g.Suite == "" {
|
||||
return errors.New("--eval names the suite, or --off takes the gate away")
|
||||
}
|
||||
body, _ := json.Marshal(g)
|
||||
if err := client().put(cmd.Context(), path, body, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
at := "the suite's threshold"
|
||||
if g.MinScore > 0 {
|
||||
at = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s's new weights must score at least %s on %s before they replace the old\n",
|
||||
args[0], at, g.Suite)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&g.Suite, "eval", "", "the suite: its name on the control plane")
|
||||
c.Flags().Float64Var(&g.MinScore, "min-score", 0, "the score to pass at, 0 to 1; the suite's threshold otherwise")
|
||||
c.Flags().StringVar(&g.Judge, "judge", "", "the model that grades the suite's rubric cases")
|
||||
c.Flags().BoolVar(&off, "off", false, "take the gate away")
|
||||
return c
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
llmgw "gitea.mynodes.xyz/ilker/llmgw"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestTheSuitesTheBinaryCarriesCanBeRun(t *testing.T) {
|
||||
sub, err := fs.Sub(llmgw.EvalSuites, "deploy/eval")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := eval.NewService(eval.Deps{Builtin: sub}).Suites()
|
||||
var names []string
|
||||
for _, s := range got {
|
||||
if s.Error != "" || s.Cases < 5 || s.Judge {
|
||||
t.Errorf("%s: %+v", s.Name, s)
|
||||
}
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
if strings.Join(names, " ") != "arithmetic json-format" {
|
||||
t.Errorf("built in: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCtlEvalSendsAFileAndFailsBelowTheScore(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var sent []string
|
||||
finished := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "mine", Score: 0.5,
|
||||
Threshold: 0.8, Cases: 2, Below: 1, Revision: "abcdef1234567", Repo: "o/r", Results: []store.EvalCase{
|
||||
{ID: "add", Kind: "number", Weight: 1, Score: 1, Passed: true, Answer: "42", Detail: "read 42"},
|
||||
{ID: "mul", Kind: "number", Weight: 1, Score: 0, Answer: "43", Detail: "read 43, wanted 42"}}},
|
||||
State: eval.Done, Done: 2}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
mu.Lock()
|
||||
sent = append(sent, r.Method+" "+r.URL.Path+" "+string(b))
|
||||
mu.Unlock()
|
||||
switch r.URL.Path {
|
||||
case "/api/v1/models/qwen/eval":
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Cases: 2},
|
||||
State: eval.Running})
|
||||
case "/api/v1/eval/runs/eval-1":
|
||||
_ = json.NewEncoder(w).Encode(finished)
|
||||
case "/api/v1/models/qwen/update":
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(modelserve.Job{ID: "update-1", Model: "qwen", State: modelserve.Done})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
file := filepath.Join(t.TempDir(), "mine.yaml")
|
||||
if err := os.WriteFile(file, []byte("cases:\n - prompt: 40 + 2?\n number: 42\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl := func(args ...string) (string, error) {
|
||||
cmd := newCtlCmd()
|
||||
var out bytes.Buffer
|
||||
cmd.SetOut(&out)
|
||||
cmd.SetErr(&out)
|
||||
cmd.SetArgs(append([]string{"--server", srv.URL}, args...))
|
||||
err := cmd.ExecuteContext(context.Background())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
out, err := ctl("eval", "qwen", "--suite", file, "--concurrency", "2")
|
||||
if err == nil || !strings.Contains(err.Error(), "qwen scored 0.50 on mine, below 0.80") {
|
||||
t.Fatalf("a run below its score did not fail: %v\n%s", err, out)
|
||||
}
|
||||
for _, want := range []string{"read 43, wanted 42", "1 of 2 cases passed", "weights o/r at abcdef1"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("the output lacks %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
mu.Lock()
|
||||
var req eval.Request
|
||||
_ = json.Unmarshal([]byte(strings.SplitN(sent[0], " ", 3)[2]), &req)
|
||||
mu.Unlock()
|
||||
if req.File != "mine.yaml" || !strings.Contains(req.Content, "40 + 2?") || req.Concurrency != 2 {
|
||||
t.Errorf("sent %+v", req)
|
||||
}
|
||||
|
||||
// A broken file is said here, before anything is sent.
|
||||
if err := os.WriteFile(file, []byte("cases: []\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ctl("eval", "qwen", "--suite", file); err == nil || !strings.Contains(err.Error(), "no cases") {
|
||||
t.Errorf("%v", err)
|
||||
}
|
||||
|
||||
// An update held to a suite by name carries it.
|
||||
if _, err := ctl("model", "update", "qwen", "--eval", "arithmetic", "--min-score", "0.9"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mu.Lock()
|
||||
last := sent[len(sent)-1]
|
||||
mu.Unlock()
|
||||
var ch modelserve.Change
|
||||
if err := json.Unmarshal([]byte(strings.SplitN(last, " ", 3)[2]), &ch); err != nil || ch.Eval == nil ||
|
||||
ch.Eval.Suite != "arithmetic" || ch.Eval.MinScore != 0.9 {
|
||||
t.Errorf("sent %s", last)
|
||||
}
|
||||
if _, err := ctl("model", "update", "qwen", "--min-score", "0.9"); err == nil {
|
||||
t.Error("--min-score without --eval was sent")
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,9 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(out, "status %d\n", v.Status)
|
||||
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||
if v.CacheHit {
|
||||
fmt.Fprintf(out, "cache answered from the project's answer cache: nothing generated, nothing charged\n")
|
||||
}
|
||||
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
||||
if v.EndUser != "" {
|
||||
fmt.Fprintf(out, "end user %s (the client's own name for them)\n", v.EndUser)
|
||||
|
||||
@@ -110,9 +110,10 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
|
||||
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
if len(f.GPUs) > 0 {
|
||||
fmt.Fprintln(w, "\nGPU\tNAME\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
|
||||
fmt.Fprintln(w, "\nGPU\tNAME\tCLASS\tNUMA\tNVLINK PEERS\tNEAREST ADAPTER")
|
||||
for _, g := range f.GPUs {
|
||||
fmt.Fprintf(w, "%d\t%s\t%d\t%s\t%s\n", g.Index, g.Name, g.NUMANode,
|
||||
// A MIG instance as 0/mig2, under its card.
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%s\t%s\n", discovery.DeviceLabel(g.Index), g.Name, g.Class(), g.NUMANode,
|
||||
orDash(ints(g.NVLinkPeers)), orDash(nearest(g.NICPaths)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,8 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
cfg.VLLM.Bin = bin
|
||||
case "sglang":
|
||||
cfg.SGLang.Python = bin
|
||||
case "trtllm":
|
||||
cfg.TRTLLM.Python = bin
|
||||
case "sim":
|
||||
cfg.Sim.Bin = bin
|
||||
}
|
||||
@@ -97,7 +99,7 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
continue
|
||||
default:
|
||||
var err error
|
||||
if text, err = runHelp(cmd.Context(), c); err != nil {
|
||||
if text, err = runHelp(cmd.Context(), eng, c); err != nil {
|
||||
skipped++
|
||||
fmt.Fprintf(errOut, " skipped: %v\n", err)
|
||||
continue
|
||||
@@ -167,9 +169,10 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
|
||||
// representativeShard is the widest command llmgw would ever build, because a
|
||||
// flag that only appears in the multi-node case is exactly the one nobody
|
||||
// notices has been renamed.
|
||||
// notices has been renamed. For an engine that runs a replica on one node,
|
||||
// that is one node.
|
||||
func representativeShard(engine string, nodes, tp, perNode int) engines.ShardSpec {
|
||||
if nodes < 1 {
|
||||
if nodes < 1 || engine == "trtllm" {
|
||||
nodes = 1
|
||||
}
|
||||
return engines.ShardSpec{
|
||||
@@ -203,9 +206,10 @@ var helpArgs = map[string][]string{
|
||||
}
|
||||
|
||||
// runHelp asks one program to describe itself.
|
||||
func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
func runHelp(ctx context.Context, eng engines.Engine, c engines.Command) (string, error) {
|
||||
// Everything up to the first flag, plus the help request. Passing the real
|
||||
// arguments would start the engine.
|
||||
path := c.Path
|
||||
var args []string
|
||||
for _, a := range c.Args {
|
||||
if strings.HasPrefix(a, "-") {
|
||||
@@ -218,11 +222,16 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
ask = []string{"--help"}
|
||||
}
|
||||
args = append(args, ask...)
|
||||
// An engine started through a launcher of llmgw's is asked through its
|
||||
// own entry point: the interpreter's --help describes the interpreter.
|
||||
if h, ok := eng.(engines.HelpCommand); ok {
|
||||
path, args = h.HelpCommand(c)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
ec := exec.CommandContext(ctx, c.Path, args...)
|
||||
ec := exec.CommandContext(ctx, path, args...)
|
||||
// vLLM refuses to do anything at all on a machine with no accelerator,
|
||||
// including print its help, unless it is told what to assume. Asking an
|
||||
// engine about its flags should not require the hardware it runs on.
|
||||
@@ -231,7 +240,7 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
raw, err := ec.CombinedOutput()
|
||||
if err != nil && len(raw) == 0 {
|
||||
return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)",
|
||||
c.Path, err)
|
||||
path, err)
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -69,3 +74,75 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
|
||||
t.Error("a truncated help was taken for a complete one")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeInterpreter stands in for TensorRT-LLM's Python: asked for
|
||||
// trtllm-serve's help through its module, it prints the help given; asked
|
||||
// anything else, it fails, as running the launcher here would.
|
||||
func fakeInterpreter(t *testing.T, help string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(dir, "python")
|
||||
body := "#!/bin/sh\n" +
|
||||
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
"echo \"unexpected: $*\" >&2; exit 2\n"
|
||||
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// trtllmHelp is the part of trtllm-serve serve --help (1.3) that holds the
|
||||
// flags llmgw builds.
|
||||
const trtllmHelp = `Usage: trtllm-serve serve [OPTIONS] MODEL
|
||||
|
||||
Options:
|
||||
--host TEXT Hostname of the server.
|
||||
--port INTEGER Port of the server.
|
||||
--tensor_parallel_size, --tp_size INTEGER
|
||||
Tensor parallelism size.
|
||||
--pipeline_parallel_size, --pp_size INTEGER
|
||||
Pipeline parallelism size.
|
||||
--config, --extra_llm_api_options TEXT
|
||||
Path to a YAML configuration file.
|
||||
--served_model_name TEXT The model name used in the API.
|
||||
--help Show this message and exit.
|
||||
`
|
||||
|
||||
func runEngineCheck(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
c := newEngineCheckCmd()
|
||||
var out bytes.Buffer
|
||||
c.SetOut(&out)
|
||||
c.SetErr(&out)
|
||||
c.SetArgs(args)
|
||||
err := c.ExecuteContext(context.Background())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
|
||||
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, trtllmHelp))
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "every flag llmgw builds is accepted") {
|
||||
t.Errorf("unexpected report:\n%s", out)
|
||||
}
|
||||
// The interpreter's own help would describe Python, not trtllm-serve; and
|
||||
// one node is what trtllm runs, so no multi-node flag is in the command.
|
||||
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
|
||||
t.Errorf("the help was not asked of trtllm-serve:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
||||
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
||||
// never be seen serving the name the agent waits for.
|
||||
old := strings.ReplaceAll(trtllmHelp, " --served_model_name TEXT The model name used in the API.\n", "")
|
||||
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, old))
|
||||
if err == nil || !strings.Contains(out, "trtllm-serve: --served_model_name") {
|
||||
t.Errorf("an old trtllm-serve passed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,7 @@ func newTenancyCmd() *cobra.Command {
|
||||
newThreadsCmd(open),
|
||||
newRequestLogCmd(open),
|
||||
newResidencyCmd(open),
|
||||
newAnswerCacheCmd(open),
|
||||
newMemoryCmd(open),
|
||||
newMoveStoreCmd(),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
func TestAPlanSeesANodesMIGInstancesAsAClassOfTheirOwn(t *testing.T) {
|
||||
// Card 0 cut into two 3g.40gb instances, both free, and card 1 whole and
|
||||
// busy: the instances are a pool of forty-gigabyte devices, and the busy
|
||||
// card no pool at all.
|
||||
hw := hardwareFrom([]adminapi.NodeView{{ID: "a100", State: "ready", GPUTotal: 3, GPUFree: 2,
|
||||
Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"},
|
||||
ClassGPUs: []adminapi.ClassGPUs{
|
||||
{Class: "a100-80g-mig-3g.40gb", Total: 2, Free: 2, MIG: true},
|
||||
{Class: "a100-80g", Total: 1, Free: 0},
|
||||
}}})
|
||||
if len(hw) != 1 {
|
||||
t.Fatalf("pools: %+v", hw)
|
||||
}
|
||||
want := modelplan.Hardware{Class: "a100-80g-mig-3g.40gb", GPUMemGB: 40, GPUsPerNode: 2, Nodes: 1}
|
||||
if hw[0] != want {
|
||||
t.Errorf("pool %+v, want %+v", hw[0], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCtlNodesCountsANodesDevicesByClassWhenItHasMIG(t *testing.T) {
|
||||
mig := adminapi.NodeView{Classes: []string{"a100-80g-mig-3g.40gb", "a100-80g"}, ClassGPUs: []adminapi.ClassGPUs{
|
||||
{Class: "a100-80g-mig-3g.40gb", Total: 2, MIG: true}, {Class: "a100-80g", Total: 1}}}
|
||||
if got := classesColumn(mig); got != "a100-80g-mig-3g.40gb×2,a100-80g×1" {
|
||||
t.Errorf("MIG node: %q", got)
|
||||
}
|
||||
// A node of one kind of whole card reads as it always did.
|
||||
plain := adminapi.NodeView{Classes: []string{"h100-80g"}, ClassGPUs: []adminapi.ClassGPUs{{Class: "h100-80g", Total: 8}}}
|
||||
if got := classesColumn(plain); got != "h100-80g" {
|
||||
t.Errorf("plain node: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,8 @@ func newMirrorCmd() *cobra.Command {
|
||||
|
||||
c.AddCommand(
|
||||
newMirrorAddCmd(&dir),
|
||||
newMirrorImportCmd(&dir),
|
||||
newMirrorImportOllamaCmd(&dir),
|
||||
newMirrorListCmd(&dir),
|
||||
newMirrorUnusedCmd(&dir),
|
||||
newMirrorRemoveCmd(&dir),
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// placeholderBase is how the mirror's address is written when the operator
|
||||
// has not said it, as `mirror add` writes it.
|
||||
const placeholderBase = "http://<this-host>:8081"
|
||||
|
||||
func newMirrorImportCmd(dir *string) *cobra.Command {
|
||||
var model, version, as, sum, base string
|
||||
|
||||
c := &cobra.Command{
|
||||
Use: "import <file>... --model NAME",
|
||||
Short: "Copy model files on this machine into the mirror",
|
||||
Long: "For weights already here: a GGUF you converted or downloaded by hand,\n" +
|
||||
"or the files of a safetensors model. The same discipline as add — copied\n" +
|
||||
"under a temporary name and renamed when whole, the checksum kept beside\n" +
|
||||
"it for nodes to check against, an existing version never replaced.\n\n" +
|
||||
"A file named .gguf is checked to be one. Several files go into one\n" +
|
||||
"version, as the files of one model; its source is then the version,\n" +
|
||||
"ending in a slash. Run it on the mirror; for a model in Ollama, use\n" +
|
||||
"import-ollama.",
|
||||
Example: " llmgw mirror import ./qwen2.5-7b-instruct-q4_k_m.gguf --model qwen2.5-7b-instruct",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if model == "" {
|
||||
return fmt.Errorf("--model is required: it is the mirror's directory for the model")
|
||||
}
|
||||
if as != "" && len(args) > 1 {
|
||||
return fmt.Errorf("--as names one file; with %d files each keeps its own name", len(args))
|
||||
}
|
||||
if sum != "" && len(args) > 1 {
|
||||
return fmt.Errorf("--sha256 is one file's checksum; with %d files, import them one at a time", len(args))
|
||||
}
|
||||
if version == "" {
|
||||
version = time.Now().Format("2006-01-02")
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
var added []mirror.Added
|
||||
for _, src := range args {
|
||||
abs, err := filepath.Abs(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
file := as
|
||||
if file == "" {
|
||||
file = filepath.Base(src)
|
||||
}
|
||||
e := mirror.Entry{Model: model, Version: version, File: file}
|
||||
fmt.Fprintf(out, "copying %s\n", src)
|
||||
a, err := mirror.Import(cmd.Context(), *dir, abs, e, sum, "file://"+abs, progressTo(out))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n", model, version, file, fileSize(a.Entry.Bytes), a.SHA256)
|
||||
added = append(added, a)
|
||||
}
|
||||
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", importedSource(base, added))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
|
||||
c.Flags().StringVar(&version, "version", "", "the version, one directory level (default today's date)")
|
||||
c.Flags().StringVar(&as, "as", "", "the filename to keep it as, for one file; its own name when omitted")
|
||||
c.Flags().StringVar(&sum, "sha256", "", "the checksum its publisher gave, to refuse a file that does not match")
|
||||
c.Flags().StringVar(&base, "base", placeholderBase, "the address nodes reach this mirror at, for the source printed")
|
||||
return c
|
||||
}
|
||||
|
||||
func newMirrorImportOllamaCmd(dir *string) *cobra.Command {
|
||||
var o ollamaImport
|
||||
|
||||
c := &cobra.Command{
|
||||
Use: "import-ollama [NAME[:TAG]]",
|
||||
Short: "Put a model you already have in Ollama into the mirror, for llama.cpp",
|
||||
Long: "Ollama keeps each model as a GGUF under its models directory, named by\n" +
|
||||
"its sha256. This copies that file into the mirror — checked against the\n" +
|
||||
"digest it is named by, and with a vision model's projector beside it —\n" +
|
||||
"so llama.cpp can serve it on any node, from your own network.\n\n" +
|
||||
"With no name it lists the models Ollama has. The mirror version is the\n" +
|
||||
"tag and the start of the digest, since a tag such as latest moves in\n" +
|
||||
"Ollama and a mirror version must not; --model and --version choose\n" +
|
||||
"otherwise.\n\n" +
|
||||
"Ollama's prompt template is not carried: llama.cpp renders the chat\n" +
|
||||
"template inside the GGUF. Run it on the mirror, where Ollama's directory\n" +
|
||||
"is readable — ~/.ollama/models, /usr/share/ollama/.ollama/models for\n" +
|
||||
"Ollama's Linux service, or OLLAMA_MODELS.",
|
||||
Example: " llmgw mirror import-ollama\n" +
|
||||
" sudo llmgw mirror import-ollama llama3.2:3b --ollama-dir /usr/share/ollama/.ollama/models",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
o.mirrorDir = *dir
|
||||
if len(args) == 1 {
|
||||
o.name = args[0]
|
||||
}
|
||||
return o.run(cmd.Context(), cmd.OutOrStdout())
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&o.ollamaDir, "ollama-dir", "", "Ollama's models directory (default: found where Ollama keeps it)")
|
||||
c.Flags().StringVar(&o.model, "model", "", "the mirror's directory for it (default the Ollama model's name)")
|
||||
c.Flags().StringVar(&o.version, "version", "", "the mirror version (default its tag and the start of its digest)")
|
||||
c.Flags().StringVar(&o.base, "base", placeholderBase, "the address nodes reach this mirror at, for the source printed")
|
||||
return c
|
||||
}
|
||||
|
||||
// ollamaImport is one import-ollama, from the command line or the console.
|
||||
type ollamaImport struct {
|
||||
mirrorDir, ollamaDir, name, model, version, base string
|
||||
// quiet leaves out the copy's progress, for the console, which shows
|
||||
// the output once the import is over.
|
||||
quiet bool
|
||||
}
|
||||
|
||||
func (o ollamaImport) run(ctx context.Context, out io.Writer) error {
|
||||
dir, err := mirror.OllamaDir(o.ollamaDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o.name == "" {
|
||||
names, err := mirror.ListOllama(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(names) == 0 {
|
||||
fmt.Fprintf(out, "Ollama in %s has no models.\n", dir)
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "Ollama's models in %s:\n", dir)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(out, " %s\n", n)
|
||||
}
|
||||
fmt.Fprintln(out, "\nImport one with: llmgw mirror import-ollama NAME:TAG")
|
||||
return nil
|
||||
}
|
||||
|
||||
n, err := mirror.ParseOllamaName(o.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := mirror.ReadOllama(dir, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "importing %s from %s\n", n, dir)
|
||||
last := ""
|
||||
added, err := mirror.ImportOllama(ctx, o.mirrorDir, dir, m, o.model, o.version, func(file string, done, total int64) {
|
||||
if file != last {
|
||||
fmt.Fprintf(out, "copying %s\n", file)
|
||||
last = file
|
||||
}
|
||||
if !o.quiet {
|
||||
progressTo(out)(done, total)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range added {
|
||||
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s, as Ollama named it\n",
|
||||
a.Entry.Model, a.Entry.Version, a.Entry.File, fileSize(a.Entry.Bytes), a.SHA256)
|
||||
}
|
||||
for _, l := range m.Left {
|
||||
fmt.Fprintf(out, "Not carried: %s.\n", l)
|
||||
}
|
||||
|
||||
name := n.Model
|
||||
if n.Tag != "latest" {
|
||||
name += "-" + n.Tag
|
||||
}
|
||||
fmt.Fprintf(out, "\nA model definition for it, for llmgw ctl model add -f:\n\n"+
|
||||
"name: %s\nsource: %s\nprofiles:\n - name: cpu\n engine: llamacpp\n",
|
||||
name, importedSource(o.base, added))
|
||||
if m.ContextLength > 0 {
|
||||
// Ollama's context, which llama.cpp's own default would not match;
|
||||
// llama.cpp divides it between its parallel slots.
|
||||
fmt.Fprintf(out, " args: [\"--ctx-size\", %q] # Ollama ran it with num_ctx %d\n",
|
||||
strconv.Itoa(m.ContextLength), m.ContextLength)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importedSource is what a model's source says for what was just imported:
|
||||
// the file, or the version's directory when it is several.
|
||||
func importedSource(base string, added []mirror.Added) string {
|
||||
if base == "" {
|
||||
base = placeholderBase
|
||||
}
|
||||
if len(added) == 1 {
|
||||
return added[0].Entry.URL(base)
|
||||
}
|
||||
e := added[0].Entry
|
||||
return strings.TrimSuffix(mirror.Entry{Model: e.Model, Version: e.Version}.URL(base), "/") + "/"
|
||||
}
|
||||
|
||||
// progressTo reports a copy's progress the way add reports a download's.
|
||||
func progressTo(out io.Writer) func(done, total int64) {
|
||||
return func(done, total int64) {
|
||||
if total > 0 {
|
||||
fmt.Fprintf(out, " %s of %s (%d%%)\n", fileSize(done), fileSize(total), done*100/total)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s so far\n", fileSize(done))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
// runMirror runs `llmgw mirror ...` as an operator would and returns what it
|
||||
// printed.
|
||||
func runMirror(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
c := Root()
|
||||
var out bytes.Buffer
|
||||
c.SetOut(&out)
|
||||
c.SetErr(&out)
|
||||
c.SetArgs(append([]string{"mirror"}, args...))
|
||||
err := c.ExecuteContext(t.Context())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
// ollamaWith lays out an Ollama models directory holding one model, as
|
||||
// Ollama writes it, and returns the directory and the weights' sha256.
|
||||
func ollamaWith(t *testing.T, name, tag string, weights []byte, params string) (string, string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(t.TempDir(), "models")
|
||||
blob := func(content []byte) string {
|
||||
s := sha256.Sum256(content)
|
||||
h := hex.EncodeToString(s[:])
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "blobs", "sha256-"+h), content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
w := blob(weights)
|
||||
p := blob([]byte(params))
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json",` +
|
||||
`"config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:` + blob([]byte("{}")) + `","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:` + w + `","size":` + strconv.Itoa(len(weights)) + `},` +
|
||||
`{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:` + p + `","size":` + strconv.Itoa(len(params)) + `}]}`
|
||||
mdir := filepath.Join(dir, "manifests", "registry.ollama.ai", "library", name)
|
||||
if err := os.MkdirAll(mdir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(mdir, tag), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir, w
|
||||
}
|
||||
|
||||
func TestImportOllamaListsThenImportsAModelForLlamaCpp(t *testing.T) {
|
||||
weights := []byte("GGUF\x03\x00\x00\x00qwen weights")
|
||||
ollama, sum := ollamaWith(t, "qwen2.5", "7b", weights, `{"num_ctx":16384}`)
|
||||
root := t.TempDir()
|
||||
|
||||
out, err := runMirror(t, "--dir", root, "import-ollama", "--ollama-dir", ollama)
|
||||
if err != nil || !strings.Contains(out, " qwen2.5:7b\n") {
|
||||
t.Fatalf("listing: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
out, err = runMirror(t, "--dir", root, "import-ollama", "qwen2.5:7b", "--ollama-dir", ollama,
|
||||
"--base", "http://models.internal:8081")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
file := filepath.Join(root, "qwen2.5", "7b-"+sum[:12], "qwen2.5-7b.gguf")
|
||||
if b, err := os.ReadFile(file); err != nil || !bytes.Equal(b, weights) {
|
||||
t.Fatalf("the GGUF is not in the mirror at %s: %v", file, err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"sha256 " + sum,
|
||||
"source: http://models.internal:8081/qwen2.5/7b-" + sum[:12] + "/qwen2.5-7b.gguf",
|
||||
"engine: llamacpp",
|
||||
`args: ["--ctx-size", "16384"]`,
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("the output does not say %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleImportsFromOllamaIntoTheMirrorBesideIt(t *testing.T) {
|
||||
weights := []byte("GGUF\x03\x00\x00\x00llama weights")
|
||||
ollama, sum := ollamaWith(t, "llama3.2", "3b", weights, `{}`)
|
||||
root := t.TempDir()
|
||||
|
||||
m, _ := consoleWith(t, cMirror)
|
||||
var tm tea.Model = press(m, "i")
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.title != "Import from Ollama" {
|
||||
t.Fatalf("I opened %+v", f)
|
||||
}
|
||||
f.fields[0].text.SetValue("llama3.2:3b")
|
||||
f.fields[1].text.SetValue(ollama)
|
||||
f.fields[2].text.SetValue(root)
|
||||
for range len(f.fields) {
|
||||
tm = press(tm, "tab")
|
||||
}
|
||||
tm = pressRun(tm, "enter")
|
||||
|
||||
if _, err := os.Stat(filepath.Join(root, "llama3.2", "3b-"+sum[:12], "llama3.2-3b.gguf")); err != nil {
|
||||
t.Fatalf("not imported: %v", err)
|
||||
}
|
||||
text := tm.(consoleModel).dialogText
|
||||
if !strings.Contains(text, "engine: llamacpp") || strings.Contains(text, "(100%)") {
|
||||
t.Errorf("the console said:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCopiesALocalGGUFIntoTheMirror(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "q4_k_m.gguf")
|
||||
content := []byte("GGUF\x03\x00\x00\x00converted")
|
||||
if err := os.WriteFile(src, content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
out, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(root, "qwen", "2026-10", "q4_k_m.gguf")); err != nil || !bytes.Equal(b, content) {
|
||||
t.Fatalf("not imported: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "source: http://<this-host>:8081/qwen/2026-10/q4_k_m.gguf") {
|
||||
t.Errorf("no source given:\n%s", out)
|
||||
}
|
||||
// The layout is the point: a version is never replaced.
|
||||
if _, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10"); err == nil {
|
||||
t.Error("an existing version was replaced")
|
||||
}
|
||||
if _, err := runMirror(t, "--dir", root, "import", src); err == nil {
|
||||
t.Error("imported without --model")
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,8 @@ func newModelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
newTargetCmd(client, asJSON), newAdaptersCmd(client),
|
||||
newModelServeCmd(client), newModelUpdatesCmd(client, asJSON), newModelUpdateCmd(client),
|
||||
newModelRollbackCmd(client), newModelOriginCmd(client),
|
||||
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON))
|
||||
newModelCanaryCmd(client), newModelPromoteCmd(client), newModelCanariesCmd(client, asJSON),
|
||||
newModelGateCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -83,17 +83,29 @@ func newModelCanaryCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
|
||||
func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
|
||||
var detach bool
|
||||
var (
|
||||
detach bool
|
||||
gate gateFlags
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "promote MODEL",
|
||||
Short: "Move a model to its canary's weights, and take the canary out",
|
||||
Long: "The update the canary was a trial of: the catalog names the canary's\n" +
|
||||
"files, the model's replicas are replaced one at a time, and the canary\n" +
|
||||
"is removed. `llmgw ctl model rollback MODEL` goes back.",
|
||||
"is removed. `llmgw ctl model rollback MODEL` goes back.\n\n" +
|
||||
"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
|
||||
"gate`), the canary is asked it first, and below its score nothing\n" +
|
||||
"moves: the canary stays, with its share, until it is taken out.",
|
||||
Example: " llmgw ctl model promote qwen-7b\n" +
|
||||
" llmgw ctl model promote qwen-7b --eval ./support.yaml --min-score 0.9 --judge llama-70b",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, err := gate.body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", []byte("{}"), &job); err != nil {
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/promote", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
if detach {
|
||||
@@ -103,6 +115,7 @@ func newModelPromoteCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
|
||||
gate.add(c)
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -82,7 +82,10 @@ func updateCell(u modelserve.UpdateInfo) string {
|
||||
}
|
||||
|
||||
func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
|
||||
var detach bool
|
||||
var (
|
||||
detach bool
|
||||
gate gateFlags
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "update MODEL",
|
||||
Short: "Move a model to the weights its repository publishes now",
|
||||
@@ -90,12 +93,23 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
|
||||
"checked against the registry's checksums and the nodes' llama.cpp; then\n" +
|
||||
"the catalog names them and the replicas are replaced one at a time, each\n" +
|
||||
"new one up before an old one stops, and one is asked to say hello.\n\n" +
|
||||
"Held to a quality suite (--eval, or the model's own: `llmgw ctl model\n" +
|
||||
"gate`), the new weights first run beside the old as MODEL-canary, given\n" +
|
||||
"none of its requests, and are asked the suite; below its score they are\n" +
|
||||
"taken out again and the model keeps its weights. That needs room for one\n" +
|
||||
"more replica while it runs.\n\n" +
|
||||
"The old files stay on the mirror: `llmgw ctl model rollback MODEL` goes\n" +
|
||||
"back to them. A model from server.yaml is changed there instead.",
|
||||
Example: " llmgw ctl model update qwen-7b\n" +
|
||||
" llmgw ctl model update qwen-7b --eval arithmetic --min-score 0.8",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
body, err := gate.body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", []byte("{}"), &job); err != nil {
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/"+args[0]+"/update", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
if detach {
|
||||
@@ -105,6 +119,7 @@ func newModelUpdateCmd(client func() *ctlClient) *cobra.Command {
|
||||
},
|
||||
}
|
||||
c.Flags().BoolVar(&detach, "detach", false, "start it and return: it runs on the control plane; `llmgw ctl jobs` follows it")
|
||||
gate.add(c)
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
+35
-32
@@ -155,28 +155,41 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
|
||||
if len(n.Fabrics) > 0 {
|
||||
fabric = n.Fabrics[0]
|
||||
}
|
||||
class := ""
|
||||
if len(n.Classes) > 0 {
|
||||
class = n.Classes[0]
|
||||
// Each class of the node's devices is a pool of its own: a node with
|
||||
// a card in MIG mode has its instances' classes, and any whole card's
|
||||
// beside them. A node that does not count by class is its first
|
||||
// class, as it always was.
|
||||
kinds := n.ClassGPUs
|
||||
if len(kinds) == 0 {
|
||||
class := ""
|
||||
if len(n.Classes) > 0 {
|
||||
class = n.Classes[0]
|
||||
}
|
||||
kinds = []adminapi.ClassGPUs{{Class: class, Total: n.GPUTotal, Free: n.GPUFree}}
|
||||
}
|
||||
k := key{class, fabric}
|
||||
if pools[k] == nil {
|
||||
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
|
||||
}
|
||||
p := pools[k]
|
||||
p.Nodes++
|
||||
// The widest a single node could contribute, not the total: a tensor
|
||||
// group takes the same number of cards from every node it spans.
|
||||
if n.GPUFree > p.GPUsPerNode {
|
||||
p.GPUsPerNode = n.GPUFree
|
||||
}
|
||||
if n.MemoryGB > p.HostMemGB {
|
||||
p.HostMemGB = n.MemoryGB
|
||||
}
|
||||
// Card memory from the class label, which carries it by construction:
|
||||
// "h100-80g" is eighty gigabytes and says so.
|
||||
if class != "" && p.GPUMemGB == 0 {
|
||||
p.GPUMemGB = memFromClass(class)
|
||||
for _, kind := range kinds {
|
||||
class := kind.Class
|
||||
k := key{class, fabric}
|
||||
if pools[k] == nil {
|
||||
pools[k] = &modelplan.Hardware{Class: class, Fabric: fabric}
|
||||
}
|
||||
p := pools[k]
|
||||
p.Nodes++
|
||||
// The widest a single node could contribute, not the total: a
|
||||
// tensor group takes the same number of cards from every node it
|
||||
// spans.
|
||||
if kind.Free > p.GPUsPerNode {
|
||||
p.GPUsPerNode = kind.Free
|
||||
}
|
||||
if n.MemoryGB > p.HostMemGB {
|
||||
p.HostMemGB = n.MemoryGB
|
||||
}
|
||||
// Card memory from the class label, which carries it by
|
||||
// construction: "h100-80g" is eighty gigabytes and says so, and
|
||||
// "a100-80g-mig-3g.40gb" is an instance of forty.
|
||||
if class != "" && p.GPUMemGB == 0 {
|
||||
p.GPUMemGB = memFromClass(class)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,17 +204,7 @@ func hardwareFrom(nodes []adminapi.NodeView) []modelplan.Hardware {
|
||||
}
|
||||
|
||||
// memFromClass reads the card size out of a class label such as "h100-80g".
|
||||
func memFromClass(class string) int {
|
||||
i := strings.LastIndex(class, "-")
|
||||
if i < 0 || !strings.HasSuffix(class, "g") {
|
||||
return 0
|
||||
}
|
||||
var gb int
|
||||
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
|
||||
return 0
|
||||
}
|
||||
return gb
|
||||
}
|
||||
func memFromClass(class string) int { return modelplan.MemFromClass(class) }
|
||||
|
||||
// catalogEntry renders a fit as the YAML that would serve it.
|
||||
func catalogEntry(m modelplan.Model, f modelplan.Fit, source string, context int) string {
|
||||
|
||||
+12
-2
@@ -9,6 +9,7 @@ import (
|
||||
"maps"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"slices"
|
||||
"strings"
|
||||
@@ -32,6 +33,7 @@ import (
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/audit"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
@@ -102,10 +104,12 @@ func newServerCmd() *cobra.Command {
|
||||
"or give --endpoint for a router-only run")
|
||||
}
|
||||
|
||||
// Before the router, so its tracer is the configured one.
|
||||
// Before the router, so its tracer is the configured one. The
|
||||
// host tells two control planes' traces apart.
|
||||
host, _ := os.Hostname()
|
||||
shutdownTracing, err := obs.SetupTracing(cmd.Context(), obs.TracingOptions{
|
||||
Endpoint: cfg.Tracing.Endpoint, SampleRatio: cfg.Tracing.SampleRatio,
|
||||
Service: "llmgw-server", Version: version.Version,
|
||||
Service: "llmgw-server", Version: version.Version, Instance: host,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -286,6 +290,7 @@ func newServerCmd() *cobra.Command {
|
||||
var (
|
||||
adminAPI *adminapi.API
|
||||
catalogSvc *modelsvc.Service
|
||||
evals *eval.Service
|
||||
)
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
@@ -303,6 +308,9 @@ func newServerCmd() *cobra.Command {
|
||||
api.SetReleases(cfg.Releases)
|
||||
api.SetNotifier(notifier)
|
||||
api.SetReplayer(rt.Replay)
|
||||
// Quality suites, asked of a model the way a replay is.
|
||||
evals = newEvalService(cfg, rt, admin, log)
|
||||
api.SetEval(evals)
|
||||
if st := rt.Store(); st != nil && gate != nil {
|
||||
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
|
||||
}
|
||||
@@ -419,6 +427,8 @@ func newServerCmd() *cobra.Command {
|
||||
Pool: adminAPI.PoolNodes, Notify: notifier,
|
||||
// Canaries: how the router saw each side serve.
|
||||
CanaryStats: rt.CanaryStats, ResetCanary: rt.ResetCanary,
|
||||
// Updates and promotions held to a quality suite.
|
||||
Eval: evals,
|
||||
})
|
||||
go serve.WatchCanaries(ctx, 30*time.Second)
|
||||
adminAPI.SetServe(serve)
|
||||
|
||||
@@ -423,6 +423,7 @@ func (m consoleModel) modelMenu(md adminapi.ModelView) *menuDialog {
|
||||
}},
|
||||
}
|
||||
items = append(items, m.updateItems(md)...)
|
||||
items = append(items, m.evalItems(md)...)
|
||||
if md.Parked {
|
||||
items = append(items, menuItem{"Try again (it is parked)", func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Retry "+md.Name, http.MethodPost, path+"/retry", map[string]string{}, nil)
|
||||
@@ -706,7 +707,7 @@ func (m consoleModel) canaryItems(md adminapi.ModelView) []menuItem {
|
||||
{"Promote the canary…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Promote " + cn.Model,
|
||||
text: md.Name + " moves to the canary's weights: its replicas are replaced one at a time,\n" +
|
||||
"and the canary is removed. Going back is \"Go back to\" on this menu.",
|
||||
"and the canary is removed. Going back is \"Go back to\" on this menu." + gateNote(md),
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Promote "+cn.Model, http.MethodPost, path+"/promote", map[string]string{}, startedSays)
|
||||
}}
|
||||
@@ -780,8 +781,8 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
|
||||
m.confirmD = &confirmDialog{title: "Update " + md.Name,
|
||||
text: fmt.Sprintf("%s has new weights at %s: %s.\n\n"+
|
||||
"They are fetched into the mirror beside the ones in use and checked, then the\n"+
|
||||
"replicas are replaced one at a time. The old files stay, to go back to.",
|
||||
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", ")),
|
||||
"replicas are replaced one at a time. The old files stay, to go back to.%s",
|
||||
u.Repo, catalog.ShortRevision(u.Latest), strings.Join(u.Changed, ", "), gateNote(md)),
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Update "+md.Name, http.MethodPost, "/api/v1/models/"+url.PathEscape(md.Name)+"/update",
|
||||
map[string]string{}, startedSays)
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -154,6 +155,11 @@ type consoleModel struct {
|
||||
formD *formDialog
|
||||
confirmD *confirmDialog
|
||||
busy string
|
||||
// busyNote says how far the request under way has got, when it can;
|
||||
// evalWatch is the quality run being followed, which Esc stops
|
||||
// following while it carries on.
|
||||
busyNote string
|
||||
evalWatch string
|
||||
|
||||
quotas []adminapi.QuotaView
|
||||
quotasErr error
|
||||
@@ -337,8 +343,12 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
m.people = peopleData(msg)
|
||||
case consoleQuotasMsg:
|
||||
m.quotas, m.quotasErr = msg.rows, msg.err
|
||||
case evalSuitesMsg:
|
||||
return m.onEvalSuites(msg)
|
||||
case evalPollMsg:
|
||||
return m.onEvalPoll(msg)
|
||||
case actionDoneMsg:
|
||||
m.busy = ""
|
||||
m.busy, m.busyNote, m.evalWatch = "", "", ""
|
||||
m.dialog, m.dialogTitle = "error", msg.title
|
||||
if msg.err != nil {
|
||||
m.dialogText = sBad.Render("✗ ") + msg.err.Error()
|
||||
@@ -367,6 +377,9 @@ func (m consoleModel) storeReplicated() bool { return len(m.data.sum.Store) > 0
|
||||
|
||||
func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
if m.busy != "" {
|
||||
if m.evalWatch != "" && k.Type == tea.KeyEsc {
|
||||
m.busy, m.busyNote, m.evalWatch = "", "", ""
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
if m.dialog != "" {
|
||||
@@ -536,6 +549,10 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
case cOrgs:
|
||||
m.createOrgForm()
|
||||
}
|
||||
case "i":
|
||||
if m.menu == cMirror {
|
||||
m.mirrorImportOllamaForm()
|
||||
}
|
||||
case "n":
|
||||
if m.menu == cMonitor {
|
||||
return m, m.runCLI("Test the webhooks", "ctl", "--server="+m.client.base, "notify", "test")
|
||||
@@ -973,7 +990,11 @@ func (m consoleModel) View() string {
|
||||
} else if t, d := m.docs.dialog(); d != "" && m.menu == cManual {
|
||||
f.DialogTitle, f.Dialog = t, d
|
||||
} else if m.busy != "" {
|
||||
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" asking the control plane…"
|
||||
note := "asking the control plane…"
|
||||
if m.busyNote != "" {
|
||||
note = m.busyNote
|
||||
}
|
||||
f.DialogTitle, f.Dialog = m.busy, m.spin.View()+" "+note
|
||||
} else if t, d := m.actionDialog(); d != "" {
|
||||
f.DialogTitle, f.Dialog = t, d
|
||||
}
|
||||
@@ -1080,7 +1101,7 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
for _, r := range m.data.replicas {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, s.GPUs))
|
||||
nodes = append(nodes, fmt.Sprintf("%s%v", s.Node, discovery.DeviceLabels(s.GPUs)))
|
||||
}
|
||||
rows = append(rows, []string{r.ID, r.Model, r.State, r.Pool, strings.Join(nodes, " "),
|
||||
(time.Duration(r.AgeSec) * time.Second).Round(time.Second).String()})
|
||||
@@ -1224,14 +1245,31 @@ func (m consoleModel) gpuRows() [][]string {
|
||||
for _, n := range m.data.nodes {
|
||||
for _, g := range n.GPUs {
|
||||
busy, mem, temp, power := "—", "—", "—", "—"
|
||||
// A MIG instance under its card: its own memory, the card's heat
|
||||
// and power, and no utilisation, which no driver measures per
|
||||
// instance — said, rather than shown as the card's or as zero.
|
||||
mig := g.MIGProfile != ""
|
||||
if mig {
|
||||
busy = sDim.Render("n/a on MIG")
|
||||
if g.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", float64(g.MemoryUsedMB)/1024, float64(g.MemoryTotalMB)/1024)
|
||||
}
|
||||
}
|
||||
card := g.Card
|
||||
if g.Label == "" {
|
||||
// A control plane older than MIG, which sends no card.
|
||||
card = g.Index
|
||||
}
|
||||
if n.Load != nil {
|
||||
for _, s := range n.Load.GPUs {
|
||||
if s.Index != g.Index {
|
||||
if s.Index != card {
|
||||
continue
|
||||
}
|
||||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||||
if s.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||||
if !mig {
|
||||
busy = fmt.Sprintf("%.0f%%", s.UtilPercent)
|
||||
if s.MemoryTotalMB > 0 {
|
||||
mem = fmt.Sprintf("%.1f/%.0f GB", s.MemoryUsedMB/1024, s.MemoryTotalMB/1024)
|
||||
}
|
||||
}
|
||||
if s.TempC > 0 {
|
||||
temp = fmt.Sprintf("%.0f°C", s.TempC)
|
||||
@@ -1245,7 +1283,12 @@ func (m consoleModel) gpuRows() [][]string {
|
||||
if holder == "" {
|
||||
holder = sDim.Render("free")
|
||||
}
|
||||
rows = append(rows, []string{n.ID, strconv.Itoa(g.Index), g.Name, busy, mem, temp, power, holder})
|
||||
label := g.Label
|
||||
if label == "" {
|
||||
// A control plane older than labels.
|
||||
label = strconv.Itoa(g.Index)
|
||||
}
|
||||
rows = append(rows, []string{n.ID, label, g.Name, busy, mem, temp, power, holder})
|
||||
}
|
||||
}
|
||||
return rows
|
||||
@@ -1266,6 +1309,15 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
|
||||
}
|
||||
kv("disk", fmt.Sprintf("%d GB free of %d", n.DiskFreeGB, n.DiskTotalGB))
|
||||
kv("replicas", strconv.Itoa(n.Replicas))
|
||||
// By class, which on a node with a card in MIG mode is the instances'
|
||||
// profiles beside any whole card: what a profile's gpu_class can name.
|
||||
for _, c := range n.ClassGPUs {
|
||||
what := fmt.Sprintf("%d, %d free", c.Total, c.Free)
|
||||
if c.MIG {
|
||||
what += " · MIG instances, one to a replica"
|
||||
}
|
||||
kv(c.Class, what)
|
||||
}
|
||||
if len(n.Provision) > 0 {
|
||||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("Engines") + "\n")
|
||||
for _, p := range n.Provision {
|
||||
|
||||
@@ -2,7 +2,6 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -24,60 +23,17 @@ type docTopic struct {
|
||||
src func() string
|
||||
}
|
||||
|
||||
// docOrder is the documents in the order someone new reads them.
|
||||
var docOrder = []string{"console", "quick-install", "install", "node-onboarding", "onboarding",
|
||||
"guardrails", "choosing-models", "fabrics", "facility-aware-placement", "data", "reference", "development"}
|
||||
|
||||
// docTopics is the manual: the documents in llmgw.Manual's order, which the
|
||||
// gateway's docs portal shares, with a page for the commands after the
|
||||
// configuration reference — the one page only the command tree can write.
|
||||
func docTopics() []docTopic {
|
||||
read := func(path string) func() string {
|
||||
return func() string {
|
||||
b, err := fs.ReadFile(llmgw.Docs, path)
|
||||
if err != nil {
|
||||
return "# " + path + "\n\nNot in this build."
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
}
|
||||
titleOf := func(path string) string {
|
||||
b, _ := fs.ReadFile(llmgw.Docs, path)
|
||||
for _, l := range strings.Split(string(b), "\n") {
|
||||
if strings.HasPrefix(l, "# ") {
|
||||
return strings.TrimPrefix(l, "# ")
|
||||
}
|
||||
}
|
||||
return path
|
||||
}
|
||||
var out []docTopic
|
||||
seen := map[string]bool{}
|
||||
add := func(key, path string) {
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
if _, err := fs.Stat(llmgw.Docs, path); err != nil {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, docTopic{key: key, title: titleOf(path), src: read(path)})
|
||||
}
|
||||
for _, k := range docOrder {
|
||||
add(k, "docs/"+k+".md")
|
||||
if k == "reference" {
|
||||
for _, d := range llmgw.Manual() {
|
||||
out = append(out, docTopic{key: d.Key, title: d.Title, src: d.Source})
|
||||
if d.Key == "reference" {
|
||||
out = append(out, docTopic{key: "commands", title: "Commands", src: commandsManual})
|
||||
seen["commands"] = true
|
||||
}
|
||||
}
|
||||
// Any document not placed above, so a new one is never missing.
|
||||
entries, _ := fs.ReadDir(llmgw.Docs, "docs")
|
||||
var rest []string
|
||||
for _, e := range entries {
|
||||
rest = append(rest, strings.TrimSuffix(e.Name(), ".md"))
|
||||
}
|
||||
sort.Strings(rest)
|
||||
for _, k := range rest {
|
||||
add(k, "docs/"+k+".md")
|
||||
}
|
||||
add("readme", "README.md")
|
||||
add("changes", "CHANGELOG.md")
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -178,10 +179,41 @@ func (m *consoleModel) mirrorFetchForm() {
|
||||
}})
|
||||
}
|
||||
|
||||
// mirrorImportOllamaForm puts a model from an Ollama on this machine into a
|
||||
// mirror on this machine: the control plane's own, when the mirror runs
|
||||
// beside it. A mirror elsewhere cannot read this machine's files, and its
|
||||
// API is given none; there it is `llmgw mirror import-ollama`, on the mirror.
|
||||
func (m *consoleModel) mirrorImportOllamaForm() {
|
||||
m.openForm(&formDialog{title: "Import from Ollama",
|
||||
note: "From an Ollama on this machine into the mirror's directory on this machine.\n" +
|
||||
"A mirror elsewhere: run llmgw mirror import-ollama there.",
|
||||
fields: []formField{
|
||||
textF("model", "", "as Ollama names it, like llama3.2:3b"),
|
||||
textF("ollama dir", "", "optional: found where Ollama keeps it"),
|
||||
textF("mirror dir", "/srv/models", "the mirror's directory on this machine"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if v[0] == "" {
|
||||
return nil, fmt.Errorf("a model, as Ollama names it")
|
||||
}
|
||||
if st, err := os.Stat(v[2]); err != nil || !st.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory on this machine", v[2])
|
||||
}
|
||||
title := "Import " + v[0]
|
||||
m.busy = title
|
||||
o := ollamaImport{name: v[0], ollamaDir: v[1], mirrorDir: v[2], base: m.mirror.res.Base, quiet: true}
|
||||
return func() tea.Msg {
|
||||
var out bytes.Buffer
|
||||
err := o.run(context.Background(), &out)
|
||||
return actionDoneMsg{title: title, text: strings.TrimSpace(out.String()), err: err}
|
||||
}, nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter actions · A download one · ← back · Q leave"
|
||||
keys = "↑↓ row · Enter actions · A download one · I import from Ollama · ← back · Q leave"
|
||||
}
|
||||
if m.mirror.err != nil {
|
||||
return "Mirror", "\n " + sBad.Render(m.mirror.err.Error()), keys
|
||||
@@ -216,7 +248,7 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one; I imports one from Ollama."), keys
|
||||
}
|
||||
return "Mirror", head + table([]string{"MODEL", "VERSION", "FILE", "STATE", "SIZE", "USED BY"}, rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
|
||||
@@ -72,6 +72,8 @@ type peopleData struct {
|
||||
notes map[string]int // project id → hours kept, agent notes
|
||||
// requests is how each project keeps its requests and answers.
|
||||
requests map[string]store.RequestLog
|
||||
// cache is how each project reuses answers.
|
||||
cache map[string]store.AnswerCache
|
||||
// guardPolicies are server.yaml's guardrail policies, guardDefault those
|
||||
// applied everywhere, and guardAt which are attached where.
|
||||
guardPolicies []string
|
||||
@@ -125,6 +127,7 @@ func (m consoleModel) fetchPeople() tea.Cmd {
|
||||
d.guardAt, _ = s.GuardAttachments(ctx)
|
||||
}
|
||||
d.threads, d.notes, d.requests = map[string]int{}, map[string]int{}, map[string]store.RequestLog{}
|
||||
d.cache = map[string]store.AnswerCache{}
|
||||
for _, r := range d.orgs {
|
||||
if r.level != "project" {
|
||||
continue
|
||||
@@ -138,6 +141,9 @@ func (m consoleModel) fetchPeople() tea.Cmd {
|
||||
if l, err := s.RequestLogOf(ctx, r.id); err == nil {
|
||||
d.requests[r.id] = l
|
||||
}
|
||||
if c, err := s.AnswerCacheOf(ctx, r.id); err == nil {
|
||||
d.cache[r.id] = c
|
||||
}
|
||||
}
|
||||
return consolePeopleMsg(d)
|
||||
}
|
||||
@@ -552,6 +558,39 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
|
||||
return m.runCLI("Keep requests", append(args, "--retain-hours="+v[0], "--sample="+v[1])...), nil
|
||||
}})
|
||||
return nil
|
||||
}},
|
||||
menuItem{"Cache answers…", func(m *consoleModel) tea.Cmd {
|
||||
c := m.people.cache[r.id]
|
||||
ttl, which := "off", "reproducible"
|
||||
if c.TTL > 0 {
|
||||
ttl = c.TTL.String()
|
||||
}
|
||||
if c.Any {
|
||||
which = "any"
|
||||
}
|
||||
m.openForm(&formDialog{title: "Cache answers · " + r.path,
|
||||
note: "Off by default. On, the same request to the same model within the time below is\n" +
|
||||
"answered again from the gateway's memory, without the model, and costs nothing.\n" +
|
||||
"Reproducible: only temperature 0 or a seed, and embeddings. Any: sampled answers too.",
|
||||
fields: []formField{
|
||||
textF("reuse for", ttl, "like 1h; off reuses none"),
|
||||
choiceF("answers", []string{"reproducible", "any"}, which),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
args := []string{"keys", "cache", "--project=" + r.id}
|
||||
if v[0] == "off" || v[0] == "0" {
|
||||
return m.runCLI("Cache answers", append(args, "--off")...), nil
|
||||
}
|
||||
if d, err := time.ParseDuration(v[0]); err != nil || d < time.Second {
|
||||
return nil, fmt.Errorf("a duration like 1h, or off")
|
||||
}
|
||||
args = append(args, "--ttl="+v[0])
|
||||
if v[1] == "any" {
|
||||
args = append(args, "--any")
|
||||
}
|
||||
return m.runCLI("Cache answers", args...), nil
|
||||
}})
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
|
||||
|
||||
@@ -0,0 +1,291 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// Quality suites from the model menu: ask a model one of the suites the
|
||||
// control plane has (its eval directory's, and those built in) and watch
|
||||
// its score come in; list its past runs; hold its updates to one — the same
|
||||
// as `llmgw ctl eval` and `llmgw ctl model gate`.
|
||||
|
||||
// evalSuitesMsg is the suites the control plane can run, fetched to open
|
||||
// the form that runs one (or, with gate, the one that sets a model's).
|
||||
type evalSuitesMsg struct {
|
||||
model adminapi.ModelView
|
||||
suites []eval.SuiteInfo
|
||||
gate bool
|
||||
err error
|
||||
}
|
||||
|
||||
// evalPollMsg is a run still going, to be asked about again.
|
||||
type evalPollMsg struct {
|
||||
title string
|
||||
live eval.Live
|
||||
err error
|
||||
}
|
||||
|
||||
// gateNote says, in an update's or a promotion's question, that the new
|
||||
// weights are asked the model's suite first.
|
||||
func gateNote(md adminapi.ModelView) string {
|
||||
g := md.Eval
|
||||
if g == nil {
|
||||
return ""
|
||||
}
|
||||
at := "its threshold"
|
||||
if g.MinScore > 0 {
|
||||
at = fmt.Sprintf("%.2f", g.MinScore)
|
||||
}
|
||||
return fmt.Sprintf("\n\nFirst the new weights are asked the quality suite %s, and must score %s;\n"+
|
||||
"below it nothing moves.", g.Suite, at)
|
||||
}
|
||||
|
||||
// evalItems are the model menu's quality entries.
|
||||
func (m consoleModel) evalItems(md adminapi.ModelView) []menuItem {
|
||||
gate := "Hold its updates to a quality suite…"
|
||||
if g := md.Eval; g != nil {
|
||||
gate = "Its updates' quality suite (" + g.Suite + ")…"
|
||||
}
|
||||
return []menuItem{
|
||||
{"Ask it a quality suite…", func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, false) }},
|
||||
{"Its quality runs", func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Quality runs of "+md.Name, http.MethodGet,
|
||||
"/api/v1/eval/runs?limit=15&model="+url.QueryEscape(md.Name), nil, func(raw json.RawMessage) string {
|
||||
var runs []eval.Live
|
||||
if err := json.Unmarshal(raw, &runs); err != nil {
|
||||
return string(raw)
|
||||
}
|
||||
return evalRunsText(runs)
|
||||
})
|
||||
}},
|
||||
{gate, func(m *consoleModel) tea.Cmd { return m.fetchSuites(md, true) }},
|
||||
}
|
||||
}
|
||||
|
||||
func (m *consoleModel) fetchSuites(md adminapi.ModelView, gate bool) tea.Cmd {
|
||||
m.busy = "Quality suites"
|
||||
c := m.client
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
var list []eval.SuiteInfo
|
||||
err := c.get(ctx, "/api/v1/eval/suites", &list)
|
||||
return evalSuitesMsg{model: md, suites: list, gate: gate, err: err}
|
||||
}
|
||||
}
|
||||
|
||||
func (m consoleModel) onEvalSuites(msg evalSuitesMsg) (tea.Model, tea.Cmd) {
|
||||
m.busy = ""
|
||||
md := msg.model
|
||||
var names []string
|
||||
for _, s := range msg.suites {
|
||||
if s.Error == "" {
|
||||
names = append(names, s.Name)
|
||||
}
|
||||
}
|
||||
if msg.err == nil && len(names) == 0 {
|
||||
msg.err = errors.New("the control plane has no suite it can run: put one in its control.eval_dir " +
|
||||
"(/etc/llmgw/eval); docs/install.md, \"Quality suites\", says how one is written")
|
||||
}
|
||||
if msg.err != nil {
|
||||
m.dialog, m.dialogTitle = "error", "Quality suites"
|
||||
m.dialogText = sBad.Render("✗ ") + msg.err.Error() + "\n\n" + sDim.Render("any key")
|
||||
return m, nil
|
||||
}
|
||||
about := func(name string) string {
|
||||
for _, s := range msg.suites {
|
||||
if s.Name == name {
|
||||
return fmt.Sprintf("%s: %d cases, passed at %.2f, %s", s.Name, s.Cases, s.Threshold, s.From)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
path := "/api/v1/models/" + url.PathEscape(md.Name)
|
||||
if msg.gate {
|
||||
current, minScore := "none", ""
|
||||
if g := md.Eval; g != nil {
|
||||
current = g.Suite
|
||||
if g.MinScore > 0 {
|
||||
minScore = strconv.FormatFloat(g.MinScore, 'f', -1, 64)
|
||||
}
|
||||
if !slices.Contains(names, current) {
|
||||
names = append(names, current)
|
||||
}
|
||||
}
|
||||
m.openForm(&formDialog{title: "Quality gate of " + md.Name,
|
||||
note: "An update or a promotion asks the new weights this suite first, and stops below its score;\n" +
|
||||
"an update tries them as a canary given none of " + md.Name + "'s requests, and takes it out again.",
|
||||
fields: []formField{
|
||||
choiceF("suite", append([]string{"none"}, names...), current),
|
||||
textF("min score", minScore, "0 to 1; empty is the suite's own threshold"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if v[0] == "none" {
|
||||
return m.call("Quality gate of "+md.Name, http.MethodDelete, path+"/eval-gate", nil,
|
||||
func(json.RawMessage) string { return md.Name + "'s updates are no longer held to a suite." }), nil
|
||||
}
|
||||
score, err := minScoreOf(v[1])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.call("Quality gate of "+md.Name, http.MethodPut, path+"/eval-gate",
|
||||
adminapi.EvalGateRequest{Suite: v[0], MinScore: score}, func(json.RawMessage) string {
|
||||
return fmt.Sprintf("%s's new weights must pass %s before they replace the old.", md.Name, v[0])
|
||||
}), nil
|
||||
}})
|
||||
return m, nil
|
||||
}
|
||||
m.openForm(&formDialog{title: "Ask " + md.Name + " a quality suite",
|
||||
note: "Asked on the control plane, through the router as its own requests: past quotas and guardrails.\n" +
|
||||
about(names[0]),
|
||||
fields: []formField{
|
||||
choiceF("suite", names, names[0]),
|
||||
textF("at once", "4", "cases asked at the same time"),
|
||||
textF("min score", "", "0 to 1; empty is the suite's own threshold"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
n, err := strconv.Atoi(v[1])
|
||||
if err != nil || n < 1 {
|
||||
return nil, fmt.Errorf("at once is a whole number, 1 or more")
|
||||
}
|
||||
score, err := minScoreOf(v[2])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return m.startEval(md.Name, eval.Request{Suite: v[0], Concurrency: n, MinScore: score}), nil
|
||||
}})
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func minScoreOf(s string) (float64, error) {
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil || f < 0 || f > 1 {
|
||||
return 0, fmt.Errorf("the min score is a number from 0 to 1")
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// startEval starts a run and follows it.
|
||||
func (m *consoleModel) startEval(model string, req eval.Request) tea.Cmd {
|
||||
title := model + ": " + req.Suite
|
||||
m.busy, m.busyNote = title, "starting…"
|
||||
c := m.client
|
||||
return func() tea.Msg {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
var l eval.Live
|
||||
err := c.do(ctx, http.MethodPost, "/api/v1/models/"+url.PathEscape(model)+"/eval", req, &l)
|
||||
if err != nil {
|
||||
return actionDoneMsg{title: title, err: err}
|
||||
}
|
||||
return evalPollMsg{title: title, live: l}
|
||||
}
|
||||
}
|
||||
|
||||
func (m consoleModel) onEvalPoll(msg evalPollMsg) (tea.Model, tea.Cmd) {
|
||||
if m.evalWatch != "" && m.evalWatch != msg.live.ID {
|
||||
return m, nil
|
||||
}
|
||||
if m.evalWatch == "" && m.busy != msg.title {
|
||||
// Esc: no longer followed here; it carries on on the control plane.
|
||||
return m, nil
|
||||
}
|
||||
m.evalWatch = msg.live.ID
|
||||
m.busy = msg.title
|
||||
m.busyNote = fmt.Sprintf("%d of %d answered · Esc stops watching (it carries on)", msg.live.Done, msg.live.Cases)
|
||||
c, id, title := m.client, msg.live.ID, msg.title
|
||||
return m, func() tea.Msg {
|
||||
time.Sleep(time.Second)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
var l eval.Live
|
||||
if err := c.get(ctx, "/api/v1/eval/runs/"+url.PathEscape(id), &l); err != nil {
|
||||
return actionDoneMsg{title: title, err: err}
|
||||
}
|
||||
if l.State == eval.Running {
|
||||
return evalPollMsg{title: title, live: l}
|
||||
}
|
||||
var runs []eval.Live
|
||||
_ = c.get(ctx, "/api/v1/eval/runs?limit=6&model="+url.QueryEscape(l.Model), &runs)
|
||||
text := evalRunText(l)
|
||||
if len(runs) > 1 {
|
||||
text += "\n\nIts last runs:\n" + evalRunsText(runs)
|
||||
}
|
||||
if l.State == eval.Failed || !l.Passed {
|
||||
return actionDoneMsg{title: title, err: errors.New(text)}
|
||||
}
|
||||
return actionDoneMsg{title: title, text: text}
|
||||
}
|
||||
}
|
||||
|
||||
// evalRunText is a finished run: its score, then each case.
|
||||
func evalRunText(l eval.Live) string {
|
||||
if l.State == eval.Failed {
|
||||
return "The run could not be scored: " + l.Error
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(evalSummary(l) + "\n")
|
||||
if l.Error != "" {
|
||||
b.WriteString(l.Error + "\n")
|
||||
}
|
||||
b.WriteString("\n")
|
||||
shown := min(len(l.Results), 14)
|
||||
for _, r := range l.Results[:shown] {
|
||||
mark := sOK.Render("✓")
|
||||
if !r.Passed {
|
||||
mark = sBad.Render("✗")
|
||||
}
|
||||
why := r.Detail
|
||||
if r.Error != "" {
|
||||
why = r.Error
|
||||
}
|
||||
fmt.Fprintf(&b, "%s %-18s %4.2f %s\n", mark, truncate(r.ID, 18), r.Score, truncate(why, 60))
|
||||
}
|
||||
if len(l.Results) > shown {
|
||||
fmt.Fprintf(&b, " and %d more: llmgw ctl eval --run %s\n", len(l.Results)-shown, l.ID)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
|
||||
// evalRunsText is a model's runs, one a line.
|
||||
func evalRunsText(runs []eval.Live) string {
|
||||
if len(runs) == 0 {
|
||||
return "No runs kept yet: \"Ask it a quality suite\" on this menu makes one."
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, r := range runs {
|
||||
result := sOK.Render("passed")
|
||||
switch {
|
||||
case r.State == eval.Running:
|
||||
result = fmt.Sprintf("running, %d of %d", r.Done, r.Cases)
|
||||
case r.State == eval.Failed:
|
||||
result = sBad.Render("could not run")
|
||||
case !r.Passed:
|
||||
result = sBad.Render("failed")
|
||||
}
|
||||
who := r.Model
|
||||
if r.Target != "" {
|
||||
who = r.Target
|
||||
}
|
||||
fmt.Fprintf(&b, "%-9s %-14s %-16s %4.2f of %4.2f %s\n", ageShort(r.At), truncate(r.Suite, 14),
|
||||
truncate(who, 16), r.Score, r.Threshold, result)
|
||||
}
|
||||
return strings.TrimRight(b.String(), "\n")
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestAModelIsAskedASuiteFromItsMenuAndItsScoreShown(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
var asked eval.Request
|
||||
polls := 0
|
||||
done := eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen", Suite: "arithmetic", Score: 0.75,
|
||||
Threshold: 0.7, Passed: true, Cases: 2, Below: 1, Results: []store.EvalCase{
|
||||
{ID: "add", Kind: "number", Score: 1, Passed: true, Detail: "read 42"},
|
||||
{ID: "mul", Kind: "number", Score: 0, Detail: "read 43, wanted 42"}}}, State: eval.Done, Done: 2}
|
||||
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
switch {
|
||||
case r.URL.Path == "/api/v1/eval/suites":
|
||||
_ = json.NewEncoder(w).Encode([]eval.SuiteInfo{
|
||||
{Name: "arithmetic", From: "built in", Cases: 2, Threshold: 0.7},
|
||||
{Name: "broken", From: "/etc/llmgw/eval/broken.yaml", Error: "no cases"},
|
||||
{Name: "json-format", From: "built in", Cases: 7, Threshold: 0.7}})
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/api/v1/models/qwen/eval":
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(b, &asked)
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
|
||||
Suite: "arithmetic", Cases: 2}, State: eval.Running})
|
||||
case r.URL.Path == "/api/v1/eval/runs/eval-1":
|
||||
polls++
|
||||
if polls == 1 {
|
||||
_ = json.NewEncoder(w).Encode(eval.Live{EvalRun: store.EvalRun{ID: "eval-1", Model: "qwen",
|
||||
Suite: "arithmetic", Cases: 2}, State: eval.Running, Done: 1})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(done)
|
||||
case r.URL.Path == "/api/v1/eval/runs":
|
||||
older := done
|
||||
older.ID, older.Score, older.Passed = "eval-0", 0.5, false
|
||||
_ = json.NewEncoder(w).Encode([]eval.Live{done, older})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(admin.Close)
|
||||
m := newConsoleModel(admin.URL, "/nonexistent/server.yaml")
|
||||
m.loaded, m.menu, m.inPanel = true, cModels, true
|
||||
md := adminapi.ModelView{Name: "qwen"}
|
||||
|
||||
// run plays a command and what it leads to, the way the program would.
|
||||
run := func(tm tea.Model, cmd tea.Cmd) tea.Model {
|
||||
for cmd != nil {
|
||||
msg := cmd()
|
||||
tm, cmd = tm.Update(msg)
|
||||
if _, ok := msg.(actionDoneMsg); ok {
|
||||
break
|
||||
}
|
||||
}
|
||||
return tm
|
||||
}
|
||||
var tm tea.Model = m
|
||||
items := m.evalItems(md)
|
||||
if len(items) != 3 || !strings.Contains(items[0].label, "quality suite") {
|
||||
t.Fatalf("%+v", items)
|
||||
}
|
||||
tm = run(tm, items[0].act(&m))
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.fields[0].options == nil || len(f.fields[0].options) != 2 {
|
||||
t.Fatalf("no form offering the suites that can run: %+v", f)
|
||||
}
|
||||
tm = press(tm, "tab")
|
||||
f.fields[1].text.SetValue("2")
|
||||
tm = press(tm, "tab", "tab")
|
||||
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||||
if busy := tm.(consoleModel).busy; busy != "qwen: arithmetic" {
|
||||
t.Fatalf("busy %q", busy)
|
||||
}
|
||||
// Started, then followed: one poll still running, then its score.
|
||||
tm, next := tm.Update(cmd())
|
||||
if note := tm.(consoleModel).busyNote; !strings.Contains(note, "0 of 2 answered") {
|
||||
t.Errorf("progress %q", note)
|
||||
}
|
||||
tm, next = tm.Update(next())
|
||||
if note := tm.(consoleModel).busyNote; !strings.Contains(note, "1 of 2 answered") {
|
||||
t.Errorf("progress %q", note)
|
||||
}
|
||||
tm, _ = tm.Update(next())
|
||||
view := tm.View()
|
||||
mu.Lock()
|
||||
sent := asked
|
||||
mu.Unlock()
|
||||
if sent.Suite != "arithmetic" || sent.Concurrency != 2 {
|
||||
t.Errorf("asked %+v", sent)
|
||||
}
|
||||
for _, want := range []string{"qwen scored 0.75 on arithmetic", "1 of 2 cases passed", "read 43, wanted 42", "Its last runs"} {
|
||||
if !strings.Contains(view, want) {
|
||||
t.Errorf("the result does not say %q:\n%s", want, view)
|
||||
}
|
||||
}
|
||||
if c := tm.(consoleModel); c.busy != "" || c.evalWatch != "" {
|
||||
t.Errorf("still busy: %q %q", c.busy, c.evalWatch)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user