diff --git a/CHANGELOG.md b/CHANGELOG.md index 1927185c..d52d8b63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,18 @@ more than 26 hours old or there has never been one. docs/install.md now says how to restore, including why running engines survive it: the engine-key secret comes back with the store. +### A model from the registry in one command + +`llmgw ctl model serve OWNER/REPO` takes a model from the registry to a replica +that answers. It picks the GGUF file (`--quant`, Q4_K_M by default), has the +mirror fetch it and checks what arrived against the registry's published +sha256, checks that the nodes' llama.cpp can load it, sizes its memory from its +header, adds it to the catalog, waits for a replica — stopping early with the +reason if it fits nowhere or keeps failing — and asks it to say hello. Each step +is shown as it happens. It runs in the control plane as a job, so a download of +hours does not depend on a terminal staying open. `--on-demand` adds it without +starting a replica. + ### Upgrading from one command `sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that diff --git a/internal/cli/model.go b/internal/cli/model.go index bcf4c61d..82bcf968 100644 --- a/internal/cli/model.go +++ b/internal/cli/model.go @@ -28,7 +28,8 @@ func newModelCmd(client func() *ctlClient) *cobra.Command { Use: "model", Short: "Add or remove models while the cluster is running", } - c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client)) + c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client), + newModelServeCmd(client)) return c } diff --git a/internal/cli/model_serve.go b/internal/cli/model_serve.go new file mode 100644 index 00000000..f5a452cc --- /dev/null +++ b/internal/cli/model_serve.go @@ -0,0 +1,80 @@ +package cli + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" +) + +func newModelServeCmd(client func() *ctlClient) *cobra.Command { + var ( + req modelserve.Request + onDemand bool + ) + c := &cobra.Command{ + Use: "serve OWNER/REPO", + Short: "Take a model from the registry to a replica that answers, in one command", + Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" + + "fetch it and checks what arrived against the registry's checksum, checks\n" + + "that the nodes' llama.cpp can load it and sizes its memory from its own\n" + + "header, adds it to the catalog, waits for a replica, and asks it to say\n" + + "hello. Each step is shown as it happens; the first that fails stops it.\n\n" + + " llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" + + " llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" + + " llmgw ctl model serve owner/big-GGUF --on-demand", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + req.Repo = args[0] + if onDemand { + req.Replicas = 0 + } + body, _ := json.Marshal(req) + var job modelserve.Job + if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "job %s\n", job.ID) + shown := map[string]string{} + for { + for _, s := range job.Steps { + line := s.State + " " + s.Detail + if shown[s.Name] == line { + continue + } + shown[s.Name] = line + mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok", + modelserve.Failed: "!!"}[s.State] + fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail) + } + switch job.State { + case modelserve.Done: + fmt.Fprintf(out, "\n%s is served.\n", job.Model) + return nil + case modelserve.Failed: + return errors.New(job.Error) + } + time.Sleep(2 * time.Second) + if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil { + return err + } + } + }, + } + f := c.Flags() + f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)") + f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant") + f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)") + f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable") + f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models") + f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run") + f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for") + f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots") + f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use") + return c +} diff --git a/internal/cli/server.go b/internal/cli/server.go index f2501a80..91bbd0b6 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "errors" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" "gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck" "github.com/prometheus/client_golang/prometheus" @@ -171,8 +172,16 @@ func newServerCmd() *cobra.Command { if err != nil { return err } + // Kept past this block: serving a model from the registry needs + // both the admin API and the mirror, and the mirror is set up + // below. + var ( + adminAPI *adminapi.API + catalogSvc *modelsvc.Service + ) if admin != nil { api := adminapi.New(admin) + adminAPI = api api.SetReleases(cfg.Releases) // Catalog changes need somewhere to persist, so this is wired // only when there is a store. Without one, the endpoints say so @@ -190,6 +199,7 @@ func newServerCmd() *cobra.Command { admin.ReplaceCatalog(ctx, built) } api.SetModels(svc) + catalogSvc = svc // And the mirror page, so a model that has just been // fetched can be served without writing YAML by hand. rt.SetCatalogAdmin(svc) @@ -259,7 +269,16 @@ func newServerCmd() *cobra.Command { // Browsing the registry is what turns "paste a resolve URL" // into "find the model". It needs a way out to the internet // from this machine, which the mirror's own fetching does not. - rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)) + registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout) + rt.SetRegistry(registry) + // One command from a registry repository to a replica that + // answers: `llmgw ctl model serve`. + if adminAPI != nil && catalogSvc != nil { + adminAPI.SetServe(modelserve.New(modelserve.Deps{ + Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin, + LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log, + })) + } log.Info("model mirror configured", "api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled()) } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index f5de4217..8aecaa38 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -10,6 +10,7 @@ import ( "context" "encoding/json" "fmt" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/release" "io" "net/http" @@ -33,6 +34,8 @@ import ( type API struct { // releases is where release binaries are fetched from, for upgrades. releases string + // serve runs registry-to-replica jobs; nil without a mirror and a store. + serve *modelserve.Server models Models capacity Capacity @@ -83,6 +86,8 @@ func (a *API) Register(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest) mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs) mux.HandleFunc("POST /api/v1/restart", a.restart) + mux.HandleFunc("POST /api/v1/models/serve", a.serveModel) + mux.HandleFunc("GET /api/v1/jobs/{id}", a.job) mux.HandleFunc("GET /api/v1/requests/{id}", a.request) mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel) } @@ -1204,3 +1209,39 @@ func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) { } writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version}) } + +// SetServe supplies what serves a model from the registry. +func (a *API) SetServe(s *modelserve.Server) { a.serve = s } + +func (a *API) serveModel(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotImplemented, + "serving from the registry needs a mirror and a store, and this control plane lacks one") + return + } + var req modelserve.Request + if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error()) + return + } + req.By = r.Header.Get("X-Llmgw-Actor") + job, err := a.serve.Start(req) + if err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + writeJSON(w, http.StatusAccepted, job) +} + +func (a *API) job(w http.ResponseWriter, r *http.Request) { + if a.serve == nil { + writeError(w, http.StatusNotFound, "no such job") + return + } + job, ok := a.serve.Get(r.PathValue("id")) + if !ok { + writeError(w, http.StatusNotFound, "no such job; jobs are kept only until the control plane restarts") + return + } + writeJSON(w, http.StatusOK, job) +} diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go new file mode 100644 index 00000000..68429e98 --- /dev/null +++ b/internal/control/modelserve/modelserve.go @@ -0,0 +1,552 @@ +// Package modelserve takes a model from a registry repository to a replica +// answering requests, as one job. +// +// Adding a model was four commands and a hand-written YAML file: plan it, +// fetch it into the mirror, write its catalog entry with a memory figure +// somebody guessed, scale it — and then find out whether it loads. This runs +// those steps in the control plane, where the mirror, the registry, the +// catalog and the fleet already are, and reports each one as it goes, so +// whatever asks — ctl, a page — only has to watch. +package modelserve + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/gguf" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +// Registry reads a repository. +type Registry interface { + Repo(ctx context.Context, id string) (hf.Repo, error) +} + +// Mirror fetches files and reports on them. +type Mirror interface { + Fetch(ctx context.Context, req mirror.FetchRequest) (mirror.FetchResponse, error) + List(ctx context.Context) (mirror.ModelsResponse, error) +} + +// Catalog adds a model. +type Catalog interface { + Add(ctx context.Context, m catalog.Model, by string) error +} + +// Fleet is what the job watches once the model is in the catalog. +type Fleet interface { + Replicas() []reconciler.Replica + ModelHealth() []reconciler.ModelHealth + EngineKey(replicaID string) string +} + +// Deps are what a job needs. +type Deps struct { + Registry Registry + Mirror Mirror + Catalog Catalog + Fleet Fleet + // LlamaVersion is the llama.cpp build the nodes install, which decides + // which architectures load. + LlamaVersion string + HTTP *http.Client + Log *slog.Logger + // Poll is how often the job looks at the mirror and the fleet. + Poll time.Duration + // ReadyTimeout bounds the wait for a replica: the node fetching the + // weights from the mirror and the engine loading them. + ReadyTimeout time.Duration +} + +// Request is what to serve. +type Request struct { + // Repo is the registry repository, owner/name. + Repo string `json:"repo"` + // Quant picks the file by its quantisation, Q4_K_M unless given. + Quant string `json:"quant,omitempty"` + // File picks it by name instead. + File string `json:"file,omitempty"` + // Name is the catalog name; the repository suggests one otherwise. + Name string `json:"name,omitempty"` + Aliases []string `json:"aliases,omitempty"` + Description string `json:"description,omitempty"` + // Replicas to run; 0 makes it on demand, loaded when first asked for. + Replicas int `json:"replicas"` + // CtxPool is llama.cpp's --ctx-size; ContextLength what one request may + // use. They default to 16384 and 4096, what four parallel slots divide. + CtxPool int `json:"ctx_pool,omitempty"` + ContextLength int `json:"context_length,omitempty"` + // By is who asked, for the catalog's record. + By string `json:"-"` +} + +// The states a job and its steps can be in. +const ( + Running = "running" + Done = "done" + Failed = "failed" + Skipped = "skipped" +) + +// Step is one stage of a job. +type Step struct { + Name string `json:"name"` + State string `json:"state"` + Detail string `json:"detail,omitempty"` + At time.Time `json:"at"` +} + +// Job is one model being served, and where it stands. +type Job struct { + ID string `json:"id"` + Request Request `json:"request"` + State string `json:"state"` + Error string `json:"error,omitempty"` + Model string `json:"model,omitempty"` + Steps []Step `json:"steps"` + Started time.Time `json:"started"` + // Reply is what the model said to the smoke test, and TokensPerSecond + // how fast it said it. + Reply string `json:"reply,omitempty"` + TokensPerSecond float64 `json:"tokens_per_second,omitempty"` +} + +// Server runs jobs and remembers them. +type Server struct { + deps Deps + mu sync.Mutex + jobs map[string]*Job + seq int +} + +// New builds a server. +func New(d Deps) *Server { + if d.Poll <= 0 { + d.Poll = 2 * time.Second + } + if d.ReadyTimeout <= 0 { + d.ReadyTimeout = 30 * time.Minute + } + if d.HTTP == nil { + d.HTTP = &http.Client{Timeout: 5 * time.Minute} + } + if d.Log == nil { + d.Log = slog.Default() + } + return &Server{deps: d, jobs: map[string]*Job{}} +} + +var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) + +// Start begins a job and returns at once; the job runs on its own. +func (s *Server) Start(req Request) (Job, error) { + req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/") + if !repoPattern.MatchString(req.Repo) { + return Job{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo) + } + if req.Replicas < 0 { + return Job{}, errors.New("replicas cannot be negative") + } + if req.Quant == "" && req.File == "" { + req.Quant = "Q4_K_M" + } + if req.CtxPool <= 0 { + req.CtxPool = 16384 + } + if req.ContextLength <= 0 { + req.ContextLength = 4096 + } + s.mu.Lock() + s.seq++ + j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req, + State: Running, Started: time.Now()} + s.jobs[j.ID] = j + s.mu.Unlock() + go s.run(j) + return s.snapshot(j), nil +} + +// Get returns a job as it stands. +func (s *Server) Get(id string) (Job, bool) { + s.mu.Lock() + j, ok := s.jobs[id] + s.mu.Unlock() + if !ok { + return Job{}, false + } + return s.snapshot(j), true +} + +func (s *Server) snapshot(j *Job) Job { + s.mu.Lock() + defer s.mu.Unlock() + out := *j + out.Steps = slices.Clone(j.Steps) + return out +} + +// step starts a new step, or updates the detail of the current one. +func (s *Server) step(j *Job, name, state, detail string) { + s.mu.Lock() + defer s.mu.Unlock() + if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name { + j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now() + return + } + j.Steps = append(j.Steps, Step{Name: name, State: state, Detail: detail, At: time.Now()}) +} + +func (s *Server) fail(j *Job, name string, err error) { + s.step(j, name, Failed, err.Error()) + s.mu.Lock() + j.State, j.Error = Failed, err.Error() + s.mu.Unlock() + s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err) +} + +func (s *Server) run(j *Job) { + ctx := context.Background() + req := j.Request + + // 1. The registry: which files. + s.step(j, "registry", Running, "reading "+req.Repo) + repo, err := s.deps.Registry.Repo(ctx, req.Repo) + if err != nil { + s.fail(j, "registry", err) + return + } + if repo.Gated.Is() { + s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo)) + return + } + set, err := pick(repo, req) + if err != nil { + s.fail(j, "registry", err) + return + } + name := req.Name + if name == "" { + name = hf.ModelName(repo.ID) + } + s.mu.Lock() + j.Model = name + s.mu.Unlock() + s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s", + set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision())) + + // 2. The mirror: fetch, and check what arrived against what was published. + source, err := s.mirror(ctx, j, repo, set, name) + if err != nil { + s.fail(j, "mirror", err) + return + } + + // 3. Whether the nodes' llama.cpp loads it, and how much memory it needs. + s.step(j, "check", Running, "reading its header from the mirror") + var header gguf.Header + if !strings.HasSuffix(source, "/") { + v := gguf.Check(ctx, source, s.deps.LlamaVersion) + if !v.Servable { + s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason)) + return + } + header = v.Header + } + memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30) + basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine", + gb(set.Bytes), gb(int64(req.CtxPool)*header.KVBytesPerToken()), req.CtxPool) + if memGB == 0 { + // A set of shards, or a header that did not say: weights plus half + // again plus two, the rough figure from before shapes were read. + memGB = int(gb(set.Bytes)*1.5) + 2 + basis = "weights plus half again plus two; the header gave no shape" + } + s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis)) + + // 4. The catalog. + m := catalog.Model{ + Name: name, Aliases: req.Aliases, Description: req.Description, Source: source, + RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024, + Profiles: []catalog.Profile{{ + Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100, + Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, + }}, + Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)}, + } + s.step(j, "catalog", Running, "adding "+name) + if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil { + s.fail(j, "catalog", err) + return + } + if req.Replicas == 0 { + s.step(j, "catalog", Done, name+" added on demand: it loads when first asked for") + s.finish(j) + return + } + s.step(j, "catalog", Done, fmt.Sprintf("%s added, %d replica(s)", name, req.Replicas)) + + // 5. A replica, ready. + rep, err := s.waitReady(ctx, j, name) + if err != nil { + s.fail(j, "placement", err) + return + } + + // 6. Does it answer. + s.step(j, "smoke test", Running, "asking "+rep.ID+" to say hello") + reply, tps, err := s.smoke(ctx, rep, name) + if err != nil { + s.fail(j, "smoke test", err) + return + } + s.mu.Lock() + j.Reply, j.TokensPerSecond = reply, tps + s.mu.Unlock() + s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply)) + s.finish(j) +} + +func (s *Server) finish(j *Job) { + s.mu.Lock() + j.State = Done + s.mu.Unlock() + s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model) +} + +// pick chooses the file set to serve. +func pick(repo hf.Repo, req Request) (hf.FileSet, error) { + var weights []hf.File + for _, f := range repo.Files { + lower := strings.ToLower(f.Path) + if !strings.HasSuffix(lower, ".gguf") || strings.Contains(lower, "mmproj") || + strings.Contains(lower, "imatrix") { + continue + } + weights = append(weights, f) + } + sets := hf.Group(weights) + if len(sets) == 0 { + return hf.FileSet{}, fmt.Errorf("%s has no GGUF files; the nodes run llama.cpp, which needs one", repo.ID) + } + var names []string + for _, st := range sets { + names = append(names, st.Name) + if req.File != "" && (st.Name == req.File || slices.ContainsFunc(st.Files, func(f hf.File) bool { return f.Path == req.File })) { + return st, nil + } + } + if req.File == "" { + want := strings.ToLower(req.Quant) + var match []hf.FileSet + for _, st := range sets { + if strings.Contains(strings.ToLower(st.Name), want) { + match = append(match, st) + } + } + if len(match) == 1 { + return match[0], nil + } + if len(match) > 1 { + // Both a split and a joined copy of one quantisation: the joined + // one, which is one download to verify rather than several. + slices.SortFunc(match, func(a, b hf.FileSet) int { return a.Shards - b.Shards }) + return match[0], nil + } + } + return hf.FileSet{}, fmt.Errorf("%s has no file matching %q; it has: %s", + repo.ID, or(req.File, req.Quant), strings.Join(names, ", ")) +} + +// mirror fetches the set into the mirror and returns the source a catalog +// entry names: the file, or the directory for a set of shards. +func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name string) (string, error) { + version := repo.ShortRevision() + s.step(j, "mirror", Running, "asking the mirror for "+set.Name) + var base string + for _, f := range set.Files { + file := mirror.LastPathElement(f.Path) + resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{ + Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file, + }) + // Already there is not a failure: the same commit fetched before. + if err != nil && !strings.Contains(err.Error(), "already") { + return "", err + } + base = resp.URL + } + deadline := time.Now().Add(6 * time.Hour) + for { + got, err := s.deps.Mirror.List(ctx) + if err != nil { + return "", err + } + var done, total int64 + present, failed := 0, "" + sums := map[string]string{} + for _, st := range got.Models { + if st.Model != name || st.Version != version { + continue + } + done += st.BytesDone + total += max(st.BytesTotal, st.BytesDone) + switch st.State { + case mirror.Present: + present++ + sums[st.File] = st.SHA256 + case mirror.Failed: + failed = st.Err + } + } + if failed != "" { + return "", fmt.Errorf("the mirror could not fetch it: %s", failed) + } + if present >= len(set.Files) { + // What arrived against what the registry published: a mirror + // copies one file to every node, and is the one place a corrupted + // download is caught once instead of loaded everywhere. + for _, f := range set.Files { + file := mirror.LastPathElement(f.Path) + if f.SHA256 != "" && sums[file] != "" && !strings.EqualFold(f.SHA256, sums[file]) { + return "", fmt.Errorf("%s arrived with sha256 %s, and the registry published %s", + file, sums[file][:12], f.SHA256[:12]) + } + } + source := base + if len(set.Files) > 1 { + source = strings.TrimSuffix(source, mirror.LastPathElement(source)) + } + s.step(j, "mirror", Done, fmt.Sprintf("%.1f GB on the mirror, checksums match the registry", gb(total))) + return source, nil + } + if total > 0 { + s.step(j, "mirror", Running, fmt.Sprintf("fetching: %.1f of %.1f GB (%d%%)", + gb(done), gb(total), done*100/total)) + } + if time.Now().After(deadline) { + return "", errors.New("the mirror had not finished after six hours") + } + time.Sleep(s.deps.Poll) + } +} + +// waitReady waits for a replica of the model to serve, reporting what it is +// doing and giving up early when it cannot be placed or keeps failing. +func (s *Server) waitReady(ctx context.Context, j *Job, name string) (reconciler.Replica, error) { + s.step(j, "placement", Running, "waiting for the control plane to place it") + started := time.Now() + deadline := started.Add(s.deps.ReadyTimeout) + for { + for _, h := range s.deps.Fleet.ModelHealth() { + if h.Model != name { + continue + } + if h.Parked { + return reconciler.Replica{}, fmt.Errorf("it failed to start %d times and is parked: %s", h.Failures, h.Reason) + } + // A minute's grace from when this step began, not from the job: + // the download before it can take hours, and the first pass may + // not have run yet. + if h.Unplaceable && time.Since(started) > time.Minute { + return reconciler.Replica{}, fmt.Errorf("no node can host it:\n%s", h.Placement) + } + } + for _, r := range s.deps.Fleet.Replicas() { + if r.Model != name { + continue + } + if r.State == reconciler.Ready { + s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID, + strings.Join(r.NodeIDs(), ","), r.Endpoint)) + return r, nil + } + s.step(j, "placement", Running, fmt.Sprintf("%s on %s is %s", r.ID, + strings.Join(r.NodeIDs(), ","), r.State)) + } + if time.Now().After(deadline) { + return reconciler.Replica{}, fmt.Errorf("no replica was ready after %s", s.deps.ReadyTimeout) + } + select { + case <-ctx.Done(): + return reconciler.Replica{}, ctx.Err() + case <-time.After(s.deps.Poll): + } + } +} + +// smoke sends one short request straight to the replica, with its engine key. +func (s *Server) smoke(ctx context.Context, r reconciler.Replica, name string) (string, float64, error) { + body, _ := json.Marshal(map[string]any{ + "model": name, + "max_tokens": 32, + "messages": []map[string]string{{"role": "user", "content": "Say hello in one short sentence."}}, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + strings.TrimRight(r.Endpoint, "/")+"/v1/chat/completions", bytes.NewReader(body)) + if err != nil { + return "", 0, err + } + req.Header.Set("Content-Type", "application/json") + if k := s.deps.Fleet.EngineKey(r.ID); k != "" { + req.Header.Set("Authorization", "Bearer "+k) + } + start := time.Now() + resp, err := s.deps.HTTP.Do(req) + if err != nil { + return "", 0, err + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return "", 0, fmt.Errorf("the replica answered %s: %s", resp.Status, strings.TrimSpace(string(b))) + } + var out struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage struct { + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(b, &out); err != nil || len(out.Choices) == 0 { + return "", 0, fmt.Errorf("the replica's answer was not a completion: %s", strings.TrimSpace(string(b))) + } + secs := time.Since(start).Seconds() + tps := 0.0 + if secs > 0 { + tps = float64(out.Usage.CompletionTokens) / secs + } + return strings.TrimSpace(out.Choices[0].Message.Content), tps, nil +} + +func gb(b int64) float64 { return float64(b) / (1 << 30) } + +func or(a, b string) string { + if a != "" { + return a + } + return b +} + +func orUnknown(arch string) string { + if arch == "" { + return "architecture not read" + } + return arch +} diff --git a/internal/control/modelserve/modelserve_test.go b/internal/control/modelserve/modelserve_test.go new file mode 100644 index 00000000..7c4fdfe3 --- /dev/null +++ b/internal/control/modelserve/modelserve_test.go @@ -0,0 +1,225 @@ +package modelserve + +import ( + "bytes" + "context" + "encoding/binary" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" +) + +// ggufHeader is enough of a GGUF file for its architecture and shape. +func ggufHeader() []byte { + var b bytes.Buffer + w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) } + str := func(s string) { w(uint64(len(s))); b.WriteString(s) } + u32 := func(k string, v uint32) { str(k); w(uint32(4)); w(v) } + b.WriteString("GGUF") + w(uint32(3)) + w(uint64(0)) + w(uint64(5)) + str("general.architecture") + w(uint32(8)) + str("qwen2") + u32("qwen2.block_count", 28) + u32("qwen2.attention.head_count", 12) + u32("qwen2.attention.head_count_kv", 2) + u32("qwen2.embedding_length", 1536) + return b.Bytes() +} + +type fakeRegistry struct{ repo hf.Repo } + +func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil } + +// fakeMirror serves the header at every file URL and reports each fetched +// file present, with the checksum it was told to. +type fakeMirror struct { + mu sync.Mutex + srv *httptest.Server + got []mirror.FetchRequest + sums map[string]string +} + +func (m *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + m.got = append(m.got, r) + return mirror.FetchResponse{URL: m.srv.URL + "/" + r.Model + "/" + r.Version + "/" + r.File}, nil +} + +func (m *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out mirror.ModelsResponse + for _, r := range m.got { + out.Models = append(out.Models, mirror.Status{Model: r.Model, Version: r.Version, File: r.File, + State: mirror.Present, BytesDone: 1 << 30, BytesTotal: 1 << 30, SHA256: m.sums[r.File]}) + } + return out, nil +} + +type fakeCatalog struct { + mu sync.Mutex + added []catalog.Model +} + +func (c *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error { + c.mu.Lock() + defer c.mu.Unlock() + c.added = append(c.added, m) + return nil +} + +// fakeFleet has a replica ready at engine once the catalog holds the model. +type fakeFleet struct { + cat *fakeCatalog + engine string + health []reconciler.ModelHealth +} + +func (f *fakeFleet) Replicas() []reconciler.Replica { + f.cat.mu.Lock() + defer f.cat.mu.Unlock() + var out []reconciler.Replica + for _, m := range f.cat.added { + out = append(out, reconciler.Replica{ID: m.Name + "-r1", Model: m.Name, + State: reconciler.Ready, Endpoint: f.engine, + Shards: []reconciler.Shard{{NodeID: "node01"}}}) + } + return out +} +func (f *fakeFleet) ModelHealth() []reconciler.ModelHealth { return f.health } +func (f *fakeFleet) EngineKey(string) string { return "k" } + +func setup(t *testing.T, files []hf.File, sums map[string]string) (*Server, *fakeMirror, *fakeCatalog, *fakeFleet) { + t.Helper() + header := ggufHeader() + files0 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(header) + })) + t.Cleanup(files0.Close) + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer k" { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hello!"}}],"usage":{"completion_tokens":3}}`)) + })) + t.Cleanup(engine.Close) + + m := &fakeMirror{srv: files0, sums: sums} + c := &fakeCatalog{} + f := &fakeFleet{cat: c, engine: engine.URL} + s := New(Deps{ + Registry: fakeRegistry{repo: hf.Repo{ID: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abcdef123456", Files: files}}, + Mirror: m, Catalog: c, Fleet: f, Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second, + }) + return s, m, c, f +} + +func wait(t *testing.T, s *Server, id string) Job { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if j, _ := s.Get(id); j.State != Running { + return j + } + time.Sleep(10 * time.Millisecond) + } + t.Fatal("the job never finished") + return Job{} +} + +func TestAModelGoesFromTheRegistryToAnAnsweringReplica(t *testing.T) { + files := []hf.File{ + {Path: "Qwen2.5-1.5B-Instruct-Q8_0.gguf", Bytes: 2 << 30, SHA256: "aa"}, + {Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}, + {Path: "mmproj-Q4_K_M.gguf", Bytes: 1 << 20}, + } + s, m, c, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"}) + job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1}) + if err != nil { + t.Fatal(err) + } + j := wait(t, s, job.ID) + if j.State != Done { + t.Fatalf("job %s: %s\n%+v", j.State, j.Error, j.Steps) + } + if len(m.got) != 1 || m.got[0].File != "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" || m.got[0].Version != "abcdef12" { + t.Errorf("fetched %+v; expected the Q4_K_M file at its revision", m.got) + } + mod := c.added[0] + if mod.Name != "qwen2.5-1.5b-instruct" || mod.Profiles[0].MinMemoryGB == 0 || mod.Scaling.Min != 1 { + t.Errorf("catalog entry %+v", mod) + } + if j.Reply != "Hello!" { + t.Errorf("the smoke test's answer was %q", j.Reply) + } + var names []string + for _, st := range j.Steps { + names = append(names, st.Name) + } + if strings.Join(names, ",") != "registry,mirror,check,catalog,placement,smoke test" { + t.Errorf("steps %v", names) + } +} + +func TestAQuantTheRepositoryDoesNotHaveIsNamedWithWhatItHas(t *testing.T) { + s, _, c, _ := setup(t, []hf.File{{Path: "m-Q8_0.gguf", Bytes: 1}}, nil) + job, _ := s.Start(Request{Repo: "a/b", Replicas: 1}) + j := wait(t, s, job.ID) + if j.State != Failed || !strings.Contains(j.Error, "m-Q8_0.gguf") { + t.Errorf("%s: %s", j.State, j.Error) + } + if len(c.added) != 0 { + t.Error("something was added to the catalog") + } +} + +func TestADownloadThatDoesNotMatchTheRegistryIsNotServed(t *testing.T) { + files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}} + s, _, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"}) + job, _ := s.Start(Request{Repo: "a/b", Replicas: 1}) + j := wait(t, s, job.ID) + if j.State != Failed || !strings.Contains(j.Error, "registry published") { + t.Errorf("%s: %s", j.State, j.Error) + } + if len(c.added) != 0 { + t.Error("a corrupted download reached the catalog") + } +} + +func TestAParkedModelStopsTheWait(t *testing.T) { + s, _, _, f := setup(t, []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30}}, nil) + f.health = []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Parked: true, Failures: 5, Reason: "unknown model architecture"}} + // A fleet where it never becomes ready, only parked. + s.deps.Fleet = parkedFleet{f} + job, _ := s.Start(Request{Repo: "a/b", Replicas: 1}) + j := wait(t, s, job.ID) + if j.State != Failed || !strings.Contains(j.Error, "parked") { + t.Errorf("%s: %s", j.State, j.Error) + } +} + +type parkedFleet struct{ *fakeFleet } + +func (parkedFleet) Replicas() []reconciler.Replica { return nil } + +func TestARepositoryThatIsNotOneIsRefused(t *testing.T) { + s := New(Deps{}) + for _, r := range []string{"", "noslash", "../etc/passwd", "a/b/c"} { + if _, err := s.Start(Request{Repo: r}); err == nil { + t.Errorf("%q accepted", r) + } + } +} diff --git a/internal/hf/hf.go b/internal/hf/hf.go index c890d2df..65a2bb48 100644 --- a/internal/hf/hf.go +++ b/internal/hf/hf.go @@ -558,3 +558,16 @@ func Group(files []File) []FileSet { } return out } + +// ModelName is the catalog name a repository suggests: its last element, +// lowercased, with the -GGUF marker dropped. The owner is not part of what +// anybody calls the model, and "-GGUF" describes the file format. +func ModelName(repoID string) string { + name := repoID + if i := strings.LastIndex(name, "/"); i >= 0 { + name = name[i+1:] + } + name = strings.TrimSuffix(name, "-GGUF") + name = strings.TrimSuffix(name, "-gguf") + return strings.ToLower(name) +} diff --git a/internal/web/admin_mirror.go b/internal/web/admin_mirror.go index da50e169..b37bcdf7 100644 --- a/internal/web/admin_mirror.go +++ b/internal/web/admin_mirror.go @@ -621,12 +621,7 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string // is not part of what anybody calls the model, and "-GGUF" describes the file // format rather than the model, which is already clear from the file being // fetched. -func defaultModelName(repoID string) string { - name := mirror.LastPathElement(repoID) - name = strings.TrimSuffix(name, "-GGUF") - name = strings.TrimSuffix(name, "-gguf") - return strings.ToLower(name) -} +func defaultModelName(repoID string) string { return hf.ModelName(repoID) } // verdictFor judges a registry file set by the repository's architecture. A // vision projector is its own architecture whatever the repository says.