Serve a model from the registry with one command
Release / build (push) Successful in 1m8s
Release / notes (push) Successful in 3s
CI / test (push) Successful in 3m50s

Adding a model took four steps and a YAML file — plan it, fetch it into the
mirror, write its catalog entry with a memory figure somebody estimated,
scale it — and whether it would load was found out at the end.

`llmgw ctl model serve OWNER/REPO` does it as one job in the control plane,
where the registry, the mirror, the catalog and the fleet already are:

- the registry: the GGUF file set matching --quant (Q4_K_M by default) or
  --file, leaving out vision projectors and importance matrices, and naming
  what the repository does have when nothing matches; gated repositories are
  refused;
- the mirror: fetched at the repository's revision, with progress, and each
  file's sha256 compared with what the registry published — the mirror is
  the one place a corrupted download is caught once rather than loaded on
  every node;
- the check: whether the nodes' llama.cpp loads the architecture, and the
  memory it needs from its own header;
- the catalog, then a wait for a replica that stops early, with the reason,
  when the model is parked or fits on no node;
- a smoke test: one short request to the replica, with its engine key, and
  the answer and tokens per second.

ctl starts the job and prints each step as it changes; a download of hours
does not depend on the terminal. --on-demand adds the model without starting
a replica. Jobs are kept in memory until the control plane restarts.

The catalog name a repository suggests moved into the registry package, so
the mirror page and this agree on it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-09-30 03:05:13 +02:00
1 parent 116e75f8cb
commit bd12b0909a
9 files changed
+946 -8

No files matched your search

+12
View File
@@ -77,6 +77,18 @@ more than 26 hours old or there has never been one. docs/install.md now says how
to restore, including why running engines survive it: the engine-key secret
comes back with the store.
### A model from the registry in one command
`llmgw ctl model serve OWNER/REPO` takes a model from the registry to a replica
that answers. It picks the GGUF file (`--quant`, Q4_K_M by default), has the
mirror fetch it and checks what arrived against the registry's published
sha256, checks that the nodes' llama.cpp can load it, sizes its memory from its
header, adds it to the catalog, waits for a replica — stopping early with the
reason if it fits nowhere or keeps failing — and asks it to say hello. Each step
is shown as it happens. It runs in the control plane as a job, so a download of
hours does not depend on a terminal staying open. `--on-demand` adds it without
starting a replica.
### Upgrading from one command
`sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that
+2 -1
View File
@@ -28,7 +28,8 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
Use: "model",
Short: "Add or remove models while the cluster is running",
}
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
newModelServeCmd(client))
return c
}
+80
View File
@@ -0,0 +1,80 @@
package cli
import (
"encoding/json"
"errors"
"fmt"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
)
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
var (
req modelserve.Request
onDemand bool
)
c := &cobra.Command{
Use: "serve OWNER/REPO",
Short: "Take a model from the registry to a replica that answers, in one command",
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
"fetch it and checks what arrived against the registry's checksum, checks\n" +
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
" llmgw ctl model serve owner/big-GGUF --on-demand",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
req.Repo = args[0]
if onDemand {
req.Replicas = 0
}
body, _ := json.Marshal(req)
var job modelserve.Job
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
return err
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "job %s\n", job.ID)
shown := map[string]string{}
for {
for _, s := range job.Steps {
line := s.State + " " + s.Detail
if shown[s.Name] == line {
continue
}
shown[s.Name] = line
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
modelserve.Failed: "!!"}[s.State]
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
}
switch job.State {
case modelserve.Done:
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
return nil
case modelserve.Failed:
return errors.New(job.Error)
}
time.Sleep(2 * time.Second)
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
return err
}
}
},
}
f := c.Flags()
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
return c
}
+20 -1
View File
@@ -5,6 +5,7 @@ import (
"crypto/tls"
"errors"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
"github.com/prometheus/client_golang/prometheus"
@@ -171,8 +172,16 @@ func newServerCmd() *cobra.Command {
if err != nil {
return err
}
// Kept past this block: serving a model from the registry needs
// both the admin API and the mirror, and the mirror is set up
// below.
var (
adminAPI *adminapi.API
catalogSvc *modelsvc.Service
)
if admin != nil {
api := adminapi.New(admin)
adminAPI = api
api.SetReleases(cfg.Releases)
// Catalog changes need somewhere to persist, so this is wired
// only when there is a store. Without one, the endpoints say so
@@ -190,6 +199,7 @@ func newServerCmd() *cobra.Command {
admin.ReplaceCatalog(ctx, built)
}
api.SetModels(svc)
catalogSvc = svc
// And the mirror page, so a model that has just been
// fetched can be served without writing YAML by hand.
rt.SetCatalogAdmin(svc)
@@ -259,7 +269,16 @@ func newServerCmd() *cobra.Command {
// Browsing the registry is what turns "paste a resolve URL"
// into "find the model". It needs a way out to the internet
// from this machine, which the mirror's own fetching does not.
rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout))
registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)
rt.SetRegistry(registry)
// One command from a registry repository to a replica that
// answers: `llmgw ctl model serve`.
if adminAPI != nil && catalogSvc != nil {
adminAPI.SetServe(modelserve.New(modelserve.Deps{
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
}))
}
log.Info("model mirror configured",
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
}
+41
View File
@@ -10,6 +10,7 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"io"
"net/http"
@@ -33,6 +34,8 @@ import (
type API struct {
// releases is where release binaries are fetched from, for upgrades.
releases string
// serve runs registry-to-replica jobs; nil without a mirror and a store.
serve *modelserve.Server
models Models
capacity Capacity
@@ -83,6 +86,8 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
mux.HandleFunc("POST /api/v1/restart", a.restart)
mux.HandleFunc("POST /api/v1/models/serve", a.serveModel)
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
}
@@ -1204,3 +1209,39 @@ func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
}
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
}
// SetServe supplies what serves a model from the registry.
func (a *API) SetServe(s *modelserve.Server) { a.serve = s }
func (a *API) serveModel(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotImplemented,
"serving from the registry needs a mirror and a store, and this control plane lacks one")
return
}
var req modelserve.Request
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
req.By = r.Header.Get("X-Llmgw-Actor")
job, err := a.serve.Start(req)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusAccepted, job)
}
func (a *API) job(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotFound, "no such job")
return
}
job, ok := a.serve.Get(r.PathValue("id"))
if !ok {
writeError(w, http.StatusNotFound, "no such job; jobs are kept only until the control plane restarts")
return
}
writeJSON(w, http.StatusOK, job)
}
+552
View File
@@ -0,0 +1,552 @@
// Package modelserve takes a model from a registry repository to a replica
// answering requests, as one job.
//
// Adding a model was four commands and a hand-written YAML file: plan it,
// fetch it into the mirror, write its catalog entry with a memory figure
// somebody guessed, scale it — and then find out whether it loads. This runs
// those steps in the control plane, where the mirror, the registry, the
// catalog and the fleet already are, and reports each one as it goes, so
// whatever asks — ctl, a page — only has to watch.
package modelserve
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"regexp"
"slices"
"strconv"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
// Registry reads a repository.
type Registry interface {
Repo(ctx context.Context, id string) (hf.Repo, error)
}
// Mirror fetches files and reports on them.
type Mirror interface {
Fetch(ctx context.Context, req mirror.FetchRequest) (mirror.FetchResponse, error)
List(ctx context.Context) (mirror.ModelsResponse, error)
}
// Catalog adds a model.
type Catalog interface {
Add(ctx context.Context, m catalog.Model, by string) error
}
// Fleet is what the job watches once the model is in the catalog.
type Fleet interface {
Replicas() []reconciler.Replica
ModelHealth() []reconciler.ModelHealth
EngineKey(replicaID string) string
}
// Deps are what a job needs.
type Deps struct {
Registry Registry
Mirror Mirror
Catalog Catalog
Fleet Fleet
// LlamaVersion is the llama.cpp build the nodes install, which decides
// which architectures load.
LlamaVersion string
HTTP *http.Client
Log *slog.Logger
// Poll is how often the job looks at the mirror and the fleet.
Poll time.Duration
// ReadyTimeout bounds the wait for a replica: the node fetching the
// weights from the mirror and the engine loading them.
ReadyTimeout time.Duration
}
// Request is what to serve.
type Request struct {
// Repo is the registry repository, owner/name.
Repo string `json:"repo"`
// Quant picks the file by its quantisation, Q4_K_M unless given.
Quant string `json:"quant,omitempty"`
// File picks it by name instead.
File string `json:"file,omitempty"`
// Name is the catalog name; the repository suggests one otherwise.
Name string `json:"name,omitempty"`
Aliases []string `json:"aliases,omitempty"`
Description string `json:"description,omitempty"`
// Replicas to run; 0 makes it on demand, loaded when first asked for.
Replicas int `json:"replicas"`
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
// use. They default to 16384 and 4096, what four parallel slots divide.
CtxPool int `json:"ctx_pool,omitempty"`
ContextLength int `json:"context_length,omitempty"`
// By is who asked, for the catalog's record.
By string `json:"-"`
}
// The states a job and its steps can be in.
const (
Running = "running"
Done = "done"
Failed = "failed"
Skipped = "skipped"
)
// Step is one stage of a job.
type Step struct {
Name string `json:"name"`
State string `json:"state"`
Detail string `json:"detail,omitempty"`
At time.Time `json:"at"`
}
// Job is one model being served, and where it stands.
type Job struct {
ID string `json:"id"`
Request Request `json:"request"`
State string `json:"state"`
Error string `json:"error,omitempty"`
Model string `json:"model,omitempty"`
Steps []Step `json:"steps"`
Started time.Time `json:"started"`
// Reply is what the model said to the smoke test, and TokensPerSecond
// how fast it said it.
Reply string `json:"reply,omitempty"`
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
}
// Server runs jobs and remembers them.
type Server struct {
deps Deps
mu sync.Mutex
jobs map[string]*Job
seq int
}
// New builds a server.
func New(d Deps) *Server {
if d.Poll <= 0 {
d.Poll = 2 * time.Second
}
if d.ReadyTimeout <= 0 {
d.ReadyTimeout = 30 * time.Minute
}
if d.HTTP == nil {
d.HTTP = &http.Client{Timeout: 5 * time.Minute}
}
if d.Log == nil {
d.Log = slog.Default()
}
return &Server{deps: d, jobs: map[string]*Job{}}
}
var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
// Start begins a job and returns at once; the job runs on its own.
func (s *Server) Start(req Request) (Job, error) {
req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/")
if !repoPattern.MatchString(req.Repo) {
return Job{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo)
}
if req.Replicas < 0 {
return Job{}, errors.New("replicas cannot be negative")
}
if req.Quant == "" && req.File == "" {
req.Quant = "Q4_K_M"
}
if req.CtxPool <= 0 {
req.CtxPool = 16384
}
if req.ContextLength <= 0 {
req.ContextLength = 4096
}
s.mu.Lock()
s.seq++
j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req,
State: Running, Started: time.Now()}
s.jobs[j.ID] = j
s.mu.Unlock()
go s.run(j)
return s.snapshot(j), nil
}
// Get returns a job as it stands.
func (s *Server) Get(id string) (Job, bool) {
s.mu.Lock()
j, ok := s.jobs[id]
s.mu.Unlock()
if !ok {
return Job{}, false
}
return s.snapshot(j), true
}
func (s *Server) snapshot(j *Job) Job {
s.mu.Lock()
defer s.mu.Unlock()
out := *j
out.Steps = slices.Clone(j.Steps)
return out
}
// step starts a new step, or updates the detail of the current one.
func (s *Server) step(j *Job, name, state, detail string) {
s.mu.Lock()
defer s.mu.Unlock()
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
return
}
j.Steps = append(j.Steps, Step{Name: name, State: state, Detail: detail, At: time.Now()})
}
func (s *Server) fail(j *Job, name string, err error) {
s.step(j, name, Failed, err.Error())
s.mu.Lock()
j.State, j.Error = Failed, err.Error()
s.mu.Unlock()
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
}
func (s *Server) run(j *Job) {
ctx := context.Background()
req := j.Request
// 1. The registry: which files.
s.step(j, "registry", Running, "reading "+req.Repo)
repo, err := s.deps.Registry.Repo(ctx, req.Repo)
if err != nil {
s.fail(j, "registry", err)
return
}
if repo.Gated.Is() {
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
return
}
set, err := pick(repo, req)
if err != nil {
s.fail(j, "registry", err)
return
}
name := req.Name
if name == "" {
name = hf.ModelName(repo.ID)
}
s.mu.Lock()
j.Model = name
s.mu.Unlock()
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
// 2. The mirror: fetch, and check what arrived against what was published.
source, err := s.mirror(ctx, j, repo, set, name)
if err != nil {
s.fail(j, "mirror", err)
return
}
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
s.step(j, "check", Running, "reading its header from the mirror")
var header gguf.Header
if !strings.HasSuffix(source, "/") {
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
if !v.Servable {
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
return
}
header = v.Header
}
memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30)
basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine",
gb(set.Bytes), gb(int64(req.CtxPool)*header.KVBytesPerToken()), req.CtxPool)
if memGB == 0 {
// A set of shards, or a header that did not say: weights plus half
// again plus two, the rough figure from before shapes were read.
memGB = int(gb(set.Bytes)*1.5) + 2
basis = "weights plus half again plus two; the header gave no shape"
}
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
// 4. The catalog.
m := catalog.Model{
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
Profiles: []catalog.Profile{{
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB,
}},
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
}
s.step(j, "catalog", Running, "adding "+name)
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
s.fail(j, "catalog", err)
return
}
if req.Replicas == 0 {
s.step(j, "catalog", Done, name+" added on demand: it loads when first asked for")
s.finish(j)
return
}
s.step(j, "catalog", Done, fmt.Sprintf("%s added, %d replica(s)", name, req.Replicas))
// 5. A replica, ready.
rep, err := s.waitReady(ctx, j, name)
if err != nil {
s.fail(j, "placement", err)
return
}
// 6. Does it answer.
s.step(j, "smoke test", Running, "asking "+rep.ID+" to say hello")
reply, tps, err := s.smoke(ctx, rep, name)
if err != nil {
s.fail(j, "smoke test", err)
return
}
s.mu.Lock()
j.Reply, j.TokensPerSecond = reply, tps
s.mu.Unlock()
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
s.finish(j)
}
func (s *Server) finish(j *Job) {
s.mu.Lock()
j.State = Done
s.mu.Unlock()
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
}
// pick chooses the file set to serve.
func pick(repo hf.Repo, req Request) (hf.FileSet, error) {
var weights []hf.File
for _, f := range repo.Files {
lower := strings.ToLower(f.Path)
if !strings.HasSuffix(lower, ".gguf") || strings.Contains(lower, "mmproj") ||
strings.Contains(lower, "imatrix") {
continue
}
weights = append(weights, f)
}
sets := hf.Group(weights)
if len(sets) == 0 {
return hf.FileSet{}, fmt.Errorf("%s has no GGUF files; the nodes run llama.cpp, which needs one", repo.ID)
}
var names []string
for _, st := range sets {
names = append(names, st.Name)
if req.File != "" && (st.Name == req.File || slices.ContainsFunc(st.Files, func(f hf.File) bool { return f.Path == req.File })) {
return st, nil
}
}
if req.File == "" {
want := strings.ToLower(req.Quant)
var match []hf.FileSet
for _, st := range sets {
if strings.Contains(strings.ToLower(st.Name), want) {
match = append(match, st)
}
}
if len(match) == 1 {
return match[0], nil
}
if len(match) > 1 {
// Both a split and a joined copy of one quantisation: the joined
// one, which is one download to verify rather than several.
slices.SortFunc(match, func(a, b hf.FileSet) int { return a.Shards - b.Shards })
return match[0], nil
}
}
return hf.FileSet{}, fmt.Errorf("%s has no file matching %q; it has: %s",
repo.ID, or(req.File, req.Quant), strings.Join(names, ", "))
}
// mirror fetches the set into the mirror and returns the source a catalog
// entry names: the file, or the directory for a set of shards.
func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name string) (string, error) {
version := repo.ShortRevision()
s.step(j, "mirror", Running, "asking the mirror for "+set.Name)
var base string
for _, f := range set.Files {
file := mirror.LastPathElement(f.Path)
resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{
Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file,
})
// Already there is not a failure: the same commit fetched before.
if err != nil && !strings.Contains(err.Error(), "already") {
return "", err
}
base = resp.URL
}
deadline := time.Now().Add(6 * time.Hour)
for {
got, err := s.deps.Mirror.List(ctx)
if err != nil {
return "", err
}
var done, total int64
present, failed := 0, ""
sums := map[string]string{}
for _, st := range got.Models {
if st.Model != name || st.Version != version {
continue
}
done += st.BytesDone
total += max(st.BytesTotal, st.BytesDone)
switch st.State {
case mirror.Present:
present++
sums[st.File] = st.SHA256
case mirror.Failed:
failed = st.Err
}
}
if failed != "" {
return "", fmt.Errorf("the mirror could not fetch it: %s", failed)
}
if present >= len(set.Files) {
// What arrived against what the registry published: a mirror
// copies one file to every node, and is the one place a corrupted
// download is caught once instead of loaded everywhere.
for _, f := range set.Files {
file := mirror.LastPathElement(f.Path)
if f.SHA256 != "" && sums[file] != "" && !strings.EqualFold(f.SHA256, sums[file]) {
return "", fmt.Errorf("%s arrived with sha256 %s, and the registry published %s",
file, sums[file][:12], f.SHA256[:12])
}
}
source := base
if len(set.Files) > 1 {
source = strings.TrimSuffix(source, mirror.LastPathElement(source))
}
s.step(j, "mirror", Done, fmt.Sprintf("%.1f GB on the mirror, checksums match the registry", gb(total)))
return source, nil
}
if total > 0 {
s.step(j, "mirror", Running, fmt.Sprintf("fetching: %.1f of %.1f GB (%d%%)",
gb(done), gb(total), done*100/total))
}
if time.Now().After(deadline) {
return "", errors.New("the mirror had not finished after six hours")
}
time.Sleep(s.deps.Poll)
}
}
// waitReady waits for a replica of the model to serve, reporting what it is
// doing and giving up early when it cannot be placed or keeps failing.
func (s *Server) waitReady(ctx context.Context, j *Job, name string) (reconciler.Replica, error) {
s.step(j, "placement", Running, "waiting for the control plane to place it")
started := time.Now()
deadline := started.Add(s.deps.ReadyTimeout)
for {
for _, h := range s.deps.Fleet.ModelHealth() {
if h.Model != name {
continue
}
if h.Parked {
return reconciler.Replica{}, fmt.Errorf("it failed to start %d times and is parked: %s", h.Failures, h.Reason)
}
// A minute's grace from when this step began, not from the job:
// the download before it can take hours, and the first pass may
// not have run yet.
if h.Unplaceable && time.Since(started) > time.Minute {
return reconciler.Replica{}, fmt.Errorf("no node can host it:\n%s", h.Placement)
}
}
for _, r := range s.deps.Fleet.Replicas() {
if r.Model != name {
continue
}
if r.State == reconciler.Ready {
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
strings.Join(r.NodeIDs(), ","), r.Endpoint))
return r, nil
}
s.step(j, "placement", Running, fmt.Sprintf("%s on %s is %s", r.ID,
strings.Join(r.NodeIDs(), ","), r.State))
}
if time.Now().After(deadline) {
return reconciler.Replica{}, fmt.Errorf("no replica was ready after %s", s.deps.ReadyTimeout)
}
select {
case <-ctx.Done():
return reconciler.Replica{}, ctx.Err()
case <-time.After(s.deps.Poll):
}
}
}
// smoke sends one short request straight to the replica, with its engine key.
func (s *Server) smoke(ctx context.Context, r reconciler.Replica, name string) (string, float64, error) {
body, _ := json.Marshal(map[string]any{
"model": name,
"max_tokens": 32,
"messages": []map[string]string{{"role": "user", "content": "Say hello in one short sentence."}},
})
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
strings.TrimRight(r.Endpoint, "/")+"/v1/chat/completions", bytes.NewReader(body))
if err != nil {
return "", 0, err
}
req.Header.Set("Content-Type", "application/json")
if k := s.deps.Fleet.EngineKey(r.ID); k != "" {
req.Header.Set("Authorization", "Bearer "+k)
}
start := time.Now()
resp, err := s.deps.HTTP.Do(req)
if err != nil {
return "", 0, err
}
defer resp.Body.Close()
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if resp.StatusCode != http.StatusOK {
return "", 0, fmt.Errorf("the replica answered %s: %s", resp.Status, strings.TrimSpace(string(b)))
}
var out struct {
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
} `json:"choices"`
Usage struct {
CompletionTokens int `json:"completion_tokens"`
} `json:"usage"`
}
if err := json.Unmarshal(b, &out); err != nil || len(out.Choices) == 0 {
return "", 0, fmt.Errorf("the replica's answer was not a completion: %s", strings.TrimSpace(string(b)))
}
secs := time.Since(start).Seconds()
tps := 0.0
if secs > 0 {
tps = float64(out.Usage.CompletionTokens) / secs
}
return strings.TrimSpace(out.Choices[0].Message.Content), tps, nil
}
func gb(b int64) float64 { return float64(b) / (1 << 30) }
func or(a, b string) string {
if a != "" {
return a
}
return b
}
func orUnknown(arch string) string {
if arch == "" {
return "architecture not read"
}
return arch
}
@@ -0,0 +1,225 @@
package modelserve
import (
"bytes"
"context"
"encoding/binary"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
)
// ggufHeader is enough of a GGUF file for its architecture and shape.
func ggufHeader() []byte {
var b bytes.Buffer
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
u32 := func(k string, v uint32) { str(k); w(uint32(4)); w(v) }
b.WriteString("GGUF")
w(uint32(3))
w(uint64(0))
w(uint64(5))
str("general.architecture")
w(uint32(8))
str("qwen2")
u32("qwen2.block_count", 28)
u32("qwen2.attention.head_count", 12)
u32("qwen2.attention.head_count_kv", 2)
u32("qwen2.embedding_length", 1536)
return b.Bytes()
}
type fakeRegistry struct{ repo hf.Repo }
func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil }
// fakeMirror serves the header at every file URL and reports each fetched
// file present, with the checksum it was told to.
type fakeMirror struct {
mu sync.Mutex
srv *httptest.Server
got []mirror.FetchRequest
sums map[string]string
}
func (m *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.got = append(m.got, r)
return mirror.FetchResponse{URL: m.srv.URL + "/" + r.Model + "/" + r.Version + "/" + r.File}, nil
}
func (m *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out mirror.ModelsResponse
for _, r := range m.got {
out.Models = append(out.Models, mirror.Status{Model: r.Model, Version: r.Version, File: r.File,
State: mirror.Present, BytesDone: 1 << 30, BytesTotal: 1 << 30, SHA256: m.sums[r.File]})
}
return out, nil
}
type fakeCatalog struct {
mu sync.Mutex
added []catalog.Model
}
func (c *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
c.mu.Lock()
defer c.mu.Unlock()
c.added = append(c.added, m)
return nil
}
// fakeFleet has a replica ready at engine once the catalog holds the model.
type fakeFleet struct {
cat *fakeCatalog
engine string
health []reconciler.ModelHealth
}
func (f *fakeFleet) Replicas() []reconciler.Replica {
f.cat.mu.Lock()
defer f.cat.mu.Unlock()
var out []reconciler.Replica
for _, m := range f.cat.added {
out = append(out, reconciler.Replica{ID: m.Name + "-r1", Model: m.Name,
State: reconciler.Ready, Endpoint: f.engine,
Shards: []reconciler.Shard{{NodeID: "node01"}}})
}
return out
}
func (f *fakeFleet) ModelHealth() []reconciler.ModelHealth { return f.health }
func (f *fakeFleet) EngineKey(string) string { return "k" }
func setup(t *testing.T, files []hf.File, sums map[string]string) (*Server, *fakeMirror, *fakeCatalog, *fakeFleet) {
t.Helper()
header := ggufHeader()
files0 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write(header)
}))
t.Cleanup(files0.Close)
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer k" {
w.WriteHeader(http.StatusUnauthorized)
return
}
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hello!"}}],"usage":{"completion_tokens":3}}`))
}))
t.Cleanup(engine.Close)
m := &fakeMirror{srv: files0, sums: sums}
c := &fakeCatalog{}
f := &fakeFleet{cat: c, engine: engine.URL}
s := New(Deps{
Registry: fakeRegistry{repo: hf.Repo{ID: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abcdef123456", Files: files}},
Mirror: m, Catalog: c, Fleet: f, Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second,
})
return s, m, c, f
}
func wait(t *testing.T, s *Server, id string) Job {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for time.Now().Before(deadline) {
if j, _ := s.Get(id); j.State != Running {
return j
}
time.Sleep(10 * time.Millisecond)
}
t.Fatal("the job never finished")
return Job{}
}
func TestAModelGoesFromTheRegistryToAnAnsweringReplica(t *testing.T) {
files := []hf.File{
{Path: "Qwen2.5-1.5B-Instruct-Q8_0.gguf", Bytes: 2 << 30, SHA256: "aa"},
{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"},
{Path: "mmproj-Q4_K_M.gguf", Bytes: 1 << 20},
}
s, m, c, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
if err != nil {
t.Fatal(err)
}
j := wait(t, s, job.ID)
if j.State != Done {
t.Fatalf("job %s: %s\n%+v", j.State, j.Error, j.Steps)
}
if len(m.got) != 1 || m.got[0].File != "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" || m.got[0].Version != "abcdef12" {
t.Errorf("fetched %+v; expected the Q4_K_M file at its revision", m.got)
}
mod := c.added[0]
if mod.Name != "qwen2.5-1.5b-instruct" || mod.Profiles[0].MinMemoryGB == 0 || mod.Scaling.Min != 1 {
t.Errorf("catalog entry %+v", mod)
}
if j.Reply != "Hello!" {
t.Errorf("the smoke test's answer was %q", j.Reply)
}
var names []string
for _, st := range j.Steps {
names = append(names, st.Name)
}
if strings.Join(names, ",") != "registry,mirror,check,catalog,placement,smoke test" {
t.Errorf("steps %v", names)
}
}
func TestAQuantTheRepositoryDoesNotHaveIsNamedWithWhatItHas(t *testing.T) {
s, _, c, _ := setup(t, []hf.File{{Path: "m-Q8_0.gguf", Bytes: 1}}, nil)
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
j := wait(t, s, job.ID)
if j.State != Failed || !strings.Contains(j.Error, "m-Q8_0.gguf") {
t.Errorf("%s: %s", j.State, j.Error)
}
if len(c.added) != 0 {
t.Error("something was added to the catalog")
}
}
func TestADownloadThatDoesNotMatchTheRegistryIsNotServed(t *testing.T) {
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
s, _, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
j := wait(t, s, job.ID)
if j.State != Failed || !strings.Contains(j.Error, "registry published") {
t.Errorf("%s: %s", j.State, j.Error)
}
if len(c.added) != 0 {
t.Error("a corrupted download reached the catalog")
}
}
func TestAParkedModelStopsTheWait(t *testing.T) {
s, _, _, f := setup(t, []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30}}, nil)
f.health = []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Parked: true, Failures: 5, Reason: "unknown model architecture"}}
// A fleet where it never becomes ready, only parked.
s.deps.Fleet = parkedFleet{f}
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
j := wait(t, s, job.ID)
if j.State != Failed || !strings.Contains(j.Error, "parked") {
t.Errorf("%s: %s", j.State, j.Error)
}
}
type parkedFleet struct{ *fakeFleet }
func (parkedFleet) Replicas() []reconciler.Replica { return nil }
func TestARepositoryThatIsNotOneIsRefused(t *testing.T) {
s := New(Deps{})
for _, r := range []string{"", "noslash", "../etc/passwd", "a/b/c"} {
if _, err := s.Start(Request{Repo: r}); err == nil {
t.Errorf("%q accepted", r)
}
}
}
+13
View File
@@ -558,3 +558,16 @@ func Group(files []File) []FileSet {
}
return out
}
// ModelName is the catalog name a repository suggests: its last element,
// lowercased, with the -GGUF marker dropped. The owner is not part of what
// anybody calls the model, and "-GGUF" describes the file format.
func ModelName(repoID string) string {
name := repoID
if i := strings.LastIndex(name, "/"); i >= 0 {
name = name[i+1:]
}
name = strings.TrimSuffix(name, "-GGUF")
name = strings.TrimSuffix(name, "-gguf")
return strings.ToLower(name)
}
+1 -6
View File
@@ -621,12 +621,7 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
// is not part of what anybody calls the model, and "-GGUF" describes the file
// format rather than the model, which is already clear from the file being
// fetched.
func defaultModelName(repoID string) string {
name := mirror.LastPathElement(repoID)
name = strings.TrimSuffix(name, "-GGUF")
name = strings.TrimSuffix(name, "-gguf")
return strings.ToLower(name)
}
func defaultModelName(repoID string) string { return hf.ModelName(repoID) }
// verdictFor judges a registry file set by the repository's architecture. A
// vision projector is its own architecture whatever the repository says.