Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
adb1b3cf2a | ||
|
|
bd12b0909a |
No files matched your search
@@ -77,6 +77,29 @@ more than 26 hours old or there has never been one. docs/install.md now says how
|
||||
to restore, including why running engines survive it: the engine-key secret
|
||||
comes back with the store.
|
||||
|
||||
### Replicas follow load between min and max
|
||||
|
||||
A model's `max` was a ceiling nothing reached: every model ran exactly its
|
||||
minimum however long its queue. Now, when requests are waiting at the engines —
|
||||
every slot of every replica busy — at one or more per replica for a minute, one
|
||||
replica is added, up to `max`; ten minutes with nothing waiting takes one of
|
||||
those away, down to `min`. One step at a time, and not while the previous one
|
||||
is still starting, so a spike that is over before a replica loads does not
|
||||
leave several behind. Each decision is logged with the load that made it.
|
||||
`llmgw ctl scale MODEL --min 1 --max 3` turns it on for a model.
|
||||
|
||||
### A model from the registry in one command
|
||||
|
||||
`llmgw ctl model serve OWNER/REPO` takes a model from the registry to a replica
|
||||
that answers. It picks the GGUF file (`--quant`, Q4_K_M by default), has the
|
||||
mirror fetch it and checks what arrived against the registry's published
|
||||
sha256, checks that the nodes' llama.cpp can load it, sizes its memory from its
|
||||
header, adds it to the catalog, waits for a replica — stopping early with the
|
||||
reason if it fits nowhere or keeps failing — and asks it to say hello. Each step
|
||||
is shown as it happens. It runs in the control plane as a job, so a download of
|
||||
hours does not depend on a terminal staying open. `--on-demand` adds it without
|
||||
starting a replica.
|
||||
|
||||
### Upgrading from one command
|
||||
|
||||
`sudo llmgw ctl upgrade VERSION`, run on the control plane, upgrades that
|
||||
|
||||
@@ -28,7 +28,8 @@ func newModelCmd(client func() *ctlClient) *cobra.Command {
|
||||
Use: "model",
|
||||
Short: "Add or remove models while the cluster is running",
|
||||
}
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client))
|
||||
c.AddCommand(newModelAddCmd(client), newModelRemoveCmd(client), newModelPlanCmd(client),
|
||||
newModelServeCmd(client))
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
)
|
||||
|
||||
func newModelServeCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
req modelserve.Request
|
||||
onDemand bool
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "serve OWNER/REPO",
|
||||
Short: "Take a model from the registry to a replica that answers, in one command",
|
||||
Long: "Picks the GGUF file (--quant, Q4_K_M by default, or --file), has the mirror\n" +
|
||||
"fetch it and checks what arrived against the registry's checksum, checks\n" +
|
||||
"that the nodes' llama.cpp can load it and sizes its memory from its own\n" +
|
||||
"header, adds it to the catalog, waits for a replica, and asks it to say\n" +
|
||||
"hello. Each step is shown as it happens; the first that fails stops it.\n\n" +
|
||||
" llmgw ctl model serve bartowski/Qwen2.5-3B-Instruct-GGUF\n" +
|
||||
" llmgw ctl model serve Qwen/Qwen2.5-7B-Instruct-GGUF --quant Q5_K_M --name qwen-7b\n" +
|
||||
" llmgw ctl model serve owner/big-GGUF --on-demand",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Repo = args[0]
|
||||
if onDemand {
|
||||
req.Replicas = 0
|
||||
}
|
||||
body, _ := json.Marshal(req)
|
||||
var job modelserve.Job
|
||||
if err := client().post(cmd.Context(), "/api/v1/models/serve", body, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "job %s\n", job.ID)
|
||||
shown := map[string]string{}
|
||||
for {
|
||||
for _, s := range job.Steps {
|
||||
line := s.State + " " + s.Detail
|
||||
if shown[s.Name] == line {
|
||||
continue
|
||||
}
|
||||
shown[s.Name] = line
|
||||
mark := map[string]string{modelserve.Running: "..", modelserve.Done: "ok",
|
||||
modelserve.Failed: "!!"}[s.State]
|
||||
fmt.Fprintf(out, "%-3s %-11s %s\n", mark, s.Name, s.Detail)
|
||||
}
|
||||
switch job.State {
|
||||
case modelserve.Done:
|
||||
fmt.Fprintf(out, "\n%s is served.\n", job.Model)
|
||||
return nil
|
||||
case modelserve.Failed:
|
||||
return errors.New(job.Error)
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
if err := client().get(cmd.Context(), "/api/v1/jobs/"+job.ID, &job); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
f := c.Flags()
|
||||
f.StringVar(&req.Quant, "quant", "", "the quantisation to serve (default Q4_K_M)")
|
||||
f.StringVar(&req.File, "file", "", "the file to serve, instead of choosing by --quant")
|
||||
f.StringVar(&req.Name, "name", "", "the catalog name (default: from the repository)")
|
||||
f.StringSliceVar(&req.Aliases, "alias", nil, "other names clients may use; repeatable")
|
||||
f.StringVar(&req.Description, "description", "", "what it is for, shown to clients in /v1/models")
|
||||
f.IntVar(&req.Replicas, "replicas", 1, "how many replicas to run")
|
||||
f.BoolVar(&onDemand, "on-demand", false, "run none until first asked for")
|
||||
f.IntVar(&req.CtxPool, "ctx-pool", 16384, "llama.cpp's --ctx-size, shared by its parallel slots")
|
||||
f.IntVar(&req.ContextLength, "context-length", 4096, "what one request may use")
|
||||
return c
|
||||
}
|
||||
+20
-1
@@ -5,6 +5,7 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -171,8 +172,16 @@ func newServerCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Kept past this block: serving a model from the registry needs
|
||||
// both the admin API and the mirror, and the mirror is set up
|
||||
// below.
|
||||
var (
|
||||
adminAPI *adminapi.API
|
||||
catalogSvc *modelsvc.Service
|
||||
)
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
adminAPI = api
|
||||
api.SetReleases(cfg.Releases)
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
// only when there is a store. Without one, the endpoints say so
|
||||
@@ -190,6 +199,7 @@ func newServerCmd() *cobra.Command {
|
||||
admin.ReplaceCatalog(ctx, built)
|
||||
}
|
||||
api.SetModels(svc)
|
||||
catalogSvc = svc
|
||||
// And the mirror page, so a model that has just been
|
||||
// fetched can be served without writing YAML by hand.
|
||||
rt.SetCatalogAdmin(svc)
|
||||
@@ -259,7 +269,16 @@ func newServerCmd() *cobra.Command {
|
||||
// Browsing the registry is what turns "paste a resolve URL"
|
||||
// into "find the model". It needs a way out to the internet
|
||||
// from this machine, which the mirror's own fetching does not.
|
||||
rt.SetRegistry(hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout))
|
||||
registry := hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)
|
||||
rt.SetRegistry(registry)
|
||||
// One command from a registry repository to a replica that
|
||||
// answers: `llmgw ctl model serve`.
|
||||
if adminAPI != nil && catalogSvc != nil {
|
||||
adminAPI.SetServe(modelserve.New(modelserve.Deps{
|
||||
Registry: registry, Mirror: mc, Catalog: catalogSvc, Fleet: admin,
|
||||
LlamaVersion: cfg.WantedEngines()["llamacpp"], Log: log,
|
||||
}))
|
||||
}
|
||||
log.Info("model mirror configured",
|
||||
"api", cfg.Mirror.API, "mtls", cfg.Mirror.TLS.Enabled())
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -33,6 +34,8 @@ import (
|
||||
type API struct {
|
||||
// releases is where release binaries are fetched from, for upgrades.
|
||||
releases string
|
||||
// serve runs registry-to-replica jobs; nil without a mirror and a store.
|
||||
serve *modelserve.Server
|
||||
|
||||
models Models
|
||||
capacity Capacity
|
||||
@@ -83,6 +86,8 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||
mux.HandleFunc("POST /api/v1/models/serve", a.serveModel)
|
||||
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
}
|
||||
@@ -1204,3 +1209,39 @@ func (a *API) upgradeNode(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]string{"node": r.PathValue("id"), "version": req.Version})
|
||||
}
|
||||
|
||||
// SetServe supplies what serves a model from the registry.
|
||||
func (a *API) SetServe(s *modelserve.Server) { a.serve = s }
|
||||
|
||||
func (a *API) serveModel(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotImplemented,
|
||||
"serving from the registry needs a mirror and a store, and this control plane lacks one")
|
||||
return
|
||||
}
|
||||
var req modelserve.Request
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<16)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
req.By = r.Header.Get("X-Llmgw-Actor")
|
||||
job, err := a.serve.Start(req)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, job)
|
||||
}
|
||||
|
||||
func (a *API) job(w http.ResponseWriter, r *http.Request) {
|
||||
if a.serve == nil {
|
||||
writeError(w, http.StatusNotFound, "no such job")
|
||||
return
|
||||
}
|
||||
job, ok := a.serve.Get(r.PathValue("id"))
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "no such job; jobs are kept only until the control plane restarts")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, job)
|
||||
}
|
||||
@@ -134,6 +134,16 @@ func (c *Controller) collectUsage() {
|
||||
return
|
||||
}
|
||||
c.rec.RecordUsage(c.usage.LastUsed())
|
||||
// And what the replicas are doing, when the source can say: what adds
|
||||
// and removes replicas between a model's min and max.
|
||||
if l, ok := c.usage.(LoadSource); ok {
|
||||
c.rec.RecordLoad(l.ModelLoad())
|
||||
}
|
||||
}
|
||||
|
||||
// LoadSource reports each model's replicas' load, keyed by catalog name.
|
||||
type LoadSource interface {
|
||||
ModelLoad() map[string]reconciler.Load
|
||||
}
|
||||
|
||||
// ReplaceCatalog puts a new catalog in force, without a restart.
|
||||
|
||||
@@ -0,0 +1,552 @@
|
||||
// Package modelserve takes a model from a registry repository to a replica
|
||||
// answering requests, as one job.
|
||||
//
|
||||
// Adding a model was four commands and a hand-written YAML file: plan it,
|
||||
// fetch it into the mirror, write its catalog entry with a memory figure
|
||||
// somebody guessed, scale it — and then find out whether it loads. This runs
|
||||
// those steps in the control plane, where the mirror, the registry, the
|
||||
// catalog and the fleet already are, and reports each one as it goes, so
|
||||
// whatever asks — ctl, a page — only has to watch.
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// Registry reads a repository.
|
||||
type Registry interface {
|
||||
Repo(ctx context.Context, id string) (hf.Repo, error)
|
||||
}
|
||||
|
||||
// Mirror fetches files and reports on them.
|
||||
type Mirror interface {
|
||||
Fetch(ctx context.Context, req mirror.FetchRequest) (mirror.FetchResponse, error)
|
||||
List(ctx context.Context) (mirror.ModelsResponse, error)
|
||||
}
|
||||
|
||||
// Catalog adds a model.
|
||||
type Catalog interface {
|
||||
Add(ctx context.Context, m catalog.Model, by string) error
|
||||
}
|
||||
|
||||
// Fleet is what the job watches once the model is in the catalog.
|
||||
type Fleet interface {
|
||||
Replicas() []reconciler.Replica
|
||||
ModelHealth() []reconciler.ModelHealth
|
||||
EngineKey(replicaID string) string
|
||||
}
|
||||
|
||||
// Deps are what a job needs.
|
||||
type Deps struct {
|
||||
Registry Registry
|
||||
Mirror Mirror
|
||||
Catalog Catalog
|
||||
Fleet Fleet
|
||||
// LlamaVersion is the llama.cpp build the nodes install, which decides
|
||||
// which architectures load.
|
||||
LlamaVersion string
|
||||
HTTP *http.Client
|
||||
Log *slog.Logger
|
||||
// Poll is how often the job looks at the mirror and the fleet.
|
||||
Poll time.Duration
|
||||
// ReadyTimeout bounds the wait for a replica: the node fetching the
|
||||
// weights from the mirror and the engine loading them.
|
||||
ReadyTimeout time.Duration
|
||||
}
|
||||
|
||||
// Request is what to serve.
|
||||
type Request struct {
|
||||
// Repo is the registry repository, owner/name.
|
||||
Repo string `json:"repo"`
|
||||
// Quant picks the file by its quantisation, Q4_K_M unless given.
|
||||
Quant string `json:"quant,omitempty"`
|
||||
// File picks it by name instead.
|
||||
File string `json:"file,omitempty"`
|
||||
// Name is the catalog name; the repository suggests one otherwise.
|
||||
Name string `json:"name,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
// Replicas to run; 0 makes it on demand, loaded when first asked for.
|
||||
Replicas int `json:"replicas"`
|
||||
// CtxPool is llama.cpp's --ctx-size; ContextLength what one request may
|
||||
// use. They default to 16384 and 4096, what four parallel slots divide.
|
||||
CtxPool int `json:"ctx_pool,omitempty"`
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
// By is who asked, for the catalog's record.
|
||||
By string `json:"-"`
|
||||
}
|
||||
|
||||
// The states a job and its steps can be in.
|
||||
const (
|
||||
Running = "running"
|
||||
Done = "done"
|
||||
Failed = "failed"
|
||||
Skipped = "skipped"
|
||||
)
|
||||
|
||||
// Step is one stage of a job.
|
||||
type Step struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Job is one model being served, and where it stands.
|
||||
type Job struct {
|
||||
ID string `json:"id"`
|
||||
Request Request `json:"request"`
|
||||
State string `json:"state"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
Steps []Step `json:"steps"`
|
||||
Started time.Time `json:"started"`
|
||||
// Reply is what the model said to the smoke test, and TokensPerSecond
|
||||
// how fast it said it.
|
||||
Reply string `json:"reply,omitempty"`
|
||||
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
|
||||
}
|
||||
|
||||
// Server runs jobs and remembers them.
|
||||
type Server struct {
|
||||
deps Deps
|
||||
mu sync.Mutex
|
||||
jobs map[string]*Job
|
||||
seq int
|
||||
}
|
||||
|
||||
// New builds a server.
|
||||
func New(d Deps) *Server {
|
||||
if d.Poll <= 0 {
|
||||
d.Poll = 2 * time.Second
|
||||
}
|
||||
if d.ReadyTimeout <= 0 {
|
||||
d.ReadyTimeout = 30 * time.Minute
|
||||
}
|
||||
if d.HTTP == nil {
|
||||
d.HTTP = &http.Client{Timeout: 5 * time.Minute}
|
||||
}
|
||||
if d.Log == nil {
|
||||
d.Log = slog.Default()
|
||||
}
|
||||
return &Server{deps: d, jobs: map[string]*Job{}}
|
||||
}
|
||||
|
||||
var repoPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
|
||||
// Start begins a job and returns at once; the job runs on its own.
|
||||
func (s *Server) Start(req Request) (Job, error) {
|
||||
req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/")
|
||||
if !repoPattern.MatchString(req.Repo) {
|
||||
return Job{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo)
|
||||
}
|
||||
if req.Replicas < 0 {
|
||||
return Job{}, errors.New("replicas cannot be negative")
|
||||
}
|
||||
if req.Quant == "" && req.File == "" {
|
||||
req.Quant = "Q4_K_M"
|
||||
}
|
||||
if req.CtxPool <= 0 {
|
||||
req.CtxPool = 16384
|
||||
}
|
||||
if req.ContextLength <= 0 {
|
||||
req.ContextLength = 4096
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.seq++
|
||||
j := &Job{ID: fmt.Sprintf("serve-%d-%d", time.Now().Unix(), s.seq), Request: req,
|
||||
State: Running, Started: time.Now()}
|
||||
s.jobs[j.ID] = j
|
||||
s.mu.Unlock()
|
||||
go s.run(j)
|
||||
return s.snapshot(j), nil
|
||||
}
|
||||
|
||||
// Get returns a job as it stands.
|
||||
func (s *Server) Get(id string) (Job, bool) {
|
||||
s.mu.Lock()
|
||||
j, ok := s.jobs[id]
|
||||
s.mu.Unlock()
|
||||
if !ok {
|
||||
return Job{}, false
|
||||
}
|
||||
return s.snapshot(j), true
|
||||
}
|
||||
|
||||
func (s *Server) snapshot(j *Job) Job {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := *j
|
||||
out.Steps = slices.Clone(j.Steps)
|
||||
return out
|
||||
}
|
||||
|
||||
// step starts a new step, or updates the detail of the current one.
|
||||
func (s *Server) step(j *Job, name, state, detail string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
|
||||
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
|
||||
return
|
||||
}
|
||||
j.Steps = append(j.Steps, Step{Name: name, State: state, Detail: detail, At: time.Now()})
|
||||
}
|
||||
|
||||
func (s *Server) fail(j *Job, name string, err error) {
|
||||
s.step(j, name, Failed, err.Error())
|
||||
s.mu.Lock()
|
||||
j.State, j.Error = Failed, err.Error()
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
|
||||
}
|
||||
|
||||
func (s *Server) run(j *Job) {
|
||||
ctx := context.Background()
|
||||
req := j.Request
|
||||
|
||||
// 1. The registry: which files.
|
||||
s.step(j, "registry", Running, "reading "+req.Repo)
|
||||
repo, err := s.deps.Registry.Repo(ctx, req.Repo)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
if repo.Gated.Is() {
|
||||
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
|
||||
return
|
||||
}
|
||||
set, err := pick(repo, req)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
name := req.Name
|
||||
if name == "" {
|
||||
name = hf.ModelName(repo.ID)
|
||||
}
|
||||
s.mu.Lock()
|
||||
j.Model = name
|
||||
s.mu.Unlock()
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
|
||||
|
||||
// 2. The mirror: fetch, and check what arrived against what was published.
|
||||
source, err := s.mirror(ctx, j, repo, set, name)
|
||||
if err != nil {
|
||||
s.fail(j, "mirror", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
var header gguf.Header
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
v := gguf.Check(ctx, source, s.deps.LlamaVersion)
|
||||
if !v.Servable {
|
||||
s.fail(j, "check", fmt.Errorf("llama.cpp %s on the nodes cannot run this: %s", s.deps.LlamaVersion, v.Reason))
|
||||
return
|
||||
}
|
||||
header = v.Header
|
||||
}
|
||||
memGB := int((header.MemoryBytes(set.Bytes, req.CtxPool) + 1<<30 - 1) >> 30)
|
||||
basis := fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens + engine",
|
||||
gb(set.Bytes), gb(int64(req.CtxPool)*header.KVBytesPerToken()), req.CtxPool)
|
||||
if memGB == 0 {
|
||||
// A set of shards, or a header that did not say: weights plus half
|
||||
// again plus two, the rough figure from before shapes were read.
|
||||
memGB = int(gb(set.Bytes)*1.5) + 2
|
||||
basis = "weights plus half again plus two; the header gave no shape"
|
||||
}
|
||||
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
|
||||
|
||||
// 4. The catalog.
|
||||
m := catalog.Model{
|
||||
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||
Profiles: []catalog.Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB,
|
||||
}},
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
}
|
||||
s.step(j, "catalog", Running, "adding "+name)
|
||||
if err := s.deps.Catalog.Add(ctx, m, req.By); err != nil {
|
||||
s.fail(j, "catalog", err)
|
||||
return
|
||||
}
|
||||
if req.Replicas == 0 {
|
||||
s.step(j, "catalog", Done, name+" added on demand: it loads when first asked for")
|
||||
s.finish(j)
|
||||
return
|
||||
}
|
||||
s.step(j, "catalog", Done, fmt.Sprintf("%s added, %d replica(s)", name, req.Replicas))
|
||||
|
||||
// 5. A replica, ready.
|
||||
rep, err := s.waitReady(ctx, j, name)
|
||||
if err != nil {
|
||||
s.fail(j, "placement", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 6. Does it answer.
|
||||
s.step(j, "smoke test", Running, "asking "+rep.ID+" to say hello")
|
||||
reply, tps, err := s.smoke(ctx, rep, name)
|
||||
if err != nil {
|
||||
s.fail(j, "smoke test", err)
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
j.Reply, j.TokensPerSecond = reply, tps
|
||||
s.mu.Unlock()
|
||||
s.step(j, "smoke test", Done, fmt.Sprintf("answered at %.1f tokens/s: %q", tps, reply))
|
||||
s.finish(j)
|
||||
}
|
||||
|
||||
func (s *Server) finish(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.State = Done
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
|
||||
}
|
||||
|
||||
// pick chooses the file set to serve.
|
||||
func pick(repo hf.Repo, req Request) (hf.FileSet, error) {
|
||||
var weights []hf.File
|
||||
for _, f := range repo.Files {
|
||||
lower := strings.ToLower(f.Path)
|
||||
if !strings.HasSuffix(lower, ".gguf") || strings.Contains(lower, "mmproj") ||
|
||||
strings.Contains(lower, "imatrix") {
|
||||
continue
|
||||
}
|
||||
weights = append(weights, f)
|
||||
}
|
||||
sets := hf.Group(weights)
|
||||
if len(sets) == 0 {
|
||||
return hf.FileSet{}, fmt.Errorf("%s has no GGUF files; the nodes run llama.cpp, which needs one", repo.ID)
|
||||
}
|
||||
var names []string
|
||||
for _, st := range sets {
|
||||
names = append(names, st.Name)
|
||||
if req.File != "" && (st.Name == req.File || slices.ContainsFunc(st.Files, func(f hf.File) bool { return f.Path == req.File })) {
|
||||
return st, nil
|
||||
}
|
||||
}
|
||||
if req.File == "" {
|
||||
want := strings.ToLower(req.Quant)
|
||||
var match []hf.FileSet
|
||||
for _, st := range sets {
|
||||
if strings.Contains(strings.ToLower(st.Name), want) {
|
||||
match = append(match, st)
|
||||
}
|
||||
}
|
||||
if len(match) == 1 {
|
||||
return match[0], nil
|
||||
}
|
||||
if len(match) > 1 {
|
||||
// Both a split and a joined copy of one quantisation: the joined
|
||||
// one, which is one download to verify rather than several.
|
||||
slices.SortFunc(match, func(a, b hf.FileSet) int { return a.Shards - b.Shards })
|
||||
return match[0], nil
|
||||
}
|
||||
}
|
||||
return hf.FileSet{}, fmt.Errorf("%s has no file matching %q; it has: %s",
|
||||
repo.ID, or(req.File, req.Quant), strings.Join(names, ", "))
|
||||
}
|
||||
|
||||
// mirror fetches the set into the mirror and returns the source a catalog
|
||||
// entry names: the file, or the directory for a set of shards.
|
||||
func (s *Server) mirror(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name string) (string, error) {
|
||||
version := repo.ShortRevision()
|
||||
s.step(j, "mirror", Running, "asking the mirror for "+set.Name)
|
||||
var base string
|
||||
for _, f := range set.Files {
|
||||
file := mirror.LastPathElement(f.Path)
|
||||
resp, err := s.deps.Mirror.Fetch(ctx, mirror.FetchRequest{
|
||||
Source: repo.DownloadURL("", f.Path), Model: name, Version: version, File: file,
|
||||
})
|
||||
// Already there is not a failure: the same commit fetched before.
|
||||
if err != nil && !strings.Contains(err.Error(), "already") {
|
||||
return "", err
|
||||
}
|
||||
base = resp.URL
|
||||
}
|
||||
deadline := time.Now().Add(6 * time.Hour)
|
||||
for {
|
||||
got, err := s.deps.Mirror.List(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var done, total int64
|
||||
present, failed := 0, ""
|
||||
sums := map[string]string{}
|
||||
for _, st := range got.Models {
|
||||
if st.Model != name || st.Version != version {
|
||||
continue
|
||||
}
|
||||
done += st.BytesDone
|
||||
total += max(st.BytesTotal, st.BytesDone)
|
||||
switch st.State {
|
||||
case mirror.Present:
|
||||
present++
|
||||
sums[st.File] = st.SHA256
|
||||
case mirror.Failed:
|
||||
failed = st.Err
|
||||
}
|
||||
}
|
||||
if failed != "" {
|
||||
return "", fmt.Errorf("the mirror could not fetch it: %s", failed)
|
||||
}
|
||||
if present >= len(set.Files) {
|
||||
// What arrived against what the registry published: a mirror
|
||||
// copies one file to every node, and is the one place a corrupted
|
||||
// download is caught once instead of loaded everywhere.
|
||||
for _, f := range set.Files {
|
||||
file := mirror.LastPathElement(f.Path)
|
||||
if f.SHA256 != "" && sums[file] != "" && !strings.EqualFold(f.SHA256, sums[file]) {
|
||||
return "", fmt.Errorf("%s arrived with sha256 %s, and the registry published %s",
|
||||
file, sums[file][:12], f.SHA256[:12])
|
||||
}
|
||||
}
|
||||
source := base
|
||||
if len(set.Files) > 1 {
|
||||
source = strings.TrimSuffix(source, mirror.LastPathElement(source))
|
||||
}
|
||||
s.step(j, "mirror", Done, fmt.Sprintf("%.1f GB on the mirror, checksums match the registry", gb(total)))
|
||||
return source, nil
|
||||
}
|
||||
if total > 0 {
|
||||
s.step(j, "mirror", Running, fmt.Sprintf("fetching: %.1f of %.1f GB (%d%%)",
|
||||
gb(done), gb(total), done*100/total))
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return "", errors.New("the mirror had not finished after six hours")
|
||||
}
|
||||
time.Sleep(s.deps.Poll)
|
||||
}
|
||||
}
|
||||
|
||||
// waitReady waits for a replica of the model to serve, reporting what it is
|
||||
// doing and giving up early when it cannot be placed or keeps failing.
|
||||
func (s *Server) waitReady(ctx context.Context, j *Job, name string) (reconciler.Replica, error) {
|
||||
s.step(j, "placement", Running, "waiting for the control plane to place it")
|
||||
started := time.Now()
|
||||
deadline := started.Add(s.deps.ReadyTimeout)
|
||||
for {
|
||||
for _, h := range s.deps.Fleet.ModelHealth() {
|
||||
if h.Model != name {
|
||||
continue
|
||||
}
|
||||
if h.Parked {
|
||||
return reconciler.Replica{}, fmt.Errorf("it failed to start %d times and is parked: %s", h.Failures, h.Reason)
|
||||
}
|
||||
// A minute's grace from when this step began, not from the job:
|
||||
// the download before it can take hours, and the first pass may
|
||||
// not have run yet.
|
||||
if h.Unplaceable && time.Since(started) > time.Minute {
|
||||
return reconciler.Replica{}, fmt.Errorf("no node can host it:\n%s", h.Placement)
|
||||
}
|
||||
}
|
||||
for _, r := range s.deps.Fleet.Replicas() {
|
||||
if r.Model != name {
|
||||
continue
|
||||
}
|
||||
if r.State == reconciler.Ready {
|
||||
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
|
||||
strings.Join(r.NodeIDs(), ","), r.Endpoint))
|
||||
return r, nil
|
||||
}
|
||||
s.step(j, "placement", Running, fmt.Sprintf("%s on %s is %s", r.ID,
|
||||
strings.Join(r.NodeIDs(), ","), r.State))
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return reconciler.Replica{}, fmt.Errorf("no replica was ready after %s", s.deps.ReadyTimeout)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return reconciler.Replica{}, ctx.Err()
|
||||
case <-time.After(s.deps.Poll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// smoke sends one short request straight to the replica, with its engine key.
|
||||
func (s *Server) smoke(ctx context.Context, r reconciler.Replica, name string) (string, float64, error) {
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"model": name,
|
||||
"max_tokens": 32,
|
||||
"messages": []map[string]string{{"role": "user", "content": "Say hello in one short sentence."}},
|
||||
})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
strings.TrimRight(r.Endpoint, "/")+"/v1/chat/completions", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if k := s.deps.Fleet.EngineKey(r.ID); k != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+k)
|
||||
}
|
||||
start := time.Now()
|
||||
resp, err := s.deps.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", 0, fmt.Errorf("the replica answered %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
var out struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &out); err != nil || len(out.Choices) == 0 {
|
||||
return "", 0, fmt.Errorf("the replica's answer was not a completion: %s", strings.TrimSpace(string(b)))
|
||||
}
|
||||
secs := time.Since(start).Seconds()
|
||||
tps := 0.0
|
||||
if secs > 0 {
|
||||
tps = float64(out.Usage.CompletionTokens) / secs
|
||||
}
|
||||
return strings.TrimSpace(out.Choices[0].Message.Content), tps, nil
|
||||
}
|
||||
|
||||
func gb(b int64) float64 { return float64(b) / (1 << 30) }
|
||||
|
||||
func or(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func orUnknown(arch string) string {
|
||||
if arch == "" {
|
||||
return "architecture not read"
|
||||
}
|
||||
return arch
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// ggufHeader is enough of a GGUF file for its architecture and shape.
|
||||
func ggufHeader() []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
u32 := func(k string, v uint32) { str(k); w(uint32(4)); w(v) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(5))
|
||||
str("general.architecture")
|
||||
w(uint32(8))
|
||||
str("qwen2")
|
||||
u32("qwen2.block_count", 28)
|
||||
u32("qwen2.attention.head_count", 12)
|
||||
u32("qwen2.attention.head_count_kv", 2)
|
||||
u32("qwen2.embedding_length", 1536)
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
type fakeRegistry struct{ repo hf.Repo }
|
||||
|
||||
func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil }
|
||||
|
||||
// fakeMirror serves the header at every file URL and reports each fetched
|
||||
// file present, with the checksum it was told to.
|
||||
type fakeMirror struct {
|
||||
mu sync.Mutex
|
||||
srv *httptest.Server
|
||||
got []mirror.FetchRequest
|
||||
sums map[string]string
|
||||
}
|
||||
|
||||
func (m *fakeMirror) Fetch(_ context.Context, r mirror.FetchRequest) (mirror.FetchResponse, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.got = append(m.got, r)
|
||||
return mirror.FetchResponse{URL: m.srv.URL + "/" + r.Model + "/" + r.Version + "/" + r.File}, nil
|
||||
}
|
||||
|
||||
func (m *fakeMirror) List(context.Context) (mirror.ModelsResponse, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var out mirror.ModelsResponse
|
||||
for _, r := range m.got {
|
||||
out.Models = append(out.Models, mirror.Status{Model: r.Model, Version: r.Version, File: r.File,
|
||||
State: mirror.Present, BytesDone: 1 << 30, BytesTotal: 1 << 30, SHA256: m.sums[r.File]})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type fakeCatalog struct {
|
||||
mu sync.Mutex
|
||||
added []catalog.Model
|
||||
}
|
||||
|
||||
func (c *fakeCatalog) Add(_ context.Context, m catalog.Model, _ string) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.added = append(c.added, m)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeFleet has a replica ready at engine once the catalog holds the model.
|
||||
type fakeFleet struct {
|
||||
cat *fakeCatalog
|
||||
engine string
|
||||
health []reconciler.ModelHealth
|
||||
}
|
||||
|
||||
func (f *fakeFleet) Replicas() []reconciler.Replica {
|
||||
f.cat.mu.Lock()
|
||||
defer f.cat.mu.Unlock()
|
||||
var out []reconciler.Replica
|
||||
for _, m := range f.cat.added {
|
||||
out = append(out, reconciler.Replica{ID: m.Name + "-r1", Model: m.Name,
|
||||
State: reconciler.Ready, Endpoint: f.engine,
|
||||
Shards: []reconciler.Shard{{NodeID: "node01"}}})
|
||||
}
|
||||
return out
|
||||
}
|
||||
func (f *fakeFleet) ModelHealth() []reconciler.ModelHealth { return f.health }
|
||||
func (f *fakeFleet) EngineKey(string) string { return "k" }
|
||||
|
||||
func setup(t *testing.T, files []hf.File, sums map[string]string) (*Server, *fakeMirror, *fakeCatalog, *fakeFleet) {
|
||||
t.Helper()
|
||||
header := ggufHeader()
|
||||
files0 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(header)
|
||||
}))
|
||||
t.Cleanup(files0.Close)
|
||||
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer k" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"Hello!"}}],"usage":{"completion_tokens":3}}`))
|
||||
}))
|
||||
t.Cleanup(engine.Close)
|
||||
|
||||
m := &fakeMirror{srv: files0, sums: sums}
|
||||
c := &fakeCatalog{}
|
||||
f := &fakeFleet{cat: c, engine: engine.URL}
|
||||
s := New(Deps{
|
||||
Registry: fakeRegistry{repo: hf.Repo{ID: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Revision: "abcdef123456", Files: files}},
|
||||
Mirror: m, Catalog: c, Fleet: f, Poll: 10 * time.Millisecond, ReadyTimeout: 5 * time.Second,
|
||||
})
|
||||
return s, m, c, f
|
||||
}
|
||||
|
||||
func wait(t *testing.T, s *Server, id string) Job {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if j, _ := s.Get(id); j.State != Running {
|
||||
return j
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatal("the job never finished")
|
||||
return Job{}
|
||||
}
|
||||
|
||||
func TestAModelGoesFromTheRegistryToAnAnsweringReplica(t *testing.T) {
|
||||
files := []hf.File{
|
||||
{Path: "Qwen2.5-1.5B-Instruct-Q8_0.gguf", Bytes: 2 << 30, SHA256: "aa"},
|
||||
{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"},
|
||||
{Path: "mmproj-Q4_K_M.gguf", Bytes: 1 << 20},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
|
||||
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Done {
|
||||
t.Fatalf("job %s: %s\n%+v", j.State, j.Error, j.Steps)
|
||||
}
|
||||
if len(m.got) != 1 || m.got[0].File != "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" || m.got[0].Version != "abcdef12" {
|
||||
t.Errorf("fetched %+v; expected the Q4_K_M file at its revision", m.got)
|
||||
}
|
||||
mod := c.added[0]
|
||||
if mod.Name != "qwen2.5-1.5b-instruct" || mod.Profiles[0].MinMemoryGB == 0 || mod.Scaling.Min != 1 {
|
||||
t.Errorf("catalog entry %+v", mod)
|
||||
}
|
||||
if j.Reply != "Hello!" {
|
||||
t.Errorf("the smoke test's answer was %q", j.Reply)
|
||||
}
|
||||
var names []string
|
||||
for _, st := range j.Steps {
|
||||
names = append(names, st.Name)
|
||||
}
|
||||
if strings.Join(names, ",") != "registry,mirror,check,catalog,placement,smoke test" {
|
||||
t.Errorf("steps %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAQuantTheRepositoryDoesNotHaveIsNamedWithWhatItHas(t *testing.T) {
|
||||
s, _, c, _ := setup(t, []hf.File{{Path: "m-Q8_0.gguf", Bytes: 1}}, nil)
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Failed || !strings.Contains(j.Error, "m-Q8_0.gguf") {
|
||||
t.Errorf("%s: %s", j.State, j.Error)
|
||||
}
|
||||
if len(c.added) != 0 {
|
||||
t.Error("something was added to the catalog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADownloadThatDoesNotMatchTheRegistryIsNotServed(t *testing.T) {
|
||||
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
|
||||
s, _, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Failed || !strings.Contains(j.Error, "registry published") {
|
||||
t.Errorf("%s: %s", j.State, j.Error)
|
||||
}
|
||||
if len(c.added) != 0 {
|
||||
t.Error("a corrupted download reached the catalog")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAParkedModelStopsTheWait(t *testing.T) {
|
||||
s, _, _, f := setup(t, []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30}}, nil)
|
||||
f.health = []reconciler.ModelHealth{{Model: "qwen2.5-1.5b-instruct", Parked: true, Failures: 5, Reason: "unknown model architecture"}}
|
||||
// A fleet where it never becomes ready, only parked.
|
||||
s.deps.Fleet = parkedFleet{f}
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
j := wait(t, s, job.ID)
|
||||
if j.State != Failed || !strings.Contains(j.Error, "parked") {
|
||||
t.Errorf("%s: %s", j.State, j.Error)
|
||||
}
|
||||
}
|
||||
|
||||
type parkedFleet struct{ *fakeFleet }
|
||||
|
||||
func (parkedFleet) Replicas() []reconciler.Replica { return nil }
|
||||
|
||||
func TestARepositoryThatIsNotOneIsRefused(t *testing.T) {
|
||||
s := New(Deps{})
|
||||
for _, r := range []string{"", "noslash", "../etc/passwd", "a/b/c"} {
|
||||
if _, err := s.Start(Request{Repo: r}); err == nil {
|
||||
t.Errorf("%q accepted", r)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,13 @@ type Config struct {
|
||||
// Without a bound, draining a node whose model cannot be placed anywhere
|
||||
// else would never finish.
|
||||
DrainGrace time.Duration
|
||||
// ScaleUpWaiting is the average number of requests waiting per ready
|
||||
// replica, held for ScaleUpAfter, that adds a replica — up to the model's
|
||||
// max. ScaleDownAfter is how long with nothing waiting before one of the
|
||||
// replicas added that way is taken away again, down to its min.
|
||||
ScaleUpWaiting float64
|
||||
ScaleUpAfter time.Duration
|
||||
ScaleDownAfter time.Duration
|
||||
// ParkAfter is how many failures in a row stop a model being retried.
|
||||
//
|
||||
// Backoff alone never gives up: it grows to MaxBackoff and keeps trying,
|
||||
@@ -140,6 +147,9 @@ func DefaultConfig() Config {
|
||||
MinLifetime: 5 * time.Minute,
|
||||
DrainGrace: 5 * time.Minute,
|
||||
ParkAfter: 5,
|
||||
ScaleUpWaiting: 1,
|
||||
ScaleUpAfter: time.Minute,
|
||||
ScaleDownAfter: 10 * time.Minute,
|
||||
PortRange: [2]int{20000, 20999},
|
||||
MgmtInterface: "eth0",
|
||||
}
|
||||
@@ -182,6 +192,15 @@ func (c *Config) normalize() {
|
||||
if c.ParkAfter <= 0 {
|
||||
c.ParkAfter = d.ParkAfter
|
||||
}
|
||||
if c.ScaleUpWaiting <= 0 {
|
||||
c.ScaleUpWaiting = d.ScaleUpWaiting
|
||||
}
|
||||
if c.ScaleUpAfter <= 0 {
|
||||
c.ScaleUpAfter = d.ScaleUpAfter
|
||||
}
|
||||
if c.ScaleDownAfter <= 0 {
|
||||
c.ScaleDownAfter = d.ScaleDownAfter
|
||||
}
|
||||
if c.PortRange[1] <= c.PortRange[0] {
|
||||
c.PortRange = d.PortRange
|
||||
}
|
||||
@@ -228,6 +247,11 @@ type Reconciler struct {
|
||||
restartQueue map[string][]string
|
||||
// parked holds models that failed ParkAfter times in a row.
|
||||
parked map[string]parkRecord
|
||||
// extra is how many replicas load has added above a model's base, and
|
||||
// busySince/quietSince when its load last crossed the thresholds.
|
||||
extra map[string]int
|
||||
busySince map[string]time.Time
|
||||
quietSince map[string]time.Time
|
||||
// placementLogged remembers the last "no placement" said per model, so it
|
||||
// is said when it changes rather than on every five-second pass.
|
||||
placementLogged map[string]placementNote
|
||||
@@ -261,6 +285,9 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
|
||||
wanted: map[string]time.Time{},
|
||||
restartQueue: map[string][]string{},
|
||||
parked: map[string]parkRecord{},
|
||||
extra: map[string]int{},
|
||||
busySince: map[string]time.Time{},
|
||||
quietSince: map[string]time.Time{},
|
||||
placementLogged: map[string]placementNote{},
|
||||
lastExplain: map[string]placement.Result{},
|
||||
wake: make(chan struct{}, 1),
|
||||
@@ -508,6 +535,19 @@ func (r *Reconciler) Want(model string) bool {
|
||||
// feature: a cluster with more models than GPUs can hold every model in its
|
||||
// catalog and only the wanted ones in its cards.
|
||||
func (r *Reconciler) desiredFor(m catalog.Model) int {
|
||||
base := r.baseFor(m)
|
||||
if base == 0 {
|
||||
return 0
|
||||
}
|
||||
r.mu.Lock()
|
||||
extra := r.extra[m.Name]
|
||||
r.mu.Unlock()
|
||||
return min(base+extra, max(m.Scaling.Max, base))
|
||||
}
|
||||
|
||||
// baseFor is what a model runs without load: its minimum, or for an on-demand
|
||||
// model one while somebody is asking and none once they stop.
|
||||
func (r *Reconciler) baseFor(m catalog.Model) int {
|
||||
if !m.Scaling.OnDemand() {
|
||||
return m.Scaling.Min
|
||||
}
|
||||
@@ -1715,3 +1755,76 @@ func firstLine(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Load is what a model's replicas are doing, as the router sees it.
|
||||
type Load struct {
|
||||
Replicas int
|
||||
Running int
|
||||
Waiting int
|
||||
}
|
||||
|
||||
// RecordLoad adds or takes away replicas between a model's min and max as its
|
||||
// load says.
|
||||
//
|
||||
// Max was a ceiling nothing reached: every model ran exactly its minimum
|
||||
// however long its queue. Now requests waiting at the engines — every slot of
|
||||
// every replica busy — held for ScaleUpAfter add one replica, and a quiet spell
|
||||
// of ScaleDownAfter takes one of those back. One step at a time, and not while
|
||||
// the last one is still starting: on a small CPU fleet each replica is minutes
|
||||
// of weights and gigabytes of memory, and a spike that is over by the time it
|
||||
// loads should not leave three behind.
|
||||
func (r *Reconciler) RecordLoad(loads map[string]Load) {
|
||||
now := r.now()
|
||||
for _, m := range r.cat.Load().Models() {
|
||||
base := r.baseFor(m)
|
||||
r.mu.Lock()
|
||||
if base == 0 || m.Scaling.Max <= base {
|
||||
delete(r.extra, m.Name)
|
||||
delete(r.busySince, m.Name)
|
||||
delete(r.quietSince, m.Name)
|
||||
r.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
starting := false
|
||||
for _, rep := range r.replicas {
|
||||
if rep.Model == m.Name && (rep.State == Placing || rep.State == Starting) {
|
||||
starting = true
|
||||
}
|
||||
}
|
||||
l := loads[m.Name]
|
||||
perReplica := float64(l.Waiting) / float64(max(l.Replicas, 1))
|
||||
extra := r.extra[m.Name]
|
||||
switch {
|
||||
case perReplica >= r.cfg.ScaleUpWaiting:
|
||||
delete(r.quietSince, m.Name)
|
||||
since, ok := r.busySince[m.Name]
|
||||
if !ok {
|
||||
r.busySince[m.Name] = now
|
||||
break
|
||||
}
|
||||
if now.Sub(since) >= r.cfg.ScaleUpAfter && base+extra < m.Scaling.Max && !starting {
|
||||
r.extra[m.Name] = extra + 1
|
||||
r.busySince[m.Name] = now
|
||||
r.log.Info("scaling up for load", "model", m.Name, "replicas", base+extra+1,
|
||||
"waiting_per_replica", perReplica, "for", now.Sub(since).Round(time.Second))
|
||||
}
|
||||
case l.Waiting == 0 && extra > 0:
|
||||
delete(r.busySince, m.Name)
|
||||
since, ok := r.quietSince[m.Name]
|
||||
if !ok {
|
||||
r.quietSince[m.Name] = now
|
||||
break
|
||||
}
|
||||
if now.Sub(since) >= r.cfg.ScaleDownAfter {
|
||||
r.extra[m.Name] = extra - 1
|
||||
r.quietSince[m.Name] = now
|
||||
r.log.Info("scaling down after a quiet spell", "model", m.Name,
|
||||
"replicas", base+extra-1, "quiet_for", now.Sub(since).Round(time.Second))
|
||||
}
|
||||
default:
|
||||
delete(r.busySince, m.Name)
|
||||
delete(r.quietSince, m.Name)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
}
|
||||
@@ -1932,3 +1932,74 @@ func TestAPlacementFailureIsLoggedWhenItChangesNotEveryPass(t *testing.T) {
|
||||
t.Errorf("health %+v", mh)
|
||||
}
|
||||
}
|
||||
|
||||
func liveCount(h *harness) int {
|
||||
n := 0
|
||||
for _, r := range h.r.Replicas() {
|
||||
if r.State.Counts() {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func TestLoadAddsReplicasUpToMaxAndQuietTakesThemBack(t *testing.T) {
|
||||
m := testModel(1)
|
||||
m.Scaling.Max = 3
|
||||
h := newHarness(t, m, gpuNode("1", 8), gpuNode("2", 8), gpuNode("3", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
busy := map[string]Load{"llama": {Replicas: 1, Running: 4, Waiting: 3}}
|
||||
|
||||
// A spike shorter than ScaleUpAfter adds nothing.
|
||||
h.r.RecordLoad(busy)
|
||||
h.advance(30 * time.Second)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := liveCount(h); n != 1 {
|
||||
t.Fatalf("%d replicas after 30s of load", n)
|
||||
}
|
||||
|
||||
// Held past it, one more.
|
||||
h.advance(31 * time.Second)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := liveCount(h); n != 2 {
|
||||
t.Fatalf("%d replicas after a minute of load; expected 2", n)
|
||||
}
|
||||
|
||||
// Not another while that one is still starting.
|
||||
h.advance(2 * time.Minute)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
if n := liveCount(h); n != 2 {
|
||||
t.Fatalf("stacked a replica on one still starting: %d", n)
|
||||
}
|
||||
|
||||
// Once it is up and load holds, the third — and never a fourth.
|
||||
h.markReady()
|
||||
for i := 0; i < 5; i++ {
|
||||
h.r.RecordLoad(busy)
|
||||
h.advance(61 * time.Second)
|
||||
h.r.RecordLoad(busy)
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
}
|
||||
if n := liveCount(h); n != 3 {
|
||||
t.Fatalf("%d replicas under sustained load; max is 3", n)
|
||||
}
|
||||
|
||||
// Quiet: back down one step per ScaleDownAfter, to min and no further.
|
||||
quiet := map[string]Load{"llama": {Replicas: 3}}
|
||||
for i := 0; i < 6; i++ {
|
||||
h.r.RecordLoad(quiet)
|
||||
h.advance(DefaultConfig().ScaleDownAfter + time.Second)
|
||||
h.r.RecordLoad(quiet)
|
||||
for j := 0; j < 3; j++ {
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
}
|
||||
if n := liveCount(h); n != 1 {
|
||||
t.Errorf("%d replicas after a long quiet spell; min is 1", n)
|
||||
}
|
||||
}
|
||||
@@ -558,3 +558,16 @@ func Group(files []File) []FileSet {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ModelName is the catalog name a repository suggests: its last element,
|
||||
// lowercased, with the -GGUF marker dropped. The owner is not part of what
|
||||
// anybody calls the model, and "-GGUF" describes the file format.
|
||||
func ModelName(repoID string) string {
|
||||
name := repoID
|
||||
if i := strings.LastIndex(name, "/"); i >= 0 {
|
||||
name = name[i+1:]
|
||||
}
|
||||
name = strings.TrimSuffix(name, "-GGUF")
|
||||
name = strings.TrimSuffix(name, "-gguf")
|
||||
return strings.ToLower(name)
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
@@ -510,6 +511,24 @@ func (r *Router) SetCatalog(c *catalog.Catalog) {
|
||||
// holding its cards out of habit.
|
||||
func (r *Router) LastUsed() map[string]time.Time { return r.reg.LastUsed() }
|
||||
|
||||
// ModelLoad sums what each model's replicas report: how many there are, and
|
||||
// requests running and waiting at their engines. Keyed by the catalog name,
|
||||
// which the control plane publishes first among each endpoint's names.
|
||||
func (r *Router) ModelLoad() map[string]reconciler.Load {
|
||||
out := map[string]reconciler.Load{}
|
||||
for _, s := range r.reg.Snapshots("") {
|
||||
if len(s.Models) == 0 {
|
||||
continue
|
||||
}
|
||||
l := out[s.Models[0]]
|
||||
l.Replicas++
|
||||
l.Running += s.Stats.Running
|
||||
l.Waiting += s.Stats.Waiting
|
||||
out[s.Models[0]] = l
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *Router) SetEndpoints(ctx context.Context, infos []endpoint.Info) {
|
||||
if r.prefix != nil {
|
||||
// A replica that has gone takes its cache with it. Leaving its entries
|
||||
|
||||
@@ -621,12 +621,7 @@ func (p *AccountPages) openRepo(r *http.Request, d *adminMirrorData, repo string
|
||||
// is not part of what anybody calls the model, and "-GGUF" describes the file
|
||||
// format rather than the model, which is already clear from the file being
|
||||
// fetched.
|
||||
func defaultModelName(repoID string) string {
|
||||
name := mirror.LastPathElement(repoID)
|
||||
name = strings.TrimSuffix(name, "-GGUF")
|
||||
name = strings.TrimSuffix(name, "-gguf")
|
||||
return strings.ToLower(name)
|
||||
}
|
||||
func defaultModelName(repoID string) string { return hf.ModelName(repoID) }
|
||||
|
||||
// verdictFor judges a registry file set by the repository's architecture. A
|
||||
// vision projector is its own architecture whatever the repository says.
|
||||
|
||||
Reference in new issue
Block a user