Compare commits
5
Commits
v0.10.4
...
v0.10.5-rc.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
235bf33dcd | ||
|
|
7099043c75 | ||
|
|
a32ea0e5ab | ||
|
|
d7f1c1d545 | ||
|
|
2ae12ae1fe |
No files matched your search
@@ -20,6 +20,54 @@ is only released once its **exit criterion** is met, not when its code lands.
|
||||
Until v1.0.0 the minor version is the breaking one: configuration formats and Go
|
||||
APIs change between minors without a deprecation period.
|
||||
|
||||
## Unreleased
|
||||
|
||||
Changes since v0.10.4, published as release candidates (v0.10.5-rc.N) until
|
||||
there are enough for v0.10.5.
|
||||
|
||||
### The mirror page in tabs
|
||||
|
||||
/admin/mirror was one long page: what the mirror holds, the Hugging Face
|
||||
explorer and the fetch form, one under another, with every visit waiting on
|
||||
Hugging Face for a list it might not show. It is now three tabs — Held (with
|
||||
how many are held, downloading and failed), Explore Hugging Face, and Fetch by
|
||||
URL — each with its own address. Only the tab shown does its work, and only
|
||||
Held reloads while something downloads. A download started from the explorer
|
||||
links to Held, where its progress is; a fetch the mirror refused comes back on
|
||||
the form it was sent from.
|
||||
|
||||
### A held model opens to where it runs, and how many
|
||||
|
||||
Clicking a model under Held opens a card beneath it: each replica with its
|
||||
node, state, age and the CPU and memory its engine uses, and how many more
|
||||
replicas the cluster has room for now, on which nodes, and why not more. The
|
||||
replicas it always runs and the most it grows to under load are changed there.
|
||||
Room is found by trying the scheduler itself on a copy of the inventory — the
|
||||
same decisions the next passes would make, not an estimate beside them — and a
|
||||
number of always-running replicas it cannot place is refused with the reason,
|
||||
rather than accepted and left short. A ceiling above the room is taken, and
|
||||
said. The same answer is at `GET /api/v1/models/{name}/headroom` on the admin
|
||||
API.
|
||||
|
||||
Each row also says how its model runs without being opened: replicas ready of
|
||||
those it should have, how many are still starting, and the nodes they are on —
|
||||
in amber when short, in red when parked or when it does not fit. It is one line
|
||||
per model beside its mode, and nodes drop the part of the name every node
|
||||
shares (node05 for llmgw-node05, the full name on hover).
|
||||
|
||||
### One line per record, in every table
|
||||
|
||||
The same rule across the admin pages. Headers do not wrap; a record is one
|
||||
line; free text is cut to its column with the whole of it on hover rather than
|
||||
given a row of its own; a row across the whole table is only a note or an open
|
||||
form. On the cluster page that means a node is one line (its address and logs
|
||||
beside its name, not under it), a model's description is a column rather than a
|
||||
second row, node names drop the prefix they all share, and the GPUs, classes,
|
||||
fabrics and fallbacks columns appear only when some row has something in them —
|
||||
on a CPU-only cluster they were three columns of dashes pushing the table past
|
||||
the page. On People, a person in many teams shows two and a count, the rest on
|
||||
hover.
|
||||
|
||||
## v0.10.4 — 2026-09-30
|
||||
|
||||
Published first as release candidates v0.10.4-rc.1 to rc.9.
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
@@ -91,6 +92,15 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /api/v1/jobs/{id}", a.job)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
v, ok := a.Headroom(r.PathValue("name"), limit)
|
||||
if !ok {
|
||||
http.Error(w, "no such model", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
})
|
||||
}
|
||||
|
||||
// --- responses ---
|
||||
@@ -336,6 +346,22 @@ type ExplainView struct {
|
||||
Rejections []RejectionView `json:"rejections,omitempty"`
|
||||
}
|
||||
|
||||
// HeadroomView is how many more replicas of a model fit now, and where.
|
||||
type HeadroomView struct {
|
||||
Model string `json:"model"`
|
||||
// Current is the replicas holding capacity: starting or ready.
|
||||
Current int `json:"current"`
|
||||
// More is how many more placement would find room for now.
|
||||
More int `json:"more"`
|
||||
// Nodes is where each of those would go.
|
||||
Nodes [][]string `json:"nodes,omitempty"`
|
||||
// Limited is set when More stopped at the limit asked rather than at the
|
||||
// cluster's edge: there may be room for more.
|
||||
Limited bool `json:"limited,omitempty"`
|
||||
// Blocked says why one more does not fit, a line per node considered.
|
||||
Blocked []string `json:"blocked,omitempty"`
|
||||
}
|
||||
|
||||
// RejectionView is one rejected candidate.
|
||||
type RejectionView struct {
|
||||
Profile string `json:"profile"`
|
||||
@@ -836,6 +862,20 @@ func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
||||
// It does not check that the model exists; the handler above does that so it
|
||||
// can answer 404, while the dashboard only ever asks about models it just read
|
||||
// out of the catalog.
|
||||
// Headroom tries placing up to limit more replicas of a model, on a copy of
|
||||
// the inventory, and says how many fit. A limit of zero or less takes 16.
|
||||
func (a *API) Headroom(model string, limit int) (HeadroomView, bool) {
|
||||
if limit <= 0 || limit > 64 {
|
||||
limit = 16
|
||||
}
|
||||
h, ok := a.ctl.Reconciler().Headroom(model, limit)
|
||||
if !ok {
|
||||
return HeadroomView{}, false
|
||||
}
|
||||
return HeadroomView{Model: model, Current: h.Current, More: h.More, Nodes: h.Nodes,
|
||||
Limited: h.Limited, Blocked: h.Blocked}, true
|
||||
}
|
||||
|
||||
func (a *API) Explain(model string) ExplainView {
|
||||
v := ExplainView{Model: model}
|
||||
for _, rep := range a.ctl.Reconciler().Replicas() {
|
||||
|
||||
@@ -392,6 +392,23 @@ func NewStore() *Store {
|
||||
return &Store{nodes: map[string]*Node{}, allocs: map[string][]Allocation{}}
|
||||
}
|
||||
|
||||
// Clone is a copy to try decisions on without making them: allocations made
|
||||
// on it are its own. Nodes are copied by value; the slices inside them are
|
||||
// shared, and a clone must not write through them.
|
||||
func (s *Store) Clone() *Store {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
c := NewStore()
|
||||
for id, n := range s.nodes {
|
||||
cp := *n
|
||||
c.nodes[id] = &cp
|
||||
}
|
||||
for id, list := range s.allocs {
|
||||
c.allocs[id] = slices.Clone(list)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Upsert records a node's registration, preserving operator-set state such as
|
||||
// a cordon: an agent reconnecting must not silently undo an operator's decision.
|
||||
// RecordSMChange keeps a port's change of master subnet manager, the last
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
)
|
||||
|
||||
// Headroom is how many more replicas of a model the cluster has room for now.
|
||||
type Headroom struct {
|
||||
// Current is the replicas the model has that hold capacity: starting or
|
||||
// ready, not those on their way out.
|
||||
Current int
|
||||
// More is how many more could be placed now, at most the limit asked.
|
||||
More int
|
||||
// Nodes is where each of those would go, in order, a replica's nodes
|
||||
// joined when it spans several.
|
||||
Nodes [][]string
|
||||
// Limited is set when More reached the limit asked, so there may be room
|
||||
// for more than that. A CPU profile that declares no memory is always
|
||||
// limited: nothing it asks for runs out.
|
||||
Limited bool
|
||||
// Blocked is why one more than More does not fit, a line per node that
|
||||
// was considered. Empty when Limited.
|
||||
Blocked []string
|
||||
}
|
||||
|
||||
// Headroom tries placing up to limit more replicas of a model on a copy of
|
||||
// the inventory, each one's allocation made on the copy before the next is
|
||||
// tried, exactly as the reconciler would place them one pass at a time.
|
||||
//
|
||||
// The scheduler's own decisions, not an estimate beside them: what the page
|
||||
// offers is then what the next passes will do, and a count that the
|
||||
// arithmetic of a second implementation allowed but placement refuses would
|
||||
// leave a model short with nobody told why.
|
||||
func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
if !ok {
|
||||
return Headroom{}, false
|
||||
}
|
||||
var h Headroom
|
||||
var avoid []string
|
||||
r.mu.Lock()
|
||||
for _, rep := range r.replicas {
|
||||
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
|
||||
h.Current++
|
||||
avoid = append(avoid, rep.NodeIDs()...)
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
|
||||
inv := r.inv.Clone()
|
||||
sched := placement.New(inv)
|
||||
for h.More < limit {
|
||||
res := sched.Place(m.PlacementProfiles(), avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
// As scaleUp does: spreading is a preference, not a rule.
|
||||
res = sched.Place(m.PlacementProfiles(), nil)
|
||||
}
|
||||
if res.Placement == nil {
|
||||
for _, rej := range res.Rejections {
|
||||
h.Blocked = append(h.Blocked, rej.String())
|
||||
}
|
||||
slices.Sort(h.Blocked)
|
||||
h.Blocked = slices.Compact(h.Blocked)
|
||||
return h, true
|
||||
}
|
||||
mem := 0
|
||||
if prof, ok := m.ProfileByName(res.Placement.Profile.Name); ok {
|
||||
mem = prof.MinMemoryGB
|
||||
}
|
||||
var nodes []string
|
||||
for _, a := range res.Placement.Nodes {
|
||||
inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem,
|
||||
})
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
avoid = append(avoid, nodes...)
|
||||
h.Nodes = append(h.Nodes, nodes)
|
||||
h.More++
|
||||
}
|
||||
h.Limited = true
|
||||
return h, true
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func TestHeadroomIsWhatPlacementWouldDoNext(t *testing.T) {
|
||||
// Two 8-GPU nodes and a 4-GPU profile: four replicas in all, one placed.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8), gpuNode("2", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
|
||||
got, ok := h.r.Headroom("llama", 10)
|
||||
if !ok {
|
||||
t.Fatal("the model was not found")
|
||||
}
|
||||
if got.Current != 1 || got.More != 3 || got.Limited {
|
||||
t.Fatalf("headroom %+v, expected 1 placed and room for 3 more", got)
|
||||
}
|
||||
// Spread first: the next one goes to the node without one.
|
||||
if first := got.Nodes[0][0]; first == h.r.Replicas()[0].NodeIDs()[0] {
|
||||
t.Errorf("the next replica would stack on %s beside the first", first)
|
||||
}
|
||||
if len(got.Blocked) == 0 || !strings.Contains(strings.Join(got.Blocked, "\n"), "GPU") {
|
||||
t.Errorf("no reason given for the fifth: %v", got.Blocked)
|
||||
}
|
||||
// Tried on a copy: nothing was taken from the real inventory.
|
||||
if free := len(h.inv.FreeGPUs("1")) + len(h.inv.FreeGPUs("2")); free != 12 {
|
||||
t.Errorf("%d GPUs free after asking, expected the 12 the one replica leaves", free)
|
||||
}
|
||||
|
||||
capped, _ := h.r.Headroom("llama", 2)
|
||||
if capped.More != 2 || !capped.Limited || capped.Blocked != nil {
|
||||
t.Errorf("asked for at most 2: %+v", capped)
|
||||
}
|
||||
if _, ok := h.r.Headroom("nope", 5); ok {
|
||||
t.Error("an unknown model has headroom")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadroomCountsMemoryAlreadyClaimed(t *testing.T) {
|
||||
cpu := func(id string) inventory.Node {
|
||||
return inventory.Node{ID: id, MgmtAddr: "10.0.0." + id, State: inventory.Ready, MemoryTotalMB: 16 * 1024}
|
||||
}
|
||||
m := catalog.Model{Name: "small", Source: "/m.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp", MinMemoryGB: 6}}}
|
||||
h := newHarness(t, m, cpu("a"), cpu("b"))
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
// 16 GB each at 6 GB a replica: two per node, one of them taken.
|
||||
got, _ := h.r.Headroom("small", 10)
|
||||
if got.Current != 1 || got.More != 3 {
|
||||
t.Errorf("headroom %+v, expected room for 3 more", got)
|
||||
}
|
||||
}
|
||||
@@ -7,11 +7,13 @@ import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
@@ -67,6 +69,18 @@ type modelUse struct {
|
||||
Name string
|
||||
Mode string // always, ondemand or stopped
|
||||
Replicas int
|
||||
// Min and Max are its scaling as the catalog holds it.
|
||||
Min, Max int
|
||||
// Ready, Wanted and Nodes are how it runs now, for the row itself:
|
||||
// replicas ready of those it should have, and the nodes they are on.
|
||||
// Fleet is false where nothing could be asked, so the row says nothing
|
||||
// rather than "0 of 0".
|
||||
Fleet bool
|
||||
Ready, Wanted int
|
||||
Starting int
|
||||
Nodes []nodeName
|
||||
Parked bool
|
||||
Unplaceable bool
|
||||
// Declared models are read from server.yaml on every start, so a mode
|
||||
// changed here would revert; the page says where to change it instead.
|
||||
Declared bool
|
||||
@@ -83,6 +97,11 @@ type mirrorRow struct {
|
||||
// operator clicked. Nil on a row that is still arriving: a catalog entry
|
||||
// pointing at a partial file is the thing versioned paths exist to prevent.
|
||||
Serve *serveForm
|
||||
// Detail is the card opened beneath the row: what runs it, on which
|
||||
// nodes, and how many replicas it runs.
|
||||
Detail *heldDetail
|
||||
// Open is the row's ?open= value.
|
||||
Open string
|
||||
|
||||
// HasPercent is false when the upstream never sent a length. The bar is
|
||||
// then not drawn at all: a progress bar that has to invent its own total is
|
||||
@@ -100,7 +119,11 @@ type mirrorRow struct {
|
||||
}
|
||||
|
||||
type adminMirrorData struct {
|
||||
Nav Nav
|
||||
Nav Nav
|
||||
// Tab is the part of the page shown: held, explore or fetch. Tabs are the
|
||||
// links to each.
|
||||
Tab string
|
||||
Tabs []mirrorTab
|
||||
Addr string
|
||||
Base string
|
||||
Rows []mirrorRow
|
||||
@@ -270,6 +293,8 @@ func (p *AccountPages) RegisterAdminMirror(mux *http.ServeMux, d MirrorDeps) {
|
||||
return name + " now runs all the time", nil
|
||||
}))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/replicas", p.mirrorAction(m, p.setReplicas))
|
||||
|
||||
mux.HandleFunc("POST /admin/mirror/serve", p.mirrorAction(m,
|
||||
func(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
model, err := buildModel(r.FormValue)
|
||||
@@ -361,7 +386,14 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
|
||||
d.HasCatalog = p.catalog != nil
|
||||
p.explore(r, &d)
|
||||
d.HasRegistry = p.registry != nil
|
||||
d.Tab = mirrorTabFor(r, errMsg)
|
||||
// Only the tab shown does its work: the registry is a server across the
|
||||
// internet, and asking it for sixty repositories to show somebody who came
|
||||
// to look at a download's progress made every visit wait on it.
|
||||
if d.Tab == tabExplore {
|
||||
p.explore(r, &d)
|
||||
}
|
||||
|
||||
got, err := m.List(r.Context())
|
||||
if err != nil {
|
||||
@@ -370,6 +402,7 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
if d.Error == "" {
|
||||
d.Error = err.Error()
|
||||
}
|
||||
d.Tabs = mirrorTabs(d)
|
||||
render(w, "admin_mirror", d)
|
||||
return
|
||||
}
|
||||
@@ -412,12 +445,46 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
}
|
||||
d.Rows = append(d.Rows, row)
|
||||
}
|
||||
var views map[string]adminapi.ModelView
|
||||
prefix := ""
|
||||
if p.fleet != nil && len(sources) > 0 {
|
||||
views = map[string]adminapi.ModelView{}
|
||||
for _, v := range p.fleet.Models() {
|
||||
views[v.Name] = v
|
||||
}
|
||||
var ids []string
|
||||
for _, n := range p.fleet.Nodes() {
|
||||
ids = append(ids, n.ID)
|
||||
}
|
||||
prefix = sharedPrefix(ids)
|
||||
}
|
||||
for i := range d.Rows {
|
||||
for _, m := range sources[d.Rows[i].URL] {
|
||||
d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, modelUse{
|
||||
u := modelUse{
|
||||
Name: m.Name, Mode: modeOf(m.Scaling), Replicas: max(m.Scaling.Max, 1),
|
||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||
Declared: p.catalog.Declared(m.Name),
|
||||
})
|
||||
}
|
||||
if v, ok := views[m.Name]; ok {
|
||||
u.Fleet, u.Ready, u.Wanted = true, v.Ready, v.Desired
|
||||
u.Parked, u.Unplaceable = v.Parked, v.Unplaceable
|
||||
var nodes []string
|
||||
nodes, u.Starting = runningOn(p.fleet.Replicas(m.Name))
|
||||
for _, n := range nodes {
|
||||
u.Nodes = append(u.Nodes, nodeName{Full: n, Short: strings.TrimPrefix(n, prefix)})
|
||||
}
|
||||
}
|
||||
d.Rows[i].UsedBy = append(d.Rows[i].UsedBy, u)
|
||||
}
|
||||
}
|
||||
|
||||
// The one row whose card is open. From the form too, so a change made in
|
||||
// the card comes back with the card still open to show it.
|
||||
open := r.FormValue("open")
|
||||
for i := range d.Rows {
|
||||
d.Rows[i].Open = openKey(d.Rows[i].Model, d.Rows[i].Version)
|
||||
if open != "" && d.Rows[i].Open == open && d.Rows[i].State == mirror.Present {
|
||||
d.Rows[i].Detail = p.heldDetailFor(d.Rows[i])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -462,12 +529,78 @@ func (p *AccountPages) renderMirror(w http.ResponseWriter, r *http.Request,
|
||||
|
||||
d.Groups = groupByModel(d.Rows)
|
||||
|
||||
if d.Fetching > 0 {
|
||||
// Only on the list: a reload anywhere else takes with it a search, an
|
||||
// open repository or a URL half typed.
|
||||
if d.Fetching > 0 && d.Tab == tabHeld && open == "" {
|
||||
d.RefreshSec = 5
|
||||
}
|
||||
d.Tabs = mirrorTabs(d)
|
||||
render(w, "admin_mirror", d)
|
||||
}
|
||||
|
||||
const (
|
||||
tabHeld = "held"
|
||||
tabExplore = "explore"
|
||||
tabFetch = "fetch"
|
||||
)
|
||||
|
||||
// mirrorTab is one of the page's tabs.
|
||||
type mirrorTab struct {
|
||||
Label string
|
||||
Count string // beside the label, empty for none
|
||||
Href string
|
||||
Active bool
|
||||
}
|
||||
|
||||
// mirrorTabFor is the tab a request is for.
|
||||
//
|
||||
// Named in the address when it is chosen, and otherwise read from what the
|
||||
// address carries, so links made before the page had tabs — a search, an
|
||||
// open repository — still land where they meant. After a change the list is
|
||||
// shown, where its effect is, unless it was refused: then the form it came
|
||||
// from, to try again.
|
||||
func mirrorTabFor(r *http.Request, errMsg string) string {
|
||||
valid := func(t string) bool { return t == tabHeld || t == tabExplore || t == tabFetch }
|
||||
if r.Method == http.MethodPost {
|
||||
if t := r.PostFormValue("tab"); errMsg != "" && valid(t) {
|
||||
return t
|
||||
}
|
||||
return tabHeld
|
||||
}
|
||||
q := r.URL.Query()
|
||||
if t := q.Get("tab"); valid(t) {
|
||||
return t
|
||||
}
|
||||
for _, k := range []string{"q", "repo", "view", "task", "size", "x"} {
|
||||
if q.Has(k) {
|
||||
return tabExplore
|
||||
}
|
||||
}
|
||||
return tabHeld
|
||||
}
|
||||
|
||||
func mirrorTabs(d adminMirrorData) []mirrorTab {
|
||||
held := mirrorTab{Label: "Held", Href: "/admin/mirror?tab=held", Active: d.Tab == tabHeld}
|
||||
switch {
|
||||
case d.Unreachable:
|
||||
held.Count = "unreachable"
|
||||
case d.Fetching > 0:
|
||||
held.Count = fmt.Sprintf("%d · %d downloading", d.Present, d.Fetching)
|
||||
default:
|
||||
held.Count = strconv.Itoa(d.Present)
|
||||
}
|
||||
if d.Failed > 0 {
|
||||
held.Count += fmt.Sprintf(" · %d failed", d.Failed)
|
||||
}
|
||||
tabs := []mirrorTab{held}
|
||||
if d.HasRegistry {
|
||||
tabs = append(tabs, mirrorTab{Label: "Explore Hugging Face",
|
||||
Href: "/admin/mirror?tab=explore", Active: d.Tab == tabExplore})
|
||||
}
|
||||
return append(tabs, mirrorTab{Label: "Fetch by URL",
|
||||
Href: "/admin/mirror?tab=fetch", Active: d.Tab == tabFetch})
|
||||
}
|
||||
|
||||
// mirrorGroup is a family of models the mirror holds — every qwen, every
|
||||
// llama — and their versions.
|
||||
type mirrorGroup struct {
|
||||
@@ -663,3 +796,43 @@ func writeJSONStatus(w http.ResponseWriter, code int, v any) {
|
||||
w.WriteHeader(code)
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
// runningOn is the nodes a model's replicas are on, each once and in order,
|
||||
// and how many of its replicas are not ready yet.
|
||||
func runningOn(reps []adminapi.ReplicaView) (nodes []string, starting int) {
|
||||
for _, r := range reps {
|
||||
if r.State != "ready" {
|
||||
starting++
|
||||
}
|
||||
for _, sh := range r.Shards {
|
||||
if !slices.Contains(nodes, sh.Node) {
|
||||
nodes = append(nodes, sh.Node)
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.Sort(nodes)
|
||||
return nodes, starting
|
||||
}
|
||||
|
||||
// nodeName is a node as a row names it: short, with the full name beside.
|
||||
type nodeName struct{ Short, Full string }
|
||||
|
||||
// sharedPrefix is what every node's name starts with, up to and including a
|
||||
// separator — "llmgw-" of llmgw-node01 and llmgw-node02 — so a row can say
|
||||
// node01 and fit on a line. Nothing when the names share no such part, or
|
||||
// when there is one node and dropping it would leave only a number.
|
||||
func sharedPrefix(ids []string) string {
|
||||
if len(ids) < 2 {
|
||||
return ""
|
||||
}
|
||||
p := ids[0]
|
||||
for _, id := range ids[1:] {
|
||||
for !strings.HasPrefix(id, p) {
|
||||
p = p[:len(p)-1]
|
||||
}
|
||||
}
|
||||
if i := strings.LastIndexAny(p, "-._"); i >= 0 {
|
||||
return p[:i+1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -341,16 +341,18 @@ func TestAFailedRowOffersRetryRatherThanRemove(t *testing.T) {
|
||||
}
|
||||
|
||||
type fakeRegistry struct {
|
||||
results []hf.Model
|
||||
repo hf.Repo
|
||||
err error
|
||||
asked string
|
||||
query hf.Query
|
||||
results []hf.Model
|
||||
repo hf.Repo
|
||||
err error
|
||||
asked string
|
||||
query hf.Query
|
||||
explored int
|
||||
}
|
||||
|
||||
func (f *fakeRegistry) Explore(_ context.Context, q hf.Query) ([]hf.Model, error) {
|
||||
f.asked = q.Search
|
||||
f.query = q
|
||||
f.explored++
|
||||
return f.results, f.err
|
||||
}
|
||||
|
||||
@@ -441,7 +443,11 @@ func TestWithNoRegistryThereIsNoSearchBox(t *testing.T) {
|
||||
if strings.Contains(body, "Search Hugging Face") {
|
||||
t.Error("the explorer should be absent without a registry")
|
||||
}
|
||||
if !strings.Contains(body, "Fetch a model") {
|
||||
if strings.Contains(body, "Explore Hugging Face") {
|
||||
t.Error("a tab for an explorer that is not there")
|
||||
}
|
||||
fetch := mirrorGet(t, mirrorPage(t, &fakeMirror{}, true), "/admin/mirror?tab=fetch").Body.String()
|
||||
if !strings.Contains(fetch, `name="source" required`) {
|
||||
t.Error("fetching by hand must still be there")
|
||||
}
|
||||
}
|
||||
@@ -464,8 +470,7 @@ func TestAShardedModelIsOneRowAndOneButton(t *testing.T) {
|
||||
"/admin/mirror?repo=Qwen/Qwen2.5-7B-Instruct").Body.String()
|
||||
|
||||
// Fifteen shards is one decision, not fifteen. Four here, and one form.
|
||||
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 2 {
|
||||
// One for the set, one for the manual form at the bottom.
|
||||
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 1 {
|
||||
t.Errorf("expected the four shards as a single form, found %d forms", n)
|
||||
}
|
||||
if !strings.Contains(body, "4 shards, fetched together") {
|
||||
@@ -534,8 +539,8 @@ func TestAlternativeQuantisationsStaySeparateRows(t *testing.T) {
|
||||
|
||||
// Two GGUFs are alternatives, not halves. Offering "fetch all" here would
|
||||
// spend gigabytes on a copy nobody asked for.
|
||||
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 3 {
|
||||
t.Errorf("expected a form each plus the manual one, found %d", n)
|
||||
if n := strings.Count(body, `action="/admin/mirror/fetch"`); n != 2 {
|
||||
t.Errorf("expected a form each, found %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -587,7 +592,7 @@ func TestHowAModelRunsIsChangedWhereItsFileIsShown(t *testing.T) {
|
||||
t.Error("the running model's mode is not shown as chosen")
|
||||
}
|
||||
// Declared in server.yaml: shown, not offered, since it would revert.
|
||||
if !strings.Contains(body, "on demand<br>set in server.yaml") {
|
||||
if !strings.Contains(body, `title="set in server.yaml, which is read again on every start">on demand · yaml</span>`) {
|
||||
t.Error("a declared model should say where its mode is set")
|
||||
}
|
||||
if strings.Count(body, `name="mode" class="mode"`) != 1 {
|
||||
@@ -655,3 +660,53 @@ func privateOK(m *mirror.Manager) *mirror.Manager {
|
||||
m.AllowPrivateSources()
|
||||
return m
|
||||
}
|
||||
|
||||
func TestEachTabDoesOnlyItsOwnWork(t *testing.T) {
|
||||
reg := &fakeRegistry{}
|
||||
mux := mirrorPageWith(t, &fakeMirror{models: []mirror.Status{
|
||||
{Model: "m", Version: "v", File: "m.gguf", State: mirror.Present, BytesDone: 1},
|
||||
{Model: "n", Version: "v", File: "n.gguf", State: mirror.Fetching, BytesDone: 1},
|
||||
}}, reg, true)
|
||||
|
||||
held := mirrorGet(t, mux, "/admin/mirror").Body.String()
|
||||
if reg.explored != 0 {
|
||||
t.Error("the list asked the registry for repositories it does not show")
|
||||
}
|
||||
if !strings.Contains(held, `class="on" aria-current="page">Held <span class="count">1 · 1 downloading</span>`) {
|
||||
t.Errorf("the Held tab should be chosen and count what it holds: %s", held)
|
||||
}
|
||||
if strings.Contains(held, `name="source" required`) || strings.Contains(held, "Search Hugging Face") {
|
||||
t.Error("other tabs' forms are on the list")
|
||||
}
|
||||
if !strings.Contains(held, `http-equiv="refresh"`) {
|
||||
t.Error("the list stopped following a download")
|
||||
}
|
||||
|
||||
explore := mirrorGet(t, mux, "/admin/mirror?tab=explore").Body.String()
|
||||
if reg.explored != 1 || !strings.Contains(explore, "Search Hugging Face") {
|
||||
t.Error("the explorer tab did not explore")
|
||||
}
|
||||
if strings.Contains(explore, `id="held"`) || strings.Contains(explore, `http-equiv="refresh"`) {
|
||||
t.Error("the explorer shows the list, or reloads away a search")
|
||||
}
|
||||
// A link from before there were tabs still lands on the explorer.
|
||||
if old := mirrorGet(t, mux, "/admin/mirror?q=qwen").Body.String(); !strings.Contains(old, "Search Hugging Face") {
|
||||
t.Error("a search address no longer opens the explorer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusedFetchReturnsToItsFormAndAStartedOneToTheList(t *testing.T) {
|
||||
m := &fakeMirror{fetchErr: errors.New("upstream said 404")}
|
||||
mux := mirrorPage(t, m, true)
|
||||
body := mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, "source": {"https://h.example/m.gguf"}, "model": {"m"},
|
||||
"version": {"v"}, "force": {"1"}, "tab": {"fetch"}}).Body.String()
|
||||
if !strings.Contains(body, "upstream said 404") || !strings.Contains(body, `name="source" required`) {
|
||||
t.Errorf("a refusal should come back on the form it came from: %s", body)
|
||||
}
|
||||
m.fetchErr = nil
|
||||
body = mirrorPost(t, mux, "/admin/mirror/fetch", url.Values{"csrf": {"tok"}, "source": {"https://h.example/m.gguf"}, "model": {"m"},
|
||||
"version": {"v"}, "force": {"1"}, "tab": {"fetch"}}).Body.String()
|
||||
if !strings.Contains(body, `id="held"`) {
|
||||
t.Errorf("a started download should be shown where it is followed: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,13 @@ type DashboardData struct {
|
||||
|
||||
HasPeople bool
|
||||
|
||||
// NodePrefix is what every node's name starts with, left off in tables
|
||||
// so a row fits on a line; the full name is on hover.
|
||||
NodePrefix string
|
||||
// Columns shown only when some row has something in them: a CPU-only
|
||||
// cluster's nodes table was three columns of dashes wide.
|
||||
AnyGPUs, AnyClasses, AnyFabrics, AnyFallbacks bool
|
||||
|
||||
// Charts are pre-rendered rather than assembled in the template: building
|
||||
// SVG out of template fragments is how a chart ends up unescaped.
|
||||
RequestChart template.HTML
|
||||
@@ -177,6 +184,17 @@ func Dashboard(src Source, usage UsageSource, version string, refresh time.Durat
|
||||
}
|
||||
}
|
||||
d.Unplaced = unplaced(src, d.Models)
|
||||
ids := make([]string, 0, len(d.Nodes))
|
||||
for _, n := range d.Nodes {
|
||||
ids = append(ids, n.ID)
|
||||
d.AnyGPUs = d.AnyGPUs || n.GPUTotal > 0
|
||||
d.AnyClasses = d.AnyClasses || len(n.Classes) > 0
|
||||
d.AnyFabrics = d.AnyFabrics || len(n.Fabrics) > 0
|
||||
}
|
||||
d.NodePrefix = sharedPrefix(ids)
|
||||
for _, m := range d.Models {
|
||||
d.AnyFallbacks = d.AnyFallbacks || len(m.Fallbacks) > 0
|
||||
}
|
||||
d.Summary = src.Summary()
|
||||
if fs, ok := src.(FabricSource); ok {
|
||||
r := fs.Fabrics()
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// HeadroomSource is a fleet that can say how many more replicas of a model
|
||||
// fit now. Optional: without it the card shows what runs and offers no change
|
||||
// it cannot check.
|
||||
type HeadroomSource interface {
|
||||
Headroom(model string, limit int) (adminapi.HeadroomView, bool)
|
||||
}
|
||||
|
||||
// maxReplicas bounds what the form accepts. Not a cluster limit, a typing one:
|
||||
// 300 where 3 was meant would otherwise be taken as a ceiling.
|
||||
const maxReplicas = 64
|
||||
|
||||
// heldDetail is the card opened beneath a held version: what runs it, where,
|
||||
// and room to change it.
|
||||
type heldDetail struct {
|
||||
Models []heldModel
|
||||
}
|
||||
|
||||
// heldModel is one catalog model running a held version.
|
||||
type heldModel struct {
|
||||
modelUse
|
||||
View adminapi.ModelView
|
||||
Replicas []heldReplica
|
||||
Headroom *adminapi.HeadroomView
|
||||
// MostNow is how many replicas could always run, Current plus the room
|
||||
// found; zero when the room was not measured or had no end in sight.
|
||||
MostNow int
|
||||
}
|
||||
|
||||
// heldReplica is one replica, by the nodes it runs on.
|
||||
type heldReplica struct {
|
||||
ID string
|
||||
State string
|
||||
Restart string
|
||||
Age int64
|
||||
Endpoint string
|
||||
Nodes []heldShard
|
||||
}
|
||||
|
||||
type heldShard struct {
|
||||
Node string
|
||||
MemoryGB int // the node's RAM, 0 when unknown
|
||||
CPU float64
|
||||
MemBytes int64
|
||||
}
|
||||
|
||||
// openKey names a row the way ?open= does.
|
||||
func openKey(model, version string) string { return model + ":" + version }
|
||||
|
||||
// heldDetailFor reads, for each model using a row, how it runs and how much
|
||||
// more room it has.
|
||||
func (p *AccountPages) heldDetailFor(row mirrorRow) *heldDetail {
|
||||
d := &heldDetail{}
|
||||
if p.fleet == nil {
|
||||
for _, u := range row.UsedBy {
|
||||
d.Models = append(d.Models, heldModel{modelUse: u})
|
||||
}
|
||||
return d
|
||||
}
|
||||
views := map[string]adminapi.ModelView{}
|
||||
for _, v := range p.fleet.Models() {
|
||||
views[v.Name] = v
|
||||
}
|
||||
memory := map[string]int{}
|
||||
for _, n := range p.fleet.Nodes() {
|
||||
memory[n.ID] = n.MemoryGB
|
||||
}
|
||||
hs, _ := p.fleet.(HeadroomSource)
|
||||
for _, u := range row.UsedBy {
|
||||
hm := heldModel{modelUse: u, View: views[u.Name]}
|
||||
for _, rep := range p.fleet.Replicas(u.Name) {
|
||||
// Built field by field: a ReplicaView carries the engine's key,
|
||||
// which is for the admin listener and never for a page.
|
||||
hr := heldReplica{ID: rep.ID, State: rep.State, Restart: rep.Restart,
|
||||
Age: rep.AgeSec, Endpoint: rep.Endpoint}
|
||||
for _, sh := range rep.Shards {
|
||||
hr.Nodes = append(hr.Nodes, heldShard{Node: sh.Node, MemoryGB: memory[sh.Node],
|
||||
CPU: sh.CPUCores, MemBytes: sh.MemoryBytes})
|
||||
}
|
||||
hm.Replicas = append(hm.Replicas, hr)
|
||||
}
|
||||
if hs != nil {
|
||||
if h, ok := hs.Headroom(u.Name, 16); ok {
|
||||
hm.Headroom = &h
|
||||
if !h.Limited {
|
||||
hm.MostNow = h.Current + h.More
|
||||
}
|
||||
}
|
||||
}
|
||||
d.Models = append(d.Models, hm)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// setReplicas changes how many replicas a model always runs and how far it
|
||||
// may grow, refusing an always-on count the cluster has no room for.
|
||||
//
|
||||
// Refused rather than taken and left short: a minimum placement cannot meet
|
||||
// is a model reported as unplaceable on every pass from then on, which is the
|
||||
// same answer given later and less plainly. A maximum above the room is
|
||||
// accepted and said: it is reached only under load, by which time something
|
||||
// may have made room, and refusing it would make the ceiling depend on what
|
||||
// happened to be running when it was set.
|
||||
func (p *AccountPages) setReplicas(ctx context.Context, admin store.User, r *http.Request) (string, error) {
|
||||
name := r.FormValue("model")
|
||||
lo, err1 := strconv.Atoi(strings.TrimSpace(r.FormValue("min")))
|
||||
hi, err2 := strconv.Atoi(strings.TrimSpace(r.FormValue("max")))
|
||||
switch {
|
||||
case name == "":
|
||||
return "", errors.New("which model?")
|
||||
case err1 != nil || err2 != nil:
|
||||
return "", errors.New("replicas are whole numbers")
|
||||
case lo < 0 || hi < lo:
|
||||
return "", fmt.Errorf("always running (%d) cannot be more than the most under load (%d)", lo, hi)
|
||||
case hi > maxReplicas:
|
||||
return "", fmt.Errorf("at most %d replicas", maxReplicas)
|
||||
}
|
||||
current, err := p.currentModel(ctx, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if p.catalog.Declared(name) {
|
||||
return "", fmt.Errorf("%s is set in server.yaml; change it there", name)
|
||||
}
|
||||
|
||||
note := ""
|
||||
if hs, ok := p.fleet.(HeadroomSource); ok && hi > 0 {
|
||||
h, ok := hs.Headroom(name, hi)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("no model %q", name)
|
||||
}
|
||||
fits := h.Current + h.More
|
||||
if lo > fits && !h.Limited {
|
||||
why := "nothing more fits"
|
||||
if len(h.Blocked) > 0 {
|
||||
why = strings.Join(h.Blocked, "; ")
|
||||
}
|
||||
return "", fmt.Errorf("%s can always run at most %d now (%d running, room for %d more): %s",
|
||||
name, fits, h.Current, h.More, why)
|
||||
}
|
||||
if hi > fits && !h.Limited {
|
||||
note = fmt.Sprintf("; there is room for %d now, so it grows past that only if room is made", fits)
|
||||
}
|
||||
}
|
||||
|
||||
sc := catalog.Scaling{Min: lo, Max: hi, IdleTimeout: current.Scaling.IdleTimeout}
|
||||
if sc.OnDemand() && sc.IdleTimeout <= 0 {
|
||||
sc.IdleTimeout = defaultIdle
|
||||
}
|
||||
if _, err := p.catalog.Scale(ctx, name, sc, admin.Email); err != nil {
|
||||
return "", err
|
||||
}
|
||||
switch {
|
||||
case hi == 0:
|
||||
return name + " is stopped; its replicas are being taken down", nil
|
||||
case lo == 0:
|
||||
return fmt.Sprintf("%s loads when asked for, up to %d under load%s", name, hi, note), nil
|
||||
case lo == hi:
|
||||
return fmt.Sprintf("%s runs %s", name, plural(lo, "replica", "replicas")), nil
|
||||
}
|
||||
return fmt.Sprintf("%s runs %d, and up to %d under load%s", name, lo, hi, note), nil
|
||||
}
|
||||
|
||||
// currentModel is a model as the catalog holds it now.
|
||||
func (p *AccountPages) currentModel(ctx context.Context, name string) (catalog.Model, error) {
|
||||
sources, err := p.catalog.Sources(ctx)
|
||||
if err != nil {
|
||||
return catalog.Model{}, err
|
||||
}
|
||||
for _, ms := range sources {
|
||||
for _, m := range ms {
|
||||
if m.Name == name {
|
||||
return m, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return catalog.Model{}, fmt.Errorf("no model %q runs from the mirror", name)
|
||||
}
|
||||
|
||||
// age renders seconds the way the replicas table does.
|
||||
func age(sec int64) string {
|
||||
d := time.Duration(sec) * time.Second
|
||||
switch {
|
||||
case d < time.Minute:
|
||||
return fmt.Sprintf("%ds", sec)
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%dh%02dm", int(d.Hours()), int(d.Minutes())%60)
|
||||
}
|
||||
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// roomyFleet runs one replica of "coder" on n2 and has room for `more`.
|
||||
type roomyFleet struct {
|
||||
fakeFleet
|
||||
more int
|
||||
blocked []string
|
||||
asked int
|
||||
}
|
||||
|
||||
func (f *roomyFleet) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{{ID: "n1", MemoryGB: 16}, {ID: "n2", MemoryGB: 32}}
|
||||
}
|
||||
func (f *roomyFleet) Models() []adminapi.ModelView {
|
||||
return []adminapi.ModelView{{Name: "coder", Min: 1, Max: 1, Ready: 1, Total: 1, Desired: 1}}
|
||||
}
|
||||
func (f *roomyFleet) Replicas(string) []adminapi.ReplicaView {
|
||||
return []adminapi.ReplicaView{{ID: "coder-abc", Model: "coder", State: "ready", AgeSec: 3700,
|
||||
EngineKey: "secret-engine-key",
|
||||
Shards: []adminapi.ShardView{{Node: "n2", CPUCores: 3.5, MemoryBytes: 6 << 30}}}}
|
||||
}
|
||||
func (f *roomyFleet) Headroom(model string, limit int) (adminapi.HeadroomView, bool) {
|
||||
f.asked = limit
|
||||
more := min(f.more, limit)
|
||||
h := adminapi.HeadroomView{Model: model, Current: 1, More: more}
|
||||
for range more {
|
||||
h.Nodes = append(h.Nodes, []string{"n1"})
|
||||
}
|
||||
if more < limit {
|
||||
h.Blocked = f.blocked
|
||||
} else {
|
||||
h.Limited = true
|
||||
}
|
||||
return h, true
|
||||
}
|
||||
|
||||
func detailPage(t *testing.T, fleet *roomyFleet, cat *fakeCatalog) func(string) string {
|
||||
t.Helper()
|
||||
m := &fakeMirror{models: []mirror.Status{
|
||||
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 5 << 30},
|
||||
}}
|
||||
mux := mirrorPageDeps(t, MirrorDeps{Mirror: m, Catalog: cat, Fleet: fleet}, true)
|
||||
return func(path string) string { return mirrorGet(t, mux, path).Body.String() }
|
||||
}
|
||||
|
||||
func coderCatalog() *fakeCatalog {
|
||||
return &fakeCatalog{sources: map[string][]catalog.Model{"http://mirror:8081/coder/v1/c.gguf": {
|
||||
{Name: "coder", Scaling: catalog.Scaling{Min: 1, Max: 1}}}}}
|
||||
}
|
||||
|
||||
func TestAHeldModelOpensToWhereItRunsAndHowMuchRoomIsLeft(t *testing.T) {
|
||||
fleet := &roomyFleet{more: 2, blocked: []string{"n2: 2 GB free of 32, profile needs 6"}}
|
||||
get := detailPage(t, fleet, coderCatalog())
|
||||
|
||||
closed := get("/admin/mirror")
|
||||
if strings.Contains(closed, "held-card") {
|
||||
t.Error("a card is open before anything was clicked")
|
||||
}
|
||||
if !strings.Contains(closed, `href="/admin/mirror?tab=held&open=coder%3av1"`) {
|
||||
t.Error("the model's name does not open its card")
|
||||
}
|
||||
|
||||
body := get("/admin/mirror?tab=held&open=coder:v1")
|
||||
for _, want := range []string{
|
||||
"held-card", `<a href="/admin/nodes/n2">n2</a>`, "32 GB", "coder-abc", "1h01m", "3.5 cores", "6.0 GiB",
|
||||
"Room for 2 more now, on n1, n1.", "n2: 2 GB free of 32, profile needs 6",
|
||||
`name="min" value="1" min="0" max="3"`, `action="/admin/mirror/replicas"`,
|
||||
`name="open" value="coder:v1"`,
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("the card is missing %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(body, "secret-engine-key") {
|
||||
t.Error("the engine's key reached the page")
|
||||
}
|
||||
if strings.Contains(body, `http-equiv="refresh"`) {
|
||||
t.Error("a page with an open form reloads under the operator")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAReplicaCountTheClusterHasNoRoomForIsRefused(t *testing.T) {
|
||||
fleet := &roomyFleet{more: 1, blocked: []string{"n1: 1 GB free of 16, profile needs 6"}}
|
||||
cat := coderCatalog()
|
||||
mux := mirrorPageDeps(t, MirrorDeps{Mirror: &fakeMirror{models: []mirror.Status{
|
||||
{Model: "coder", Version: "v1", File: "c.gguf", State: mirror.Present, BytesDone: 1},
|
||||
}}, Catalog: cat, Fleet: fleet}, true)
|
||||
post := func(lo, hi string) string {
|
||||
return mirrorPost(t, mux, "/admin/mirror/replicas", url.Values{"csrf": {"tok"},
|
||||
"model": {"coder"}, "min": {lo}, "max": {hi}, "open": {"coder:v1"}}).Body.String()
|
||||
}
|
||||
|
||||
body := post("3", "3")
|
||||
if len(cat.scaled) != 0 {
|
||||
t.Fatalf("three always-on replicas were set with room for two: %+v", cat.lastScale)
|
||||
}
|
||||
if !strings.Contains(body, "can always run at most 2 now") || !strings.Contains(body, "1 GB free of 16") {
|
||||
t.Errorf("the refusal should say how many fit and why: %s", body)
|
||||
}
|
||||
if !strings.Contains(body, "held-card") {
|
||||
t.Error("the card closed on the refusal")
|
||||
}
|
||||
|
||||
body = post("2", "4")
|
||||
if len(cat.scaled) != 1 || cat.lastScale.Min != 2 || cat.lastScale.Max != 4 {
|
||||
t.Fatalf("what fits was not set: %+v", cat.lastScale)
|
||||
}
|
||||
if !strings.Contains(body, "coder runs 2, and up to 4 under load; there is room for 2 now") {
|
||||
t.Errorf("a ceiling above the room should be taken and said: %s", body)
|
||||
}
|
||||
|
||||
post("0", "2")
|
||||
if cat.lastScale.Min != 0 || cat.lastScale.Max != 2 || cat.lastScale.IdleTimeout == 0 {
|
||||
t.Errorf("on demand was set as %+v", cat.lastScale)
|
||||
}
|
||||
for _, bad := range [][2]string{{"3", "2"}, {"-1", "1"}, {"x", "1"}, {"1", "100"}} {
|
||||
n := len(cat.scaled)
|
||||
post(bad[0], bad[1])
|
||||
if len(cat.scaled) != n {
|
||||
t.Errorf("min %s max %s was accepted", bad[0], bad[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelSetInConfigurationIsShownButNotChanged(t *testing.T) {
|
||||
cat := coderCatalog()
|
||||
cat.declared = "coder"
|
||||
body := detailPage(t, &roomyFleet{more: 20}, cat)("/admin/mirror?open=coder:v1")
|
||||
if strings.Contains(body, `action="/admin/mirror/replicas"`) || !strings.Contains(body, "change its replicas there") {
|
||||
t.Error("a declared model offered a change that would revert")
|
||||
}
|
||||
if !strings.Contains(body, "Room for at least 16 more") {
|
||||
t.Error("room that reached the limit asked should read as at least")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAHeldRowSaysHowItsModelRunsWithoutOpening(t *testing.T) {
|
||||
body := detailPage(t, &roomyFleet{more: 1}, coderCatalog())("/admin/mirror")
|
||||
if strings.Contains(body, "held-card") {
|
||||
t.Fatal("the card opened by itself")
|
||||
}
|
||||
if !strings.Contains(body, `title="replicas ready / wanted">1/1 · <span class="node" title="n2">n2</span></span>`) {
|
||||
t.Errorf("the row does not say where its replicas run: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunningOnNamesEachNodeOnceAndCountsWhatIsStarting(t *testing.T) {
|
||||
nodes, starting := runningOn([]adminapi.ReplicaView{
|
||||
{State: "ready", Shards: []adminapi.ShardView{{Node: "n2"}}},
|
||||
{State: "starting", Shards: []adminapi.ShardView{{Node: "n1"}}},
|
||||
{State: "ready", Shards: []adminapi.ShardView{{Node: "n2"}}},
|
||||
})
|
||||
if strings.Join(nodes, ",") != "n1,n2" || starting != 1 {
|
||||
t.Errorf("nodes %v, starting %d", nodes, starting)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeNamesLoseOnlyWhatEveryNodeShares(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
ids []string
|
||||
want string
|
||||
}{
|
||||
{[]string{"llmgw-node01", "llmgw-node02", "llmgw-node06"}, "llmgw-"},
|
||||
{[]string{"gpu-a.dc1", "gpu-b.dc1"}, "gpu-"},
|
||||
{[]string{"n1", "n2"}, ""},
|
||||
{[]string{"llmgw-node01"}, ""},
|
||||
{[]string{"alpha", "beta"}, ""},
|
||||
} {
|
||||
if got := sharedPrefix(c.ids); got != c.want {
|
||||
t.Errorf("%v: %q, expected %q", c.ids, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -155,7 +155,7 @@ func oneOf(s string, options ...string) bool {
|
||||
|
||||
// href is the page with these settings and one of them changed.
|
||||
func (e explorer) href(query, key, value string) template.URL {
|
||||
v := url.Values{"view": {e.View}, "task": {e.Task}, "size": {e.Size}, "x": {"1"}}
|
||||
v := url.Values{"tab": {tabExplore}, "view": {e.View}, "task": {e.Task}, "size": {e.Size}, "x": {"1"}}
|
||||
if e.Here {
|
||||
v.Set("here", "1")
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ func TestTheExplorerOpensOnTrendingModelsTheNodesCanRun(t *testing.T) {
|
||||
ggufModel("odd/Kev-GGUF", "ggmlc", 5e8, now.Add(-2*24*time.Hour)),
|
||||
}}
|
||||
body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}),
|
||||
"/admin/mirror").Body.String()
|
||||
"/admin/mirror?tab=explore").Body.String()
|
||||
|
||||
q := reg.query
|
||||
if q.Sort != hf.SortTrending || q.Task != "text-generation" || !q.GGUF || q.Search != "" {
|
||||
@@ -96,7 +96,7 @@ func TestAnAdapterIsNotListedAsAModel(t *testing.T) {
|
||||
ggufModel("a/Tiny-GGUF", "qwen2", 135e6, time.Now()),
|
||||
}}
|
||||
body := mirrorGet(t, mirrorPageLlama(t, MirrorDeps{Mirror: &fakeMirror{}, Registry: reg}),
|
||||
"/admin/mirror").Body.String()
|
||||
"/admin/mirror?tab=explore").Body.String()
|
||||
if strings.Contains(body, "Some-LoRA") || !strings.Contains(body, "a/Tiny-GGUF") {
|
||||
t.Errorf("adapters should be left out and small models kept: %s", body)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// cpuFleet is a CPU-only cluster whose nodes share a name prefix.
|
||||
type cpuFleet struct{ fakeSource }
|
||||
|
||||
func (cpuFleet) Nodes() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{
|
||||
{ID: "llmgw-node01", State: "ready", MgmtAddr: "10.0.0.141", MemoryGB: 8},
|
||||
{ID: "llmgw-node02", State: "ready", MgmtAddr: "10.0.0.142", MemoryGB: 8},
|
||||
}
|
||||
}
|
||||
func (cpuFleet) Summary() adminapi.Summary { return adminapi.Summary{} }
|
||||
func (cpuFleet) Models() []adminapi.ModelView {
|
||||
return []adminapi.ModelView{{Name: "qwen", Min: 1, Max: 1, Ready: 1,
|
||||
Description: "A sentence long enough that it once had a row of its own under the model."}}
|
||||
}
|
||||
func (cpuFleet) Replicas(string) []adminapi.ReplicaView {
|
||||
return []adminapi.ReplicaView{{ID: "qwen-1", Model: "qwen", State: "ready",
|
||||
Shards: []adminapi.ShardView{{Node: "llmgw-node02"}}}}
|
||||
}
|
||||
|
||||
func TestTheClusterPageKeepsEachRecordOnOneLine(t *testing.T) {
|
||||
h := Dashboard(cpuFleet{}, nil, "v1", time.Second, true, NavExtras{}, nil)
|
||||
_, body := get(t, h, "http://127.0.0.1:9090/", nil)
|
||||
|
||||
for _, gone := range []string{"<th>Classes</th>", "<th>Fabrics</th>", `<th class="num">GPUs</th>`, "<th>Fallbacks</th>"} {
|
||||
if strings.Contains(body, gone) {
|
||||
t.Errorf("%s is shown with nothing in it on any row", gone)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(body, `title="llmgw-node01"><code>node01</code>`) {
|
||||
t.Error("node names should drop what every node shares, keeping the whole on hover")
|
||||
}
|
||||
if !strings.Contains(body, `<code title="llmgw-node02">node02</code>`) {
|
||||
t.Error("the replicas table should name nodes the same way")
|
||||
}
|
||||
if strings.Contains(body, `class="url-row"`) || !strings.Contains(body, `<td class="clip" title="A sentence long enough`) {
|
||||
t.Error("a model's description should be a clipped cell, not a row of its own")
|
||||
}
|
||||
|
||||
// A cluster with GPUs still gets its GPU column.
|
||||
_, gpu := get(t, Dashboard(fakeSource{}, nil, "v1", time.Second, true, NavExtras{}, nil), "http://127.0.0.1:9090/", nil)
|
||||
if !strings.Contains(gpu, `<th class="num">GPUs</th>`) || !strings.Contains(gpu, "<th>Fabrics</th>") {
|
||||
t.Error("columns with something in them were hidden")
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,7 @@ li { margin: .25rem 0; }
|
||||
a { color: var(--accent); }
|
||||
.wrap.narrow { max-width: 30rem; }
|
||||
label { display: block; margin: .9rem 0; font-size: .9rem; font-weight: 600; }
|
||||
input[type=text], input[type=email], input[type=password], input[type=search] {
|
||||
input[type=text], input[type=email], input[type=password], input[type=search], input[type=number] {
|
||||
display: block; width: 100%; margin-top: .3rem; font: inherit; font-weight: 400;
|
||||
padding: .5rem .6rem; border: 1px solid var(--line); border-radius: 6px;
|
||||
background: var(--card); color: var(--fg);
|
||||
@@ -444,18 +444,29 @@ h3.limits-for { font-size: .92rem; margin: 1rem 0 .4rem; }
|
||||
table.narrow-table { max-width: 30rem; }
|
||||
.q-act a + a { margin-left: .8rem; }
|
||||
.chip-role { font-size: .68rem; font-weight: 600; color: var(--accent); text-transform: uppercase; letter-spacing: .04em; }
|
||||
/* Runs: one fixed-width selector per model, so they line up down the column
|
||||
whatever the names; a catalog name differing from the mirror's sits above. */
|
||||
table.held .use { width: 9.5rem; margin: .1rem 0; }
|
||||
table.held .use .as { display: block; font-size: .72rem; color: var(--muted);
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
table.held .use .opt { font-size: .75rem; line-height: 1.3; }
|
||||
/* Runs: one line per model — a catalog name that differs from the mirror's,
|
||||
how it runs, and where — with the selector at a fixed width so the
|
||||
replica counts line up down the column. */
|
||||
table.held .use { display: flex; align-items: center; gap: .5rem; margin: .1rem 0; white-space: nowrap; }
|
||||
table.held .use .as { font-size: .72rem; color: var(--muted); max-width: 10rem;
|
||||
overflow: hidden; text-overflow: ellipsis; }
|
||||
table.held .use form.inline { margin: 0; }
|
||||
table.held .use .mode-fixed { width: 7.5rem; font-size: .8rem; color: var(--muted); }
|
||||
table.held td.num { white-space: nowrap; }
|
||||
table.held select.mode { width: 100%; padding: .15rem .3rem; font-size: .8rem; }
|
||||
table.held select.mode { width: 7.5rem; margin: 0; padding: .1rem .3rem; font-size: .8rem; }
|
||||
|
||||
/* Link pills: the same look as .tabs, for choices that are addresses rather
|
||||
than panels on one page — a filtered listing should survive a reload and be
|
||||
worth sending to somebody. */
|
||||
/* A page's own sections, one address each. Underlined rather than pills so
|
||||
* they do not read as one more filter beside the ones inside a section. */
|
||||
.page-tabs { display: flex; flex-wrap: wrap; gap: 0 1.5rem; margin: -1rem 0 1.5rem;
|
||||
border-bottom: 1px solid var(--line); }
|
||||
.page-tabs a { padding: .55rem 0; margin-bottom: -1px; font-weight: 600; color: var(--muted);
|
||||
text-decoration: none; border-bottom: 2px solid transparent; }
|
||||
.page-tabs a:hover { color: var(--fg); }
|
||||
.page-tabs a.on { color: var(--fg); border-bottom-color: var(--accent); }
|
||||
.page-tabs .count { font-weight: 500; font-size: .8rem; color: var(--muted); margin-left: .25rem; }
|
||||
.pills { display: flex; flex-wrap: wrap; gap: .35rem; margin: .5rem 0; }
|
||||
.pills a { font-size: .85rem; font-weight: 600; color: var(--muted); padding: .35rem .85rem;
|
||||
border: 1px solid transparent; border-radius: 999px; text-decoration: none; }
|
||||
@@ -804,3 +815,42 @@ details.chat-settings output { font-variant-numeric: tabular-nums; font-weight:
|
||||
.copyable button { margin-right: .75rem; font-size: .8rem; }
|
||||
.connect-form .field-row label { flex: 1 1 12rem; }
|
||||
table.connect-fields th { white-space: nowrap; }
|
||||
|
||||
/* The card opened beneath a held version on the mirror page. */
|
||||
tr.held-detail > td { background: var(--bg); padding: .75rem 1rem 1rem; }
|
||||
.held-card .held-model { margin-top: .75rem; padding-top: .75rem; border-top: 1px solid var(--line); }
|
||||
.held-card .held-model:first-of-type { border-top: 0; padding-top: 0; }
|
||||
.held-card h3 { font-size: 1rem; margin: 0 0 .5rem; display: flex; gap: .6rem; align-items: baseline; flex-wrap: wrap; }
|
||||
.held-card table.replicas { margin: .25rem 0 .5rem; }
|
||||
.held-card .headroom { margin: .5rem 0 .25rem; font-weight: 600; }
|
||||
.held-card details.blocked { font-size: .85rem; color: var(--muted); margin-bottom: .5rem; }
|
||||
.held-card details.blocked ul { margin: .35rem 0 0 1.1rem; padding: 0; }
|
||||
.held-card .replicas-form .field-row { align-items: flex-end; }
|
||||
.held-card .replicas-form .field-row label { flex: 0 0 auto; }
|
||||
.held-card .replicas-form input[type=number] { width: 8rem; margin: .25rem 0 0; }
|
||||
.held-card .replicas-form button { margin: 0; }
|
||||
a.open-held { text-decoration: none; color: inherit; }
|
||||
a.open-held:hover strong { text-decoration: underline; }
|
||||
a.open-held[aria-expanded=true]::before { content: "▾ "; color: var(--muted); }
|
||||
a.open-held[aria-expanded=false]::before { content: "▸ "; color: var(--muted); }
|
||||
.use .runs-now { font-size: .78rem; color: var(--muted); }
|
||||
.use .runs-now.short { color: var(--warn); }
|
||||
.use .runs-now.bad { color: var(--bad); }
|
||||
.use .runs-now .node { white-space: nowrap; }
|
||||
|
||||
/* One line per record, in every table. A header never wraps; a cell of free
|
||||
* text is cut to its column with the whole of it on hover, instead of a row of
|
||||
* its own or a line break that makes one record look like two. */
|
||||
th { white-space: nowrap; }
|
||||
table.rows td { white-space: nowrap; }
|
||||
td.clip { max-width: 22rem; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
table.rows td.clip { color: var(--muted); font-size: .85rem; }
|
||||
/* A row across the whole table is a note or an open form, not a record: it
|
||||
* wraps as prose does. */
|
||||
table.rows td[colspan] { white-space: normal; }
|
||||
table.rows td.where { max-width: none; }
|
||||
table.rows .more { cursor: help; margin-left: .2rem; }
|
||||
table.explore td:first-child { max-width: 34rem; overflow: hidden; text-overflow: ellipsis; }
|
||||
table.explore .from { margin-left: .5rem; font-size: .78rem; color: var(--muted); }
|
||||
.clip-inline { display: inline-block; max-width: 24rem; overflow: hidden; text-overflow: ellipsis;
|
||||
vertical-align: bottom; font-size: .8rem; color: var(--muted); margin-left: .4rem; }
|
||||
@@ -84,7 +84,7 @@ Tell whoever you asked for access that you have signed up.</p>
|
||||
{{end}}
|
||||
{{if .Fields}}
|
||||
<h3>Settings to fill in</h3>
|
||||
<table class="compact narrow-table connect-fields">{{range .Fields}}
|
||||
<table class="rows compact narrow-table connect-fields">{{range .Fields}}
|
||||
<tr><th>{{.Label}}</th><td><code>{{.Value}}</code></td><td><button type="button" class="link copy" data-value="{{.Value}}">copy</button></td></tr>{{end}}
|
||||
</table>
|
||||
{{end}}
|
||||
@@ -149,7 +149,7 @@ document.addEventListener("click", (e) => {
|
||||
|
||||
<h2>Keys</h2>
|
||||
{{if .Keys}}
|
||||
<table><tr><th>Name</th><th>Key</th><th>Spends against</th><th>Created</th><th>Last used</th><th>State</th><th></th></tr>
|
||||
<table class="rows"><tr><th>Name</th><th>Key</th><th>Spends against</th><th>Created</th><th>Last used</th><th>State</th><th></th></tr>
|
||||
{{range .Keys}}<tr>
|
||||
<td>{{.Name}}</td>
|
||||
<td><code>{{.Prefix}}</code></td>
|
||||
@@ -192,7 +192,7 @@ old one to stop moving, then revoke it.</p>
|
||||
</div>
|
||||
{{end}}
|
||||
{{if .Rows}}
|
||||
<table class="compact narrow-table"><tr><th>Limit</th><th class="num">Value</th></tr>
|
||||
<table class="rows compact narrow-table"><tr><th>Limit</th><th class="num">Value</th></tr>
|
||||
{{range .Rows}}<tr><td>{{.Name}}</td><td class="num">{{.Value}}</td></tr>{{end}}</table>
|
||||
{{else}}<p class="empty">No limits here.</p>{{end}}
|
||||
{{else}}<p class="empty">Nowhere to spend yet.</p>{{end}}
|
||||
|
||||
@@ -8,16 +8,20 @@
|
||||
</head><body><div class="wrap">
|
||||
<header>
|
||||
<h1>Model mirror</h1>
|
||||
<p class="sub">
|
||||
{{if .Unreachable}}{{.Addr}} did not answer{{else}}{{.Present}} held ·
|
||||
{{.Fetching}} downloading · {{.Failed}} failed · {{.Addr}}{{end}}
|
||||
</p>
|
||||
<p class="sub">{{.Addr}}{{if .Unreachable}} did not answer{{end}}</p>
|
||||
{{template "nav" .Nav}}
|
||||
</header>
|
||||
|
||||
{{/* Links rather than a script's panels: each tab has an address, the back
|
||||
button works, and only the one shown asks anything of the registry. */}}
|
||||
<nav class="page-tabs" aria-label="Mirror">
|
||||
{{range .Tabs}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="page"{{end}}>{{.Label}}{{with .Count}} <span class="count">{{.}}</span>{{end}}</a>
|
||||
{{end}}</nav>
|
||||
|
||||
{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
|
||||
{{if .Message}}<p class="ok">{{.Message}}</p>{{end}}
|
||||
|
||||
{{if eq .Tab "held"}}
|
||||
{{if .Unreachable}}
|
||||
{{/* Said outright rather than shown as an empty table. A mirror holding
|
||||
nothing and a mirror that is not there look the same in a list of zero
|
||||
@@ -29,9 +33,8 @@ reachable from this machine.</p>
|
||||
{{else}}
|
||||
|
||||
<div id="held">
|
||||
<h2>What the mirror holds</h2>
|
||||
{{if .Rows}}
|
||||
<table class="compact held">
|
||||
<table class="rows compact held">
|
||||
<tr><th>Model</th><th>Files</th><th class="num">Size</th><th>Runs</th><th></th></tr>
|
||||
{{range .Groups}}{{$grouped := gt .Models 1}}
|
||||
{{if $grouped}}
|
||||
@@ -41,8 +44,10 @@ reachable from this machine.</p>
|
||||
{{if .Failed}}<span class="tag bad">failed</span>{{end}}
|
||||
{{if .InUse}}<span class="tag">in use</span>{{end}}</td></tr>
|
||||
{{end}}
|
||||
{{range .Rows}}<tr{{if $grouped}} class="model-version"{{end}}>
|
||||
<td><strong>{{.Model}}</strong> <span class="opt">{{.Version}}</span></td>
|
||||
{{range .Rows}}{{$row0 := .}}<tr{{if $grouped}} class="model-version"{{end}}{{if .Detail}} class="open"{{end}}>
|
||||
<td>{{if eq (printf "%s" .State) "present"}}<a class="open-held" aria-expanded="{{if .Detail}}true{{else}}false{{end}}"
|
||||
href="/admin/mirror?tab=held{{if not .Detail}}&open={{.Open}}{{end}}"><strong>{{.Model}}</strong></a>
|
||||
{{else}}<strong>{{.Model}}</strong>{{end}} <span class="opt">{{.Version}}</span></td>
|
||||
<td class="file" title="{{.File}}{{if .SHA256}} · sha256 {{.SHA256}}{{end}}">
|
||||
{{if gt .Files 1}}{{.Files}} files{{else}}<code>{{.File}}</code>{{end}}</td>
|
||||
<td class="num">
|
||||
@@ -62,7 +67,8 @@ reachable from this machine.</p>
|
||||
{{/* Named only when the catalog calls it something other than the
|
||||
mirror does; otherwise it is the model column again. */}}
|
||||
{{if ne .Name $row.Model}}<span class="as" title="{{.Name}}">as {{.Name}}</span>{{end}}
|
||||
{{if .Declared}}<span class="opt">{{if eq .Mode "always"}}always on{{else if eq .Mode "ondemand"}}on demand{{else}}stopped{{end}}<br>set in server.yaml</span>
|
||||
{{if .Declared}}<span class="mode-fixed" title="set in server.yaml, which is read again on every start">
|
||||
{{- if eq .Mode "always"}}always on{{else if eq .Mode "ondemand"}}on demand{{else}}stopped{{end}} · yaml</span>
|
||||
{{else}}
|
||||
{{/* Submitted on change: the choice is the action, and a second click
|
||||
to confirm it would only be a way to forget to. */}}
|
||||
@@ -78,6 +84,12 @@ reachable from this machine.</p>
|
||||
<noscript><button type="submit" class="link">set</button></noscript>
|
||||
</form>
|
||||
{{end}}
|
||||
{{if .Fleet}}<span class="runs-now{{if or .Parked .Unplaceable}} bad{{else if lt .Ready .Wanted}} short{{end}}" title="replicas ready / wanted{{if .Starting}}, + starting{{end}}">
|
||||
{{- if .Parked}}parked
|
||||
{{- else if and (eq .Wanted 0) (eq .Ready 0)}}none running
|
||||
{{- else}}{{.Ready}}/{{.Wanted}}{{if .Starting}} +{{.Starting}}{{end}}{{end}}
|
||||
{{- if .Unplaceable}} · no room{{end}}
|
||||
{{- with .Nodes}} · {{range $i, $n := .}}{{if $i}}, {{end}}<span class="node" title="{{$n.Full}}">{{$n.Short}}</span>{{end}}{{end}}</span>{{end}}
|
||||
</div>{{end}}
|
||||
{{else}}<span class="opt">not served</span>{{end}}
|
||||
</td>
|
||||
@@ -109,6 +121,7 @@ reachable from this machine.</p>
|
||||
</td>
|
||||
</tr>
|
||||
{{if .Err}}<tr><td colspan="5" class="error-row">{{.Err}}</td></tr>{{end}}
|
||||
{{with .Detail}}<tr class="held-detail"><td colspan="5">{{template "mirror_held_detail" dict "D" . "Row" $row0 "CSRF" $.CSRF}}</td></tr>{{end}}
|
||||
|
||||
{{with .Serve}}
|
||||
<tr class="editing"><td colspan="5">
|
||||
@@ -190,11 +203,14 @@ document.addEventListener('click', function (e) {
|
||||
} else { window.prompt('Address', b.dataset.url); }
|
||||
});
|
||||
</script>
|
||||
{{else}}<p class="empty">The mirror is reachable and holds nothing yet.</p>{{end}}
|
||||
{{else}}<p class="empty">The mirror is reachable and holds nothing yet.
|
||||
{{if .HasRegistry}}<a href="/admin/mirror?tab=explore">Explore Hugging Face</a> for a model, or
|
||||
{{end}}<a href="/admin/mirror?tab=fetch">fetch one by URL</a>.</p>{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if .HasRegistry}}
|
||||
<h2>Explore models</h2>
|
||||
{{if and (eq .Tab "explore") .HasRegistry}}
|
||||
{{if not .Repo}}{{with .Explore}}
|
||||
<nav class="pills" aria-label="Which models">{{range .Views}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="page"{{end}}>{{.Label}}</a>{{end}}</nav>
|
||||
<div class="explore-filters">
|
||||
@@ -203,6 +219,7 @@ document.addEventListener('click', function (e) {
|
||||
</div>
|
||||
{{end}}
|
||||
<form method="get" action="/admin/mirror" class="explore-search">
|
||||
<input type="hidden" name="tab" value="explore">
|
||||
{{with .Explore}}<input type="hidden" name="view" value="{{.View}}">
|
||||
<input type="hidden" name="task" value="{{.Task}}">
|
||||
<input type="hidden" name="size" value="{{.Size}}">
|
||||
@@ -222,13 +239,13 @@ document.addEventListener('click', function (e) {
|
||||
|
||||
{{if not .Repo}}{{with .Explore}}
|
||||
{{if .Rows}}
|
||||
<table class="explore">
|
||||
<table class="rows explore">
|
||||
<tr><th>Repository</th><th class="num">Size</th><th>Architecture</th><th class="num">Context</th>
|
||||
<th class="num">Downloads</th><th class="num">Likes</th><th class="num">Updated</th></tr>
|
||||
{{range .Rows}}<tr class="explore-row">
|
||||
<td><a href="{{.Href}}" class="open-repo" data-repo="{{.ID}}" aria-expanded="false"><code>{{.ID}}</code></a>
|
||||
{{if .New}}<span class="tag new">new</span>{{end}}{{if .Gated.Is}}<span class="tag">gated</span>{{end}}
|
||||
{{if or .BaseModel .License}}<div class="from">{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}</div>{{end}}</td>
|
||||
{{if or .BaseModel .License}}<span class="from" title="{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}">{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}</span>{{end}}</td>
|
||||
<td class="num">{{.ParamsLabel}}</td>
|
||||
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="num">{{.ContextLabel}}</td>
|
||||
@@ -307,9 +324,12 @@ document.querySelector("table.explore").addEventListener("submit", async functio
|
||||
return;
|
||||
}
|
||||
note.classList.remove("bad");
|
||||
note.textContent = "✓ " + out.message + " — its progress is in the table above.";
|
||||
note.textContent = "✓ " + out.message + " — ";
|
||||
const link = document.createElement("a");
|
||||
link.href = "/admin/mirror?tab=held";
|
||||
link.textContent = "follow it under Held";
|
||||
note.appendChild(link);
|
||||
btn.textContent = "Started";
|
||||
followHeld();
|
||||
} catch (err) {
|
||||
note.textContent = "Could not reach the gateway: " + err;
|
||||
note.classList.add("bad");
|
||||
@@ -317,28 +337,6 @@ document.querySelector("table.explore").addEventListener("submit", async functio
|
||||
btn.textContent = was;
|
||||
}
|
||||
});
|
||||
|
||||
// Refreshes only the mirror's own table, every five seconds while anything
|
||||
// in it is downloading.
|
||||
let following = false;
|
||||
async function followHeld() {
|
||||
if (following) return;
|
||||
following = true;
|
||||
try {
|
||||
for (;;) {
|
||||
const r = await fetch(location.pathname + location.search, { headers: { Accept: "text/html" } });
|
||||
if (!r.ok) break;
|
||||
const doc = new DOMParser().parseFromString(await r.text(), "text/html");
|
||||
const fresh = doc.getElementById("held"), held = document.getElementById("held");
|
||||
if (!fresh || !held) break;
|
||||
held.replaceWith(fresh);
|
||||
if (!fresh.querySelector(".state-starting")) break;
|
||||
await new Promise((ok) => setTimeout(ok, 5000));
|
||||
}
|
||||
} finally {
|
||||
following = false;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
{{if .Hidden}}<p class="note">{{.Hidden}} more {{if eq .Hidden 1}}is{{else}}are{{end}} hidden: their architecture is one the nodes' llama.cpp cannot load.
|
||||
Untick “only what these nodes can run” to see them.</p>{{end}}
|
||||
@@ -354,10 +352,11 @@ directory, or give the mirror a token.</p>
|
||||
{{if .Repo}}{{template "mirror_repo" .}}{{end}}
|
||||
{{end}}
|
||||
|
||||
<h2>Fetch a model</h2>
|
||||
{{if eq .Tab "fetch"}}
|
||||
<p class="note">By hand, for anything the search does not reach.</p>
|
||||
<form method="post" action="/admin/mirror/fetch">
|
||||
<input type="hidden" name="csrf" value="{{.CSRF}}">
|
||||
<input type="hidden" name="tab" value="fetch">
|
||||
<label>Source URL
|
||||
<input type="text" name="source" required
|
||||
placeholder="https://huggingface.co/…/resolve/main/model.gguf"></label>
|
||||
@@ -373,7 +372,7 @@ directory, or give the mirror a token.</p>
|
||||
</form>
|
||||
<p class="note">The mirror answers as soon as the download is under way, not when
|
||||
it finishes — a model is tens of gigabytes and nothing here holds a connection
|
||||
open for that. The row above will show it arriving. A version is never
|
||||
open for that. The Held tab shows it arriving. A version is never
|
||||
overwritten: fetching into one that exists is refused, so a new build is a new
|
||||
version and the old weights stay where they are.</p>
|
||||
{{end}}
|
||||
@@ -400,7 +399,7 @@ version and the old weights stay where they are.</p>
|
||||
</details>
|
||||
{{end}}
|
||||
{{if .RepoSets}}
|
||||
<table class="repo-files">
|
||||
<table class="rows repo-files">
|
||||
<tr><th>What</th><th class="num">Size</th><th>Published checksum</th><th></th></tr>
|
||||
{{range .RepoSets}}<tr>
|
||||
<td><code>{{.Name}}</code>
|
||||
@@ -410,11 +409,12 @@ version and the old weights stay where they are.</p>
|
||||
<td>
|
||||
{{if not .Verdict.Servable}}
|
||||
<span class="state state-failed" title="{{.Verdict.Reason}}">cannot be served</span>
|
||||
<div class="note">{{.Verdict.Reason}}</div>
|
||||
<span class="clip-inline" title="{{.Verdict.Reason}}">{{.Verdict.Reason}}</span>
|
||||
{{else}}
|
||||
{{if .Verdict.Architecture}}<span class="opt" title="{{.Verdict.Reason}}">{{.Verdict.Architecture}}{{if .Verdict.Checked}} ✓{{end}}</span>{{end}}
|
||||
<form method="post" action="/admin/mirror/fetch" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
||||
<input type="hidden" name="tab" value="explore">
|
||||
{{range .Sources}}<input type="hidden" name="source" value="{{.URL}}">
|
||||
<input type="hidden" name="file" value="{{.Path}}">
|
||||
{{end}}
|
||||
@@ -439,3 +439,73 @@ computes once it has it.</p>
|
||||
<p class="empty">No weight files in {{.Repo.ID}}.</p>
|
||||
{{end}}
|
||||
{{end}}
|
||||
|
||||
{{/* The card beneath a held version: what runs it, where, and how many. */}}
|
||||
{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}}
|
||||
<div class="held-card">
|
||||
<p class="repo-meta">
|
||||
<span>{{if gt $row.Files 1}}{{$row.Files}} files{{else}}<code>{{$row.File}}</code>{{end}}</span>
|
||||
<span>{{bytes $row.BytesDone}}</span>
|
||||
{{with $row.SHA256}}<span>sha256 <code title="{{.}}">{{slice . 0 12}}</code></span>{{end}}
|
||||
<span><code>{{$row.URL}}</code></span>
|
||||
</p>
|
||||
{{if not .D.Models}}
|
||||
<p class="note">Not in the catalog: nothing runs this version.
|
||||
{{if $row.Serve}}{{else}}Use <strong>serve</strong> on its row to put it into service.{{end}}</p>
|
||||
{{end}}
|
||||
{{range .D.Models}}
|
||||
<section class="held-model">
|
||||
<h3>{{.Name}}
|
||||
{{with .View}}<span class="opt">{{.Ready}} of {{.Desired}} ready{{if ne .Total .Ready}} · {{.Total}} in all{{end}}</span>
|
||||
{{if .Parked}}<span class="tag bad">parked</span>{{else if .Unplaceable}}<span class="tag bad">does not fit</span>{{end}}{{end}}
|
||||
</h3>
|
||||
{{with .View.Reason}}<p class="note error-row">{{.}}</p>{{end}}
|
||||
|
||||
{{if .Replicas}}
|
||||
<table class="rows compact replicas">
|
||||
<tr><th>Replica</th><th>Node</th><th>State</th><th class="num">Age</th><th class="num">CPU</th><th class="num">Memory</th></tr>
|
||||
{{range .Replicas}}{{$rep := .}}{{range $i, $s := .Nodes}}<tr>
|
||||
<td>{{if eq $i 0}}<code>{{$rep.ID}}</code>{{end}}</td>
|
||||
<td><a href="/admin/nodes/{{$s.Node}}">{{$s.Node}}</a>{{if $s.MemoryGB}} <span class="opt">{{$s.MemoryGB}} GB</span>{{end}}</td>
|
||||
<td>{{if eq $i 0}}<span class="state state-{{if eq $rep.State "ready"}}ready{{else if eq $rep.State "failed"}}failed{{else}}starting{{end}}">{{$rep.State}}</span>
|
||||
{{with $rep.Restart}}<span class="opt">restart {{.}}</span>{{end}}{{end}}</td>
|
||||
<td class="num">{{if eq $i 0}}{{age $rep.Age}}{{end}}</td>
|
||||
<td class="num">{{if $s.CPU}}{{printf "%.1f" $s.CPU}} cores{{else}}—{{end}}</td>
|
||||
<td class="num">{{if $s.MemBytes}}{{bytes $s.MemBytes}}{{else}}—{{end}}</td>
|
||||
</tr>{{end}}{{end}}
|
||||
</table>
|
||||
{{else}}<p class="note">No replica runs now{{if eq .Mode "ondemand"}}: it starts when somebody asks for it{{end}}.</p>{{end}}
|
||||
|
||||
{{with .Headroom}}
|
||||
<p class="headroom">
|
||||
{{if .Limited}}Room for at least {{.More}} more replicas.
|
||||
{{else if .More}}Room for {{.More}} more now{{if .Nodes}}, on {{range $i, $n := .Nodes}}{{if $i}}, {{end}}{{join $n}}{{end}}{{end}}.
|
||||
{{else}}No room for another replica now.{{end}}
|
||||
</p>
|
||||
{{if and (not .Limited) .Blocked}}<details class="blocked"><summary>Why not {{if .More}}more{{else}}another{{end}}</summary>
|
||||
<ul>{{range .Blocked}}<li>{{.}}</li>{{end}}</ul></details>{{end}}
|
||||
{{end}}
|
||||
|
||||
{{if .Declared}}
|
||||
<p class="note">Set in server.yaml, which is read again on every start: change its replicas there.</p>
|
||||
{{else}}
|
||||
<form method="post" action="/admin/mirror/replicas" class="replicas-form">
|
||||
<input type="hidden" name="csrf" value="{{$csrf}}">
|
||||
<input type="hidden" name="model" value="{{.Name}}">
|
||||
<input type="hidden" name="open" value="{{$row.Open}}">
|
||||
<div class="field-row">
|
||||
<label>Always running
|
||||
<input type="number" name="min" value="{{.Min}}" min="0"{{if .MostNow}} max="{{.MostNow}}"{{end}} required></label>
|
||||
<label>Up to, under load
|
||||
<input type="number" name="max" value="{{.Max}}" min="0" max="64" required></label>
|
||||
<button type="submit">Update</button>
|
||||
</div>
|
||||
<p class="note">{{if .MostNow}}At most {{.MostNow}} can always run with what is free now. {{end}}0 always running with a
|
||||
higher limit loads it when asked for; 0 and 0 stops it. A replica is added under load when requests wait, and
|
||||
taken away after ten quiet minutes.</p>
|
||||
</form>
|
||||
{{end}}
|
||||
</section>
|
||||
{{end}}
|
||||
</div>
|
||||
{{end}}
|
||||
@@ -34,7 +34,7 @@ budgets bind.</p>
|
||||
{{/* The tree is the interface. A quota belongs to a branch, so the form to set
|
||||
one opens on that branch rather than asking somebody to name it again in a
|
||||
box at the bottom. */}}
|
||||
<table class="quota-tree">
|
||||
<table class="rows quota-tree">
|
||||
<tr><th>Where</th><th class="num">Requests/min</th><th class="num">Tokens/min</th>
|
||||
<th class="num">Budget</th><th>Spent</th><th></th></tr>
|
||||
{{range .Rows}}
|
||||
@@ -119,7 +119,7 @@ up — hover it to see where. A value set here is in normal type.</p>
|
||||
<p class="note">A person's own limit covers everything they do, across all their
|
||||
keys, projects and the chat page. {{.Personal}}</p>
|
||||
{{if .People}}
|
||||
<table class="quota-tree people-limits">
|
||||
<table class="rows quota-tree people-limits">
|
||||
<tr><th>Person</th><th class="num">Requests/min</th><th class="num">Tokens/min</th>
|
||||
<th class="num">Budget</th><th>Spent</th><th></th></tr>
|
||||
{{range .People}}
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
</div>
|
||||
|
||||
{{if .Rows}}
|
||||
<table>
|
||||
<table class="rows">
|
||||
<tr><th>Group</th><th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
|
||||
{{range .Rows}}
|
||||
<tr>
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
{{if .Pending}}
|
||||
<h2>Waiting for approval</h2>
|
||||
<table><tr><th>Email</th><th>Name</th><th>Signed up</th><th>Approve, and put them in</th><th></th></tr>
|
||||
<table class="rows"><tr><th>Email</th><th>Name</th><th>Signed up</th><th>Approve, and put them in</th><th></th></tr>
|
||||
{{range .Pending}}<tr>
|
||||
<td>{{.Email}}</td><td>{{.Name}}</td><td>{{ago .CreatedAt}}</td>
|
||||
<td>
|
||||
@@ -149,14 +149,14 @@ project in the team — an account can only be approved into a project.</p>
|
||||
</div>
|
||||
|
||||
{{with index $.Members (printf "project:%s" $p.ID)}}
|
||||
<table class="compact"><tr><th>Member</th><th>State</th><th>Last sign-in</th><th></th></tr>
|
||||
<table class="rows compact"><tr><th>Member</th><th>State</th><th>Last sign-in</th><th></th></tr>
|
||||
{{range .}}{{template "member-row" (dict "U" . "D" $ "Scope" (printf "project:%s" $p.ID) "Open" $p.ID)}}{{end}}
|
||||
</table>
|
||||
{{else}}<p class="empty">No members of its own. Members of its team or organization can use it too.</p>{{end}}
|
||||
{{template "add-member" (dict "Kind" "project" "ID" $p.ID "D" $ "Open" $p.ID)}}
|
||||
|
||||
{{with index $.Keys $p.ID}}
|
||||
<table class="compact"><tr><th>Key</th><th>Owner</th><th>State</th><th>Last used</th><th></th></tr>
|
||||
<table class="rows compact"><tr><th>Key</th><th>Owner</th><th>State</th><th>Last used</th><th></th></tr>
|
||||
{{range .}}<tr>
|
||||
<td>{{.Name}} <code>{{.Prefix}}</code></td>
|
||||
<td>{{or .OwnerEmail "issued by an operator"}}</td>
|
||||
@@ -210,7 +210,7 @@ project in the team — an account can only be approved into a project.</p>
|
||||
|
||||
<h2 id="accounts">All accounts</h2>
|
||||
{{if .Accounts}}
|
||||
<table id="account-list"><tr><th>Person</th><th>State</th><th>Member of</th><th>Last sign-in</th><th>Add to</th><th></th></tr>
|
||||
<table class="rows" id="account-list"><tr><th>Person</th><th>State</th><th>Member of</th><th>Last sign-in</th><th>Add to</th><th></th></tr>
|
||||
{{range .Accounts}}{{template "account-row" (dict "U" . "D" $)}}{{end}}
|
||||
</table>
|
||||
<p class="note">Someone who is a member of nothing uses the gateway on their own:
|
||||
@@ -298,7 +298,7 @@ access than it had, so they need a shell on the control plane —
|
||||
<td>{{$u.Email}}{{if $u.IsAdmin}} <span class="tag">admin</span>{{end}}</td>
|
||||
<td class="state state-{{$u.State}}">{{$u.State}}</td>
|
||||
<td class="where">
|
||||
{{range $u.Memberships}}<span class="chip-m">{{.Path}}{{if eq .Role "manager"}} <b class="chip-role">manager</b>{{end}}
|
||||
{{range $i, $m := $u.Memberships}}{{if lt $i 2}}{{with $m}}<span class="chip-m">{{.Path}}{{if eq .Role "manager"}} <b class="chip-role">manager</b>{{end}}
|
||||
{{if eq (printf "%s" .Scope) "org"}}
|
||||
<form method="post" action="/admin/users/{{$u.ID}}/role" class="inline">
|
||||
<input type="hidden" name="csrf" value="{{$d.CSRF}}">
|
||||
@@ -313,8 +313,12 @@ access than it had, so they need a shell on the control plane —
|
||||
<input type="hidden" name="csrf" value="{{$d.CSRF}}">
|
||||
<input type="hidden" name="scope" value="{{.Scope}}:{{.ScopeID}}">
|
||||
<button type="submit" class="chip-x" aria-label="Take out of {{.Path}}" title="Take out of {{.Path}}">×</button>
|
||||
</form></span>
|
||||
</form></span>{{end}}{{end}}
|
||||
{{else}}<span class="opt">on their own</span>{{end}}
|
||||
{{/* Two, then a count: a person in six teams was six chips wrapping the
|
||||
row into a paragraph. The rest are named on hover and in their own
|
||||
scopes' member lists, where taking them out is done. */}}
|
||||
{{if gt (len $u.Memberships) 2}}<span class="opt more" title="{{range $i, $m := $u.Memberships}}{{if ge $i 2}}{{$m.Path}} {{end}}{{end}}">+{{sub (len $u.Memberships) 2}}</span>{{end}}
|
||||
</td>
|
||||
<td>{{ago $u.LastLoginAt}}</td>
|
||||
<td>
|
||||
@@ -365,7 +369,7 @@ access than it had, so they need a shell on the control plane —
|
||||
<details class="edit"><summary>{{len (index $d.Members $k)}} member{{if ne (len (index $d.Members $k)) 1}}s{{end}}</summary>
|
||||
<div class="scope-members">
|
||||
{{with index $d.Members $k}}
|
||||
<table class="compact">
|
||||
<table class="rows compact">
|
||||
{{range .}}{{template "member-row" (dict "U" . "D" $d "Scope" $k "Open" "")}}{{end}}
|
||||
</table>
|
||||
{{else}}<p class="note">Nobody is a member of {{.Name}} itself; a member here could use every project in it.</p>{{end}}
|
||||
|
||||
@@ -51,23 +51,23 @@
|
||||
|
||||
<h2>Nodes</h2>
|
||||
{{if .Nodes}}
|
||||
<table class="nodes"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th><th class="num">GPUs</th><th class="num">Disk</th><th>Engines</th><th>Classes</th><th>Fabrics</th><th class="num">Models</th><th class="num">Replicas</th><th>Last seen</th></tr>
|
||||
<table class="nodes rows"><tr><th>Node</th><th>State</th><th>CPU</th><th>RAM</th>{{if .AnyGPUs}}<th class="num">GPUs</th>{{end}}<th class="num">Disk</th><th>Engines</th>{{if .AnyClasses}}<th>Classes</th>{{end}}{{if .AnyFabrics}}<th>Fabrics</th>{{end}}<th class="num">Models</th><th class="num">Replicas</th><th>Seen</th></tr>
|
||||
{{range .Nodes}}<tr>
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}"><code>{{.ID}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<td><a href="{{if $.OnAdminListener}}/nodes/{{.ID}}{{else}}/admin/nodes/{{.ID}}{{end}}" title="{{.ID}}"><code>{{shortNode .ID $.NodePrefix}}</code></a>{{if .MgmtAddr}} <span class="note">{{.MgmtAddr}}</span>{{end}}
|
||||
<a class="note" href="{{if $.OnAdminListener}}/nodes/{{.ID}}/logs{{else}}/admin/nodes/{{.ID}}/logs{{end}}">logs</a></td>
|
||||
<td class="state state-{{.State}}">{{.State}}</td>
|
||||
{{/* A dash, not an empty bar, when the agent has not measured it lately:
|
||||
unknown is not idle. */}}
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .CPUPercent}}<span class="load-v">{{printf "%.0f" .CPUPercent}}%{{if .CPUs}} <span class="note">of {{.CPUs}}</span>{{end}}</span>{{else}}—{{end}}</td>
|
||||
<td class="load">{{with .Load}}{{template "minibar" pctbar .MemPercent}}<span class="load-v">{{bytes .MemUsed}} <span class="note">of {{bytes .MemTotal}}</span></span>{{else}}{{if .MemoryGB}}{{.MemoryGB}}G{{else}}—{{end}}{{end}}</td>
|
||||
<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>
|
||||
{{if $.AnyGPUs}}<td class="num">{{if .GPUTotal}}{{.GPUFree}}/{{.GPUTotal}}{{else}}—{{end}}</td>{{end}}
|
||||
{{/* Free out of total, because the first is meaningless without the
|
||||
second, and an em dash where nothing was reported: an agent that
|
||||
was never asked about its disk must not read as a full one. */}}
|
||||
<td class="num">{{if .DiskTotalGB}}{{.DiskFreeGB}}G/{{.DiskTotalGB}}G{{else}}—{{end}}</td>
|
||||
<td>{{if .Engines}}{{join .Engines}}{{else}}<span class="note">none yet</span>{{end}}</td>
|
||||
<td>{{join .Classes}}</td>
|
||||
<td>{{join .Fabrics}}</td>
|
||||
{{if $.AnyClasses}}<td>{{join .Classes}}</td>{{end}}
|
||||
{{if $.AnyFabrics}}<td>{{join .Fabrics}}</td>{{end}}
|
||||
<td class="num">{{if .Models}}{{len .Models}}{{else}}—{{end}}</td>
|
||||
<td class="num">{{.Replicas}}</td>
|
||||
<td>{{or .LastSeenAgo "—"}}</td>
|
||||
@@ -80,18 +80,18 @@
|
||||
<h2>Models</h2>
|
||||
|
||||
{{if .Models}}
|
||||
<table><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th><th>Fallbacks</th></tr>
|
||||
<table class="rows"><tr><th>Model</th><th>Aliases</th><th class="num">Ready</th><th class="num">Wanted</th><th>Profiles</th>{{if .AnyFallbacks}}<th>Fallbacks</th>{{end}}<th>What it is for</th></tr>
|
||||
{{range .Models}}<tr>
|
||||
<td><code>{{.Name}}</code></td>
|
||||
<td>{{join .Aliases}}</td>
|
||||
<td class="num">{{.Ready}}</td>
|
||||
<td class="num">{{.Min}}–{{.Max}}</td>
|
||||
<td>{{join .Profiles}}</td>
|
||||
<td>{{join .Fallbacks}}</td>
|
||||
{{if $.AnyFallbacks}}<td>{{join .Fallbacks}}</td>{{end}}
|
||||
{{/* Cut to the column, the whole sentence on hover: a row of its own for
|
||||
each description made the table twice as long as the catalog. */}}
|
||||
<td class="clip"{{with .Description}} title="{{.}}"{{end}}>{{or .Description "—"}}</td>
|
||||
</tr>
|
||||
{{/* Under the row rather than in a column: a sentence in a table cell either
|
||||
wraps every other column into a stripe or gets cut off. */}}
|
||||
{{if .Description}}<tr><td colspan="6" class="url-row">{{.Description}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||||
|
||||
@@ -106,7 +106,7 @@
|
||||
|
||||
<h2>Replicas</h2>
|
||||
{{if .Replicas}}
|
||||
<table><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||||
<table class="rows"><tr><th>Replica</th><th>Model</th><th>State</th><th>Pool</th><th>Endpoint</th><th class="num">Age</th><th>Nodes</th></tr>
|
||||
{{range .Replicas}}<tr>
|
||||
<td><code>{{.ID}}</code></td>
|
||||
<td>{{.Model}}</td>
|
||||
@@ -114,7 +114,7 @@
|
||||
<td>{{or .Pool "—"}}</td>
|
||||
<td>{{if .Endpoint}}<code>{{.Endpoint}}</code>{{else}}—{{end}}</td>
|
||||
<td class="num">{{duration .AgeSec}}</td>
|
||||
<td>{{range .Shards}}<code>{{.Node}}</code> {{end}}</td>
|
||||
<td>{{range .Shards}}<code title="{{.Node}}">{{shortNode .Node $.NodePrefix}}</code> {{end}}</td>
|
||||
</tr>{{if .Err}}<tr><td colspan="7" class="note">{{.Err}}</td></tr>{{end}}
|
||||
{{end}}</table>
|
||||
{{else}}<p class="empty">No replicas.</p>{{end}}
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
<h2>Fabric</h2>
|
||||
<p class="sub">{{if .Node.Fabrics}}On {{join .Node.Fabrics}} · <a href="{{if $.OnAdminListener}}/fabrics{{else}}/admin/fabrics{{end}}">the map →</a>{{else}}<span class="bad">On no fabric: no port a declared fabric names can carry traffic, so no multi-node replica is placed here.</span>{{end}}</p>
|
||||
{{if .Node.IBPorts}}
|
||||
<table><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
<table class="rows"><tr><th>Adapter</th><th>State</th><th class="num">Rate</th><th>Fabric</th><th>Physically on</th><th>Errors since boot</th><th class="num">Sent</th><th class="num">Received</th></tr>
|
||||
{{range .Node.IBPorts}}<tr>
|
||||
<td><code>{{.HCA}}/{{.Port}}</code>{{if eq .LinkLayer "Ethernet"}} <span class="tag">RoCE</span>{{end}}</td>
|
||||
<td class="{{if .Active}}state-ready{{else}}bad{{end}}">{{.State}}{{with .PhysState}} <span class="note">{{.}}</span>{{end}}</td>
|
||||
@@ -60,13 +60,13 @@ the adapter came up: a number that grows is the one to look at.</p>
|
||||
{{if or .Node.NVLinkDomain .Node.IMEX}} · IMEX {{if eq .Node.IMEX "active"}}<span class="state-ready">active</span>, {{.Node.IMEXChannels}} channel{{if ne .Node.IMEXChannels 1}}s{{end}}{{else}}<span class="bad">{{or .Node.IMEX "not installed"}}</span>{{end}}{{end}}
|
||||
</p>
|
||||
<div class="gpu-wiring">
|
||||
<table class="nvl-matrix" aria-label="NVLink between GPUs">
|
||||
<table class="rows nvl-matrix" aria-label="NVLink between GPUs">
|
||||
<tr><th></th>{{range .Node.GPUs}}<th>{{.Index}}</th>{{end}}</tr>
|
||||
{{range $a := .Node.GPUs}}<tr><th>GPU {{$a.Index}}</th>
|
||||
{{range $b := $.Node.GPUs}}{{if eq $a.Index $b.Index}}<td class="self"></td>{{else if hasInt $a.NVLinkPeers $b.Index}}<td class="nv" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over NVLink">NV</td>{{else}}<td class="pcie" title="GPU {{$a.Index}} ↔ GPU {{$b.Index}} over PCIe">·</td>{{end}}{{end}}
|
||||
</tr>{{end}}
|
||||
</table>
|
||||
<table class="compact gpu-list">
|
||||
<table class="rows compact gpu-list">
|
||||
<tr><th>GPU</th><th>Name</th><th class="num">NUMA</th><th>Nearest adapter</th><th>Multi-node NVLink</th></tr>
|
||||
{{range .Node.GPUs}}<tr>
|
||||
<td>{{.Index}}</td><td>{{.Name}}</td><td class="num">{{.NUMA}}</td>
|
||||
@@ -82,7 +82,7 @@ traffic between nodes should go: PIX and PXB are under the same PCIe switch, SYS
|
||||
|
||||
<h2>What runs here</h2>
|
||||
{{if .Engines}}
|
||||
<table><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
||||
<table class="rows"><tr><th>Model</th><th>Replica</th><th>State</th><th>CPU</th><th class="num">Memory</th></tr>
|
||||
{{range .Engines}}<tr>
|
||||
<td>{{.Model}}</td>
|
||||
<td><code>{{.Replica}}</code></td>
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
{{end}}
|
||||
|
||||
{{if .Rows}}
|
||||
<table class="compact">
|
||||
<table class="rows compact">
|
||||
<tr><th>{{if eq .By "user"}}Person{{else if eq .By "project"}}Project{{else}}Team{{end}}</th>
|
||||
<th class="num">Requests</th><th class="num">Tokens</th><th class="num">Spend</th><th class="num">Errors</th></tr>
|
||||
{{range .Rows}}<tr>
|
||||
@@ -56,7 +56,7 @@ people do elsewhere — in another organization, or on their own — is not show
|
||||
|
||||
<h2>Teams, projects and limits</h2>
|
||||
{{if .Quotas}}
|
||||
<table class="quota-tree">
|
||||
<table class="rows quota-tree">
|
||||
<tr><th>Where</th><th class="num">Requests/min</th><th class="num">Tokens/min</th>
|
||||
<th class="num">Budget</th><th>Spent · 30 days</th></tr>
|
||||
{{range .Quotas}}
|
||||
@@ -79,7 +79,7 @@ people do elsewhere — in another organization, or on their own — is not show
|
||||
|
||||
<h2>Members</h2>
|
||||
{{if .Members}}
|
||||
<table class="compact">
|
||||
<table class="rows compact">
|
||||
<tr><th>Person</th><th>Member of</th><th>State</th></tr>
|
||||
{{range .Members}}<tr>
|
||||
<td>{{.Email}}</td>
|
||||
|
||||
@@ -109,6 +109,9 @@ var funcs = template.FuncMap{
|
||||
return p
|
||||
},
|
||||
"ago": ago,
|
||||
// shortNode is a node's name without what every node's name starts with.
|
||||
"shortNode": strings.TrimPrefix,
|
||||
"age": age,
|
||||
// pctbar is a percentage as the minibar template takes it: whole, and
|
||||
// held to 0..100 so a measurement a little over does not draw past its box.
|
||||
"pctbar": func(v float64) struct{ P int } {
|
||||
|
||||
Reference in new issue
Block a user