diff --git a/internal/router/chat.go b/internal/router/chat.go index 99f0643..097af77 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -40,6 +40,10 @@ func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) { web.Chat(user, r.modelsFor(req, user), contexts, current, version.Version)(w, req) })) mux.HandleFunc("POST /chat/send", pages.Guard(r.chatSend)) + // Where a model stands while it loads, and a way to ask for it without + // holding a request open until it is up. + mux.HandleFunc("GET /chat/model-status", pages.Guard(r.handleChatStatus)) + mux.HandleFunc("POST /chat/wake", pages.Guard(r.handleChatWake)) // Behind the same guard as the rest of the page: it takes a POST body and // returns a file, so it wants a session and the CSRF check for the same // reasons /chat/send does. diff --git a/internal/router/chat_load.go b/internal/router/chat_load.go new file mode 100644 index 0000000..afe0d4a --- /dev/null +++ b/internal/router/chat_load.go @@ -0,0 +1,122 @@ +package router + +import ( + "encoding/json" + "io" + "net/http" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// modelStatus is how far a model is from answering, for the chat page's +// progress bar. A person who asks for a model that is not loaded should see +// it being fetched and started, not a spinner that ends in an error. +type modelStatus struct { + // State is ready, downloading, starting, waiting, failed or stopped. + State string `json:"state"` + // Percent, Done and Total describe a download; Percent is absent when the + // size is unknown, rather than a number that stops moving. + Percent *int `json:"percent,omitempty"` + Done int64 `json:"done,omitempty"` + Total int64 `json:"total,omitempty"` + // Seconds is how long the current stage has run. + Seconds int64 `json:"seconds,omitempty"` + Message string `json:"message,omitempty"` +} + +// loadStatus says where a model stands, from what the control plane reports. +func (r *Router) loadStatus(model string) modelStatus { + if len(r.reg.Snapshots(model)) > 0 { + return modelStatus{State: "ready"} + } + if r.clusterView == nil { + return modelStatus{State: "stopped", Message: "nothing is serving this model"} + } + var best *modelStatus + for _, rep := range r.clusterView.Replicas(model) { + st := modelStatus{Seconds: rep.AgeSec} + switch rep.State { + case "ready": + // Up, and about to be published to the router. + return modelStatus{State: "starting", Message: "almost ready"} + case "failed": + st.State, st.Message = "failed", rep.Err + case "placing", "starting": + st.State = "starting" + for _, sh := range rep.Shards { + if sh.Fetch == nil { + continue + } + st.State = "downloading" + st.Done += sh.Fetch.BytesDone + st.Total += sh.Fetch.BytesTotal + } + if st.State == "downloading" && st.Total > 0 { + p := int(st.Done * 100 / st.Total) + st.Percent = &p + } + default: + continue + } + // A replica making progress says more than one that failed. + if best == nil || best.State == "failed" { + s := st + best = &s + } + } + if best != nil { + return *best + } + if _, m := r.resolve(model); m.Scaling.OnDemand() { + return modelStatus{State: "waiting", Message: "waiting for a node with room for it"} + } + return modelStatus{State: "stopped", Message: "this model is not running"} +} + +// chatModel resolves the model a chat request names, checking the person may +// use it in the context they chose. Empty when they may not, which answers the +// same as a model that does not exist. +func (r *Router) chatModel(req *http.Request, user store.User, asked, context string) string { + name, _ := r.resolve(asked) + if _, known := r.modelMap()[name]; !known && len(r.reg.Snapshots(name)) == 0 { + return "" + } + p, err := r.principalFor(req, user, context) + if err != nil || !r.permits(p, name) { + return "" + } + return name +} + +func (r *Router) handleChatStatus(w http.ResponseWriter, req *http.Request, user store.User) { + name := r.chatModel(req, user, req.URL.Query().Get("model"), req.URL.Query().Get("context")) + if name == "" { + http.NotFound(w, req) + return + } + writeJSON(w, http.StatusOK, r.loadStatus(name)) +} + +// handleChatWake asks for an on-demand model and returns at once: the page +// shows the load happening rather than holding a request open until it ends. +func (r *Router) handleChatWake(w http.ResponseWriter, req *http.Request, user store.User) { + var in struct { + Model string `json:"model"` + Context string `json:"context"` + } + body, _ := io.ReadAll(io.LimitReader(req.Body, 4096)) + if err := json.Unmarshal(body, &in); err != nil || strings.TrimSpace(in.Model) == "" { + http.Error(w, "which model?", http.StatusBadRequest) + return + } + name := r.chatModel(req, user, in.Model, in.Context) + if name == "" { + http.NotFound(w, req) + return + } + if r.activator != nil && len(r.reg.Snapshots(name)) == 0 { + r.activator.Want(name) + } + writeJSON(w, http.StatusOK, r.loadStatus(name)) +} diff --git a/internal/router/chat_load_test.go b/internal/router/chat_load_test.go new file mode 100644 index 0000000..a0399ca --- /dev/null +++ b/internal/router/chat_load_test.go @@ -0,0 +1,119 @@ +package router + +import ( + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" +) + +// replicaSource is the control plane's view with only replicas in it. +type replicaSource struct { + replicas map[string][]adminapi.ReplicaView +} + +func (s replicaSource) Nodes() []adminapi.NodeView { return nil } +func (s replicaSource) Models() []adminapi.ModelView { return nil } +func (s replicaSource) Replicas(model string) []adminapi.ReplicaView { + return s.replicas[model] +} +func (s replicaSource) Explain(string) adminapi.ExplainView { return adminapi.ExplainView{} } +func (s replicaSource) Summary() adminapi.Summary { return adminapi.Summary{} } + +func onDemandRouter(t *testing.T) (*Router, *fakeActivator) { + t.Helper() + e := newFakeEngine(t, "qwen2.5-coder-7b-instruct") + rt, _ := chatRouter(t, e.srv.URL) + cpu := []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}} + c, err := catalog.New([]catalog.Model{ + {Name: "qwen2.5-coder-7b-instruct", Source: "http://m/a.gguf", Profiles: cpu}, + {Name: "nemo", Source: "http://m/b.gguf", Profiles: cpu, Scaling: catalog.Scaling{Min: 0, Max: 1}}, + }) + if err != nil { + t.Fatal(err) + } + rt.SetCatalog(c) + act := &fakeActivator{onDemand: map[string]bool{"nemo": true}} + rt.SetActivator(act) + waitForEndpoint(t, rt) + return rt, act +} + +func TestALoadingModelReportsHowMuchOfItHasArrived(t *testing.T) { + // The chat page draws its bar from this. Two shards of one replica are one + // download as far as the person waiting is concerned. + rt, _ := onDemandRouter(t) + rt.SetClusterView(replicaSource{replicas: map[string][]adminapi.ReplicaView{ + "nemo": {{State: "starting", AgeSec: 12, Shards: []adminapi.ShardView{ + {Fetch: &adminapi.FetchView{BytesDone: 100, BytesTotal: 400}}, + {Fetch: &adminapi.FetchView{BytesDone: 100, BytesTotal: 400}}, + }}}, + }}, nil) + + st := rt.loadStatus("nemo") + if st.State != "downloading" || st.Percent == nil || *st.Percent != 25 { + t.Fatalf("got %+v, expected downloading at 25%%", st) + } + if st.Done != 200 || st.Total != 800 { + t.Errorf("bytes %d of %d, expected 200 of 800", st.Done, st.Total) + } +} + +func TestAModelWithNothingPlacedYetIsWaitingNotStopped(t *testing.T) { + // Between the wake and the placement there is no replica at all. Calling + // that "stopped" would end the bar the moment it appeared. + rt, _ := onDemandRouter(t) + rt.SetClusterView(replicaSource{}, nil) + if st := rt.loadStatus("nemo"); st.State != "waiting" { + t.Errorf("got %+v, expected waiting", st) + } +} + +func TestARunningModelIsReady(t *testing.T) { + rt, _ := onDemandRouter(t) + if st := rt.loadStatus("qwen2.5-coder-7b-instruct"); st.State != "ready" { + t.Errorf("got %+v, expected ready", st) + } +} + +func TestWakingAModelAsksForItAndAnswersAtOnce(t *testing.T) { + rt, act := onDemandRouter(t) + rt.SetClusterView(replicaSource{}, nil) + _, user := chatRouter(t, "http://unused") + user.ProjectID = "" + + req := httptest.NewRequest("POST", "/chat/wake", strings.NewReader(`{"model":"nemo"}`)) + rec := httptest.NewRecorder() + rt.handleChatWake(rec, req, user) + if rec.Code != 200 { + t.Fatalf("status %d: %s", rec.Code, rec.Body.String()) + } + var st modelStatus + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if st.State != "waiting" { + t.Errorf("got %+v, expected waiting", st) + } + if len(act.asked) != 1 || act.asked[0] != "nemo" { + t.Errorf("the activator was asked for %v", act.asked) + } +} + +func TestWakingAnUnknownModelIsNotFound(t *testing.T) { + rt, act := onDemandRouter(t) + _, user := chatRouter(t, "http://unused") + user.ProjectID = "" + req := httptest.NewRequest("POST", "/chat/wake", strings.NewReader(`{"model":"nope"}`)) + rec := httptest.NewRecorder() + rt.handleChatWake(rec, req, user) + if rec.Code != 404 { + t.Errorf("status %d, expected 404", rec.Code) + } + if len(act.asked) != 0 { + t.Errorf("an unknown model was asked for: %v", act.asked) + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index f9a5d4d..ff372b7 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -146,6 +146,7 @@ func (r *Router) Handler() http.Handler { // running, rather than a placeholder that gets sent as-is. pages.ServedModels(r.exampleModels) pages.SetPersonalOff(r.personalOff) + pages.SetLlamaCppVersion(r.llamaVersion) pages.SetPersonalDefaults(r.personalDefaults) pages.Register(mux) r.registerChat(mux, pages) diff --git a/internal/router/router.go b/internal/router/router.go index 408108e..9e3c40e 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -93,6 +93,9 @@ type Router struct { // personalDefaults is what personal use is held to when nobody wrote a // limit for the person, for the quota page to say so. personalDefaults *storepkg.Limits + // llamaVersion is the llama.cpp build the nodes install, so the mirror page + // can refuse weights that build cannot load before fetching them. + llamaVersion string // ownsThreads is true when threads is not store and so must be closed here. ownsThreads bool recorder *accounting.Recorder @@ -247,6 +250,7 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) { st.SetPersonalDefaults(personalLimits(cfg.Auth.Personal)) r.personalOff = cfg.Auth.Personal.Disabled r.personalDefaults = personalLimits(cfg.Auth.Personal) + r.llamaVersion = cfg.WantedEngines()["llamacpp"] r.auth = auth.New(st, auth.Config{ TTL: cfg.Auth.CacheTTL, diff --git a/internal/web/static/base.css b/internal/web/static/base.css index 870406a..59ff2ee 100644 --- a/internal/web/static/base.css +++ b/internal/web/static/base.css @@ -96,6 +96,9 @@ select { font: inherit; padding: .35rem .4rem; border: 1px solid var(--line); .bubble.you { background: var(--accent); color: var(--bg); margin-left: auto; } .bubble.assistant { background: var(--bg); border: 1px solid var(--line); } .bubble.failed { color: var(--bad); border-color: var(--bad); } +.bubble .loading { min-width: 16rem; } +.bubble .loading .stage { font-size: .85rem; color: var(--muted); margin-bottom: .35rem; } +.bubble .loading progress { width: 100%; height: .5rem; accent-color: var(--accent); } .chat-composer { display: flex; gap: .6rem; margin-top: .75rem; align-items: flex-end; } .chat-composer textarea { flex: 1; font: inherit; padding: .5rem .6rem; resize: vertical; border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); } diff --git a/internal/web/templates/chat.html b/internal/web/templates/chat.html index 0fb7881..b448f05 100644 --- a/internal/web/templates/chat.html +++ b/internal/web/templates/chat.html @@ -40,10 +40,9 @@

@@ -168,6 +167,69 @@ access to it.

} } + const ctxValue = () => (document.getElementById("context") || {}).value || ""; + const stages = { + waiting: "Waiting for a node with room", + downloading: "Downloading weights", + starting: "Starting the engine", + ready: "Ready", + }; + function mib(n) { return (n / 1048576).toFixed(0) + " MiB"; } + + // Loads a model that is not running and shows how far along it is, in the + // reply's own bubble. Resolves true once it answers, false if it cannot. + // + // Asked for with /chat/wake rather than by sending the message and waiting: + // that request gives up after 45 seconds, and a large model takes minutes. + async function load(name, out) { + const box = document.createElement("div"); + box.className = "loading"; + const label = document.createElement("div"); + label.className = "stage"; + const bar = document.createElement("progress"); + bar.max = 100; + box.append(label, bar); + out.replaceChildren(box); + const started = Date.now(); + let failures = 0; + + let st; + try { + const resp = await fetch("/chat/wake", { + method: "POST", + headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, + body: JSON.stringify({ model: name, context: ctxValue() }), + }); + if (!resp.ok) throw new Error("(" + resp.status + ") " + (await resp.text()).slice(0, 200)); + st = await resp.json(); + for (;;) { + const secs = Math.round((Date.now() - started) / 1000); + let text = (stages[st.state] || st.state) + " — " + secs + "s"; + if (st.state === "downloading" && st.total) text += " · " + mib(st.done) + " of " + mib(st.total); + label.textContent = text; + // Indeterminate (no value) unless there is a real number: a bar that + // sits at 0% for the engine start says it is stuck when it is not. + if (st.percent != null) bar.value = st.percent; else bar.removeAttribute("value"); + if (st.state === "ready") return true; + // A replica that failed before the wake is still listed until the + // control plane places its replacement, so one sighting is not final. + failures = st.state === "failed" ? failures + 1 : 0; + if (st.state === "stopped" || failures >= 5) { + throw new Error(st.message || "the model could not be started"); + } + await new Promise((r) => setTimeout(r, 1500)); + const r = await fetch("/chat/model-status?model=" + encodeURIComponent(name) + + "&context=" + encodeURIComponent(ctxValue())); + if (!r.ok) throw new Error("(" + r.status + ") " + (await r.text()).slice(0, 200)); + st = await r.json(); + } + } catch (err) { + out.textContent = String(err.message || err); + out.classList.add("failed"); + return false; + } + } + form.onsubmit = async (e) => { e.preventDefault(); const text = input.value.trim(); @@ -181,13 +243,29 @@ access to it.

const out = bubble("assistant"); let reply = ""; let finish = ""; + const opt = model.selectedOptions[0]; try { - const resp = await fetch("/chat/send", { + if (opt && opt.dataset.ondemand === "1") { + if (!(await load(model.value, out))) { messages.pop(); return; } + // Up now: later messages go straight through. + opt.dataset.ondemand = "0"; + opt.textContent = opt.value; + showSimNote(); + out.textContent = ""; + } + const post = () => fetch("/chat/send", { method: "POST", headers: { "Content-Type": "application/json", "X-CSRF-Token": csrf }, - body: JSON.stringify({ model: model.value, messages: messages, - context: (document.getElementById("context") || {}).value || "" }), + body: JSON.stringify({ model: model.value, messages: messages, context: ctxValue() }), }); + let resp = await post(); + // Stopped since the page was drawn, or scaled away by idleness: load it + // again and retry once, rather than telling the person to. + if (resp.status === 503 && resp.headers.get("Retry-After")) { + if (!(await load(model.value, out))) { messages.pop(); return; } + out.textContent = ""; + resp = await post(); + } if (!resp.ok) { out.textContent = "(" + resp.status + ") " + (await resp.text()).slice(0, 400); out.classList.add("failed");