CPU replicas start on half the machine's CPUs, not all of them: two busy llama.cpp servers each spinning on all 8 vCPUs fell to 0.8 tok/s apiece, where at 4 threads they keep 5.5. llamacpp.threads sets it per node. Inference requests can be traced over OTLP/HTTP (tracing.endpoint), with admission, schedule and upstream spans and traceparent sent to the engine. Security: onboarding no longer puts the cluster root in the system trust store and removes it where it was; the mirror's mutual-TLS API can be limited to named certificates (--api-client); init.log no longer keeps the administrator's password; a warning when auth is off on a public listen; the recorded actor is sanitised and sudo-aware; Ray's dashboard is off, and the remaining engine gaps are documented. reconciler.go and agent.go are split by topic, the status mapping of each failure is tested, and the plan's checklist is brought up to date. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
120 lines
4.6 KiB
Go
120 lines
4.6 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/accounting"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/admission"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
// What a client is told for each way a dispatch can end. Each status is
|
|
// something a client acts on: 503 with Retry-After is "try again", 503 alone
|
|
// is "not here", 502 is "the engine failed", and a stream already under way
|
|
// cannot be given a status at all.
|
|
func TestEachWayADispatchEndsIsToldToTheClientAsItShouldBe(t *testing.T) {
|
|
rt := newTestRouter(t, baseConfig())
|
|
for _, c := range []struct {
|
|
name string
|
|
res dispatchResult
|
|
status int
|
|
body bool // an error body was written
|
|
retryAfter bool
|
|
}{
|
|
{"served", dispatchResult{outcome: proxy.Outcome{Status: 200, Wrote: true}}, 200, false, false},
|
|
{"engine refused, relayed", dispatchResult{outcome: proxy.Outcome{Status: 400, Wrote: true}}, 400, false, false},
|
|
{"client went away", dispatchResult{err: proxy.ErrClientGone}, 499, false, false},
|
|
{"stream cut short", dispatchResult{outcome: proxy.Outcome{Status: 200, Wrote: true}, err: errors.New("eof")}, 200, false, false},
|
|
{"stream cut before a status", dispatchResult{outcome: proxy.Outcome{Wrote: true}, err: errors.New("eof")}, 200, false, false},
|
|
{"still loading", dispatchResult{err: fmt.Errorf("%w: retry shortly", errLoading)}, 503, true, true},
|
|
{"nothing serves it", dispatchResult{err: fmt.Errorf("%w: no replica", errNoCapacity)}, 503, true, false},
|
|
{"engine failed", dispatchResult{err: errors.New("connection reset")}, 502, true, false},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
w := httptest.NewRecorder()
|
|
got := rt.finish(w, &parsedRequest{Model: "m"}, c.res, time.Now())
|
|
if got != c.status {
|
|
t.Errorf("status %d, expected %d", got, c.status)
|
|
}
|
|
if c.body {
|
|
if w.Code != c.status {
|
|
t.Errorf("written %d, expected %d", w.Code, c.status)
|
|
}
|
|
var e struct {
|
|
Error struct {
|
|
Message string `json:"message"`
|
|
} `json:"error"`
|
|
}
|
|
if err := json.Unmarshal(w.Body.Bytes(), &e); err != nil || e.Error.Message == "" {
|
|
t.Errorf("error body %q", w.Body.String())
|
|
}
|
|
} else if w.Body.Len() != 0 {
|
|
t.Errorf("wrote %q over a response that was not ours to write", w.Body.String())
|
|
}
|
|
if (w.Header().Get("Retry-After") != "") != c.retryAfter {
|
|
t.Errorf("Retry-After %q", w.Header().Get("Retry-After"))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAnUpstreamStatusIsKeptWhenTheStreamBreaks(t *testing.T) {
|
|
rt := newTestRouter(t, baseConfig())
|
|
w := httptest.NewRecorder()
|
|
res := dispatchResult{outcome: proxy.Outcome{Status: http.StatusTooManyRequests, Wrote: true}, err: errors.New("eof")}
|
|
if got := rt.finish(w, &parsedRequest{Model: "m"}, res, time.Now()); got != 429 {
|
|
t.Errorf("status %d", got)
|
|
}
|
|
}
|
|
|
|
func TestARefusalBeforeDispatchSaysWhetherWaitingHelps(t *testing.T) {
|
|
rt := newTestRouter(t, baseConfig())
|
|
for _, c := range []struct {
|
|
name string
|
|
refuse func(http.ResponseWriter)
|
|
status int
|
|
retryAfter string
|
|
}{
|
|
{"queue full", func(w http.ResponseWriter) {
|
|
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrQueueFull, time.Now())
|
|
}, 503, ""},
|
|
{"queued too long", func(w http.ResponseWriter) {
|
|
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrQueueTimeout, time.Now())
|
|
}, 504, ""},
|
|
{"shutting down", func(w http.ResponseWriter) {
|
|
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, admission.ErrClosed, time.Now())
|
|
}, 503, ""},
|
|
{"client left the queue", func(w http.ResponseWriter) {
|
|
rt.rejectAdmission(w, &parsedRequest{Model: "m"}, context.Canceled, time.Now())
|
|
}, 200, ""}, // nothing written: nobody is listening
|
|
{"rate limited", func(w http.ResponseWriter) {
|
|
rt.rejectLimit(w, &parsedRequest{Model: "m"},
|
|
&accounting.LimitError{Err: accounting.ErrRateLimited, RetryAfter: 3 * time.Second}, time.Now())
|
|
}, 429, "3"},
|
|
{"budget spent", func(w http.ResponseWriter) {
|
|
rt.rejectLimit(w, &parsedRequest{Model: "m"}, &accounting.LimitError{Err: accounting.ErrBudgetSpent}, time.Now())
|
|
}, 429, ""},
|
|
{"not a limit at all", func(w http.ResponseWriter) {
|
|
rt.rejectLimit(w, &parsedRequest{Model: "m"}, errors.New("store down"), time.Now())
|
|
}, 500, ""},
|
|
} {
|
|
t.Run(c.name, func(t *testing.T) {
|
|
w := httptest.NewRecorder()
|
|
c.refuse(w)
|
|
if w.Code != c.status {
|
|
t.Errorf("status %d, expected %d", w.Code, c.status)
|
|
}
|
|
if got := w.Header().Get("Retry-After"); got != c.retryAfter {
|
|
t.Errorf("Retry-After %q, expected %q", got, c.retryAfter)
|
|
}
|
|
})
|
|
}
|
|
}
|