Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3bf4b7b709 | ||
|
|
4a652fdcc1 | ||
|
|
f8572fb5ee |
No files matched your search
@@ -77,6 +77,31 @@ more than 26 hours old or there has never been one. docs/install.md now says how
|
||||
to restore, including why running engines survive it: the engine-key secret
|
||||
comes back with the store.
|
||||
|
||||
### Following a request
|
||||
|
||||
Every response carries `X-Request-Id` — the client's own when it sends a plain
|
||||
one, otherwise one the gateway makes — and the engine receives it too. One
|
||||
access log line per request records it with the model asked for and served,
|
||||
the replica, status, time queued, time to the first token, total time,
|
||||
tokens, and the key, project and user. Usage records keep the id, replica,
|
||||
queue time and time to first token, and `llmgw ctl request ID` reads one back:
|
||||
the answer to "why was this slow?".
|
||||
|
||||
### Rate-limit headers
|
||||
|
||||
Responses carry OpenAI's `x-ratelimit-limit-*`, `x-ratelimit-remaining-*` and
|
||||
`x-ratelimit-reset-*` for requests and tokens, from whichever level of the
|
||||
tenancy tree will refuse first, so SDKs and editors can slow down before they
|
||||
are refused.
|
||||
|
||||
### Memory sized from the model
|
||||
|
||||
The mirror's serve form sizes a llama.cpp model's memory from its GGUF header:
|
||||
weights, plus the KV cache for the context pool from its layers, KV heads and
|
||||
head width, plus the engine's own. At a 16k pool that cache ranges from 0.44 GB
|
||||
to 5.25 GB across the models on the live mirror, which the old "weights plus
|
||||
half again plus two" could not see.
|
||||
|
||||
### Security
|
||||
|
||||
- **Alertmanager answered anyone on the network**, including requests to
|
||||
|
||||
@@ -383,3 +383,39 @@ func keyCounter(keyID string) string {
|
||||
// they are asking is whether the budgets they are about to display mean
|
||||
// anything, and the accountant is what would enforce them.
|
||||
func (a *Accountant) AnythingPriced() bool { return a.pricing.AnythingPriced() }
|
||||
|
||||
// Headroom is the tightest of a principal's request and token limits: how
|
||||
// much is left of each, and when the minute it is counted over rolls on.
|
||||
type Headroom struct {
|
||||
RequestLimit, RequestsLeft int
|
||||
TokenLimit, TokensLeft int
|
||||
Reset time.Duration
|
||||
}
|
||||
|
||||
// Headroom reports what a principal has left, after whatever Check just
|
||||
// charged. Of several levels the one with least left is the one that will
|
||||
// refuse first, so that is the one reported. A limit of 0 means none is set.
|
||||
func (a *Accountant) Headroom(p auth.Principal) Headroom {
|
||||
var h Headroom
|
||||
firstReq, firstTok := true, true
|
||||
for _, l := range levels(p.Limits, p.KeyID) {
|
||||
if l.RPM != nil && *l.RPM > 0 {
|
||||
left := max(0, *l.RPM-int(a.requests.Rate(l.Counter)))
|
||||
if firstReq || left < h.RequestsLeft {
|
||||
h.RequestLimit, h.RequestsLeft, firstReq = *l.RPM, left, false
|
||||
}
|
||||
}
|
||||
if l.TPM != nil && *l.TPM > 0 {
|
||||
left := max(0, *l.TPM-int(a.tokens.Rate(l.Counter)))
|
||||
if firstTok || left < h.TokensLeft {
|
||||
h.TokenLimit, h.TokensLeft, firstTok = *l.TPM, left, false
|
||||
}
|
||||
}
|
||||
}
|
||||
// The windows slide, so room comes back continuously; the next minute
|
||||
// boundary is when a whole window's worth has moved on, and is the figure
|
||||
// a client pacing itself can act on.
|
||||
now := time.Now()
|
||||
h.Reset = now.Truncate(windowLen).Add(windowLen).Sub(now)
|
||||
return h
|
||||
}
|
||||
@@ -630,3 +630,31 @@ func TestAnEmptyPriceTableMeansNoBudgetCanEverBind(t *testing.T) {
|
||||
t.Error("no price table at all is not priced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadroomIsTheTightestLimitAfterThisRequest(t *testing.T) {
|
||||
// A project allowing 100 a minute under a team allowing 3: the team is
|
||||
// what will refuse, so the team is what a client is told about.
|
||||
a := New(pricing(t), nil, slog.New(slog.DiscardHandler))
|
||||
p := principal("k1", func(e *store.Effective) {
|
||||
e.Chain = []store.Limits{
|
||||
{Scope: store.ScopeProject, ScopeID: "proj", RPM: intp(100), TPM: intp(50000)},
|
||||
{Scope: store.ScopeTeam, ScopeID: "team", RPM: intp(3)},
|
||||
}
|
||||
})
|
||||
if _, err := a.Check(p, "m", 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := a.Headroom(p)
|
||||
if h.RequestLimit != 3 || h.RequestsLeft != 2 {
|
||||
t.Errorf("requests %d left of %d; expected 2 of the team's 3", h.RequestsLeft, h.RequestLimit)
|
||||
}
|
||||
if h.TokenLimit != 50000 || h.TokensLeft != 50000-10 {
|
||||
t.Errorf("tokens %d left of %d", h.TokensLeft, h.TokenLimit)
|
||||
}
|
||||
if h.Reset <= 0 || h.Reset > time.Minute {
|
||||
t.Errorf("reset %v", h.Reset)
|
||||
}
|
||||
if none := a.Headroom(principal("k2", nil)); none.RequestLimit != 0 || none.TokenLimit != 0 {
|
||||
t.Errorf("a key with no limits advertised some: %+v", none)
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,7 @@ func newCtlCmd() *cobra.Command {
|
||||
cmd.AddCommand(
|
||||
newNodesCmd(client, &asJSON),
|
||||
newLogsCmd(client, &asJSON),
|
||||
newRequestCmd(client, &asJSON),
|
||||
newModelsCmd(client, &asJSON),
|
||||
newReplicasCmd(client, &asJSON),
|
||||
newExplainCmd(client, &asJSON),
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "request ID",
|
||||
Short: "What one request went through: replica, queue, first token, tokens",
|
||||
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
|
||||
"own. Given one, this says which replica answered, how long admission held\n" +
|
||||
"it, how long the first token took and what it cost — the answer to \"why\n" +
|
||||
"was this slow\". The access log has the same fields under \"request\".",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
var v adminapi.RequestView
|
||||
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
model := v.Model
|
||||
if v.ServedModel != "" && v.ServedModel != v.Model {
|
||||
model += " (served by " + v.ServedModel + ")"
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "request %s at %s\n", v.RequestID, v.At.UTC().Format("2006-01-02 15:04:05Z"))
|
||||
fmt.Fprintf(out, "model %s\n", model)
|
||||
fmt.Fprintf(out, "replica %s\n", or(v.Replica, "- (none answered)"))
|
||||
fmt.Fprintf(out, "status %d\n", v.Status)
|
||||
fmt.Fprintf(out, "time %d ms total: %d queued, %d to the first token\n", v.LatencyMs, v.QueueMs, v.TTFTMs)
|
||||
fmt.Fprintf(out, "tokens %d in, %d out\n", v.PromptTokens, v.CompletionTokens)
|
||||
fmt.Fprintf(out, "who key %s, project %s%s\n", v.KeyID, v.ProjectID, prefixed(", user ", v.UserID))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -78,6 +78,7 @@ func (a *API) Register(mux *http.ServeMux) {
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
mux.HandleFunc("POST /api/v1/restart", a.restart)
|
||||
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
|
||||
}
|
||||
|
||||
@@ -1115,3 +1116,52 @@ func (a *API) retryModel(w http.ResponseWriter, r *http.Request) {
|
||||
was := a.ctl.Reconciler().Retry(name)
|
||||
writeJSON(w, http.StatusOK, map[string]bool{"was_parked": was})
|
||||
}
|
||||
|
||||
// requestFinder looks a request up by the id it was given. Asked of the usage
|
||||
// source by type assertion, so one without it answers 501 rather than every
|
||||
// source having to grow the method.
|
||||
type requestFinder interface {
|
||||
UsageByRequest(ctx context.Context, requestID string) (store.Usage, bool, error)
|
||||
}
|
||||
|
||||
// RequestView is what one request went through.
|
||||
type RequestView struct {
|
||||
RequestID string `json:"request_id"`
|
||||
At time.Time `json:"at"`
|
||||
KeyID string `json:"key_id"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
ProjectID string `json:"project_id"`
|
||||
Model string `json:"model"`
|
||||
ServedModel string `json:"served_model"`
|
||||
Replica string `json:"replica,omitempty"`
|
||||
Status int `json:"status"`
|
||||
LatencyMs int64 `json:"latency_ms"`
|
||||
QueueMs int64 `json:"queue_ms"`
|
||||
TTFTMs int64 `json:"ttft_ms"`
|
||||
PromptTokens int64 `json:"prompt_tokens"`
|
||||
CompletionTokens int64 `json:"completion_tokens"`
|
||||
CostMicros int64 `json:"cost_micros"`
|
||||
}
|
||||
|
||||
func (a *API) request(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := a.usage.(requestFinder)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane keeps no usage records")
|
||||
return
|
||||
}
|
||||
u, found, err := f.UsageByRequest(r.Context(), r.PathValue("id"))
|
||||
switch {
|
||||
case err != nil:
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
case !found:
|
||||
writeError(w, http.StatusNotFound, "no request with that id; records are written a few seconds after a request ends")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, RequestView{
|
||||
RequestID: u.RequestID, At: u.At, KeyID: u.KeyID, UserID: u.UserID, ProjectID: u.ProjectID,
|
||||
Model: u.Model, ServedModel: u.ServedModel, Replica: u.Replica, Status: u.Status,
|
||||
LatencyMs: u.LatencyMs, QueueMs: u.QueueMs, TTFTMs: u.TTFTMs,
|
||||
PromptTokens: u.PromptTokens, CompletionTokens: u.CompletionTokens, CostMicros: u.CostMicros,
|
||||
})
|
||||
}
|
||||
+117
-9
@@ -25,6 +25,59 @@ type Header struct {
|
||||
Version uint32
|
||||
Architecture string
|
||||
Name string
|
||||
|
||||
// The shape the KV cache is sized from, from the architecture's own keys
|
||||
// (llama.block_count, qwen2.attention.head_count_kv, ...). Zero when the
|
||||
// file does not say, which a caller has to treat as unknown.
|
||||
BlockCount uint64
|
||||
HeadCount uint64
|
||||
HeadCountKV uint64
|
||||
EmbeddingLength uint64
|
||||
// KeyLength and ValueLength are the per-head sizes when the file gives
|
||||
// them; otherwise a head is EmbeddingLength / HeadCount.
|
||||
KeyLength uint64
|
||||
ValueLength uint64
|
||||
}
|
||||
|
||||
// KVBytesPerToken is what one token of context costs in llama.cpp's default
|
||||
// f16 cache, or 0 when the header did not give the shape.
|
||||
//
|
||||
// Keys and values, per layer, per KV head — the KV heads rather than the
|
||||
// attention heads, because with grouped-query attention a model with forty
|
||||
// heads may keep only eight, and the difference is a factor of five.
|
||||
func (h Header) KVBytesPerToken() int64 {
|
||||
kvHeads := h.HeadCountKV
|
||||
if kvHeads == 0 {
|
||||
kvHeads = h.HeadCount
|
||||
}
|
||||
if h.BlockCount == 0 || kvHeads == 0 {
|
||||
return 0
|
||||
}
|
||||
k, v := h.KeyLength, h.ValueLength
|
||||
if k == 0 && h.HeadCount > 0 {
|
||||
k = h.EmbeddingLength / h.HeadCount
|
||||
}
|
||||
if v == 0 {
|
||||
v = k
|
||||
}
|
||||
if k == 0 {
|
||||
return 0
|
||||
}
|
||||
const f16 = 2
|
||||
return int64(h.BlockCount * kvHeads * (k + v) * f16) //nolint:gosec // header-sized numbers
|
||||
}
|
||||
|
||||
// MemoryBytes is what one llama.cpp server needs for this model: the weights,
|
||||
// a KV cache for a context pool of ctxPool tokens (llama.cpp's --ctx-size,
|
||||
// which --parallel divides among its slots rather than multiplies), and room
|
||||
// for compute buffers and the server itself. 0 when the shape is unknown.
|
||||
func (h Header) MemoryBytes(weights int64, ctxPool int) int64 {
|
||||
perToken := h.KVBytesPerToken()
|
||||
if perToken == 0 || weights <= 0 {
|
||||
return 0
|
||||
}
|
||||
overhead := int64(1<<30) + weights/20
|
||||
return weights + int64(ctxPool)*perToken + overhead
|
||||
}
|
||||
|
||||
// ErrNotGGUF means the bytes are not a GGUF file.
|
||||
@@ -87,16 +140,42 @@ func Read(r io.Reader) (Header, error) {
|
||||
_, err := io.CopyN(io.Discard, br, n)
|
||||
return err
|
||||
}
|
||||
// num holds the last integer value read, for the shape keys.
|
||||
var num uint64
|
||||
readUint := func(size int) error {
|
||||
b := make([]byte, size)
|
||||
if _, err := io.ReadFull(br, b); err != nil {
|
||||
return err
|
||||
}
|
||||
switch size {
|
||||
case 1:
|
||||
num = uint64(b[0])
|
||||
case 2:
|
||||
num = uint64(binary.LittleEndian.Uint16(b))
|
||||
case 4:
|
||||
num = uint64(binary.LittleEndian.Uint32(b))
|
||||
default:
|
||||
num = binary.LittleEndian.Uint64(b)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
var value func(t uint32) (string, error)
|
||||
value = func(t uint32) (string, error) {
|
||||
num = 0
|
||||
switch t {
|
||||
case tUint8, tInt8, tBool:
|
||||
case tUint8, tInt8:
|
||||
return "", readUint(1)
|
||||
case tBool:
|
||||
return "", skip(1)
|
||||
case tUint16, tInt16:
|
||||
return "", skip(2)
|
||||
case tUint32, tInt32, tFloat32:
|
||||
return "", readUint(2)
|
||||
case tUint32, tInt32:
|
||||
return "", readUint(4)
|
||||
case tFloat32:
|
||||
return "", skip(4)
|
||||
case tUint64, tInt64, tFloat64:
|
||||
case tUint64, tInt64:
|
||||
return "", readUint(8)
|
||||
case tFloat64:
|
||||
return "", skip(8)
|
||||
case tString:
|
||||
return str()
|
||||
@@ -129,18 +208,27 @@ func Read(r io.Reader) (Header, error) {
|
||||
if err != nil {
|
||||
return h, err
|
||||
}
|
||||
// Once the architecture is known, running out of the fetched bytes is
|
||||
// not a failure: what the caller came for is in hand, and the shape keys
|
||||
// are a bonus that a file with its tokenizer first may not reach.
|
||||
partial := func(err error) (Header, error) {
|
||||
if h.Architecture != "" {
|
||||
return h, nil
|
||||
}
|
||||
return h, err
|
||||
}
|
||||
for i := uint64(0); i < kvs; i++ {
|
||||
key, err := str()
|
||||
if err != nil {
|
||||
return h, err
|
||||
return partial(err)
|
||||
}
|
||||
var t uint32
|
||||
if err := binary.Read(br, binary.LittleEndian, &t); err != nil {
|
||||
return h, err
|
||||
return partial(err)
|
||||
}
|
||||
v, err := value(t)
|
||||
if err != nil {
|
||||
return h, err
|
||||
return partial(err)
|
||||
}
|
||||
switch key {
|
||||
case "general.architecture":
|
||||
@@ -148,7 +236,25 @@ func Read(r io.Reader) (Header, error) {
|
||||
case "general.name":
|
||||
h.Name = v
|
||||
}
|
||||
if h.Architecture != "" && h.Name != "" {
|
||||
if h.Architecture != "" {
|
||||
switch strings.TrimPrefix(key, h.Architecture+".") {
|
||||
case "block_count":
|
||||
h.BlockCount = num
|
||||
case "attention.head_count":
|
||||
h.HeadCount = num
|
||||
case "attention.head_count_kv":
|
||||
h.HeadCountKV = num
|
||||
case "embedding_length":
|
||||
h.EmbeddingLength = num
|
||||
case "attention.key_length":
|
||||
h.KeyLength = num
|
||||
case "attention.value_length":
|
||||
h.ValueLength = num
|
||||
}
|
||||
}
|
||||
// The tokenizer's arrays — megabytes of vocabulary — come after the
|
||||
// architecture's keys, and nothing past them is wanted.
|
||||
if strings.HasPrefix(key, "tokenizer.") && h.BlockCount > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -209,5 +315,7 @@ func Check(ctx context.Context, url, llamaVersion string) Verdict {
|
||||
if err != nil {
|
||||
return Verdict{Servable: true, Reason: "its header could not be read: " + err.Error()}
|
||||
}
|
||||
return LlamaCpp(llamaVersion, h.Architecture)
|
||||
v := LlamaCpp(llamaVersion, h.Architecture)
|
||||
v.Header = h
|
||||
return v
|
||||
}
|
||||
@@ -92,3 +92,54 @@ func TestLlamaCppKnowsWhatItCanLoad(t *testing.T) {
|
||||
}
|
||||
|
||||
var timeZero = time.Time{}
|
||||
|
||||
// shapedHeader is a header with the keys a KV cache is sized from, written
|
||||
// with the integer types llama.cpp uses for them.
|
||||
func shapedHeader(arch string, layers, heads, kvHeads, embd uint32, truncateTokenizer bool) []byte {
|
||||
var b bytes.Buffer
|
||||
w := func(v any) { _ = binary.Write(&b, binary.LittleEndian, v) }
|
||||
str := func(s string) { w(uint64(len(s))); b.WriteString(s) }
|
||||
u32 := func(k string, v uint32) { str(k); w(uint32(tUint32)); w(v) }
|
||||
b.WriteString("GGUF")
|
||||
w(uint32(3))
|
||||
w(uint64(0))
|
||||
w(uint64(7))
|
||||
str("general.architecture")
|
||||
w(uint32(tString))
|
||||
str(arch)
|
||||
u32(arch+".block_count", layers)
|
||||
u32(arch+".attention.head_count", heads)
|
||||
u32(arch+".attention.head_count_kv", kvHeads)
|
||||
u32(arch+".embedding_length", embd)
|
||||
u32(arch+".context_length", 32768)
|
||||
str("tokenizer.ggml.tokens")
|
||||
w(uint32(tArray))
|
||||
w(uint32(tString))
|
||||
w(uint64(1 << 20))
|
||||
if !truncateTokenizer {
|
||||
for i := 0; i < 1<<20; i++ {
|
||||
str("t")
|
||||
}
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func TestTheKVCacheIsSizedFromTheModelsShape(t *testing.T) {
|
||||
// Qwen2.5-14B: 48 layers, 40 heads of which 8 are kept for K and V,
|
||||
// 5120 wide — 196608 bytes a token in f16, as read from the live mirror.
|
||||
h, err := Read(bytes.NewReader(shapedHeader("qwen2", 48, 40, 8, 5120, true)))
|
||||
if err != nil {
|
||||
t.Fatalf("a header cut off in its tokenizer failed to read: %v", err)
|
||||
}
|
||||
if got := h.KVBytesPerToken(); got != 196608 {
|
||||
t.Errorf("%d bytes a token, expected 196608", got)
|
||||
}
|
||||
weights := int64(9 << 30)
|
||||
want := weights + 16384*196608 + (1<<30 + weights/20)
|
||||
if got := h.MemoryBytes(weights, 16384); got != want {
|
||||
t.Errorf("memory %d, expected %d", got, want)
|
||||
}
|
||||
if (Header{}).MemoryBytes(weights, 16384) != 0 {
|
||||
t.Error("a header with no shape produced a figure")
|
||||
}
|
||||
}
|
||||
@@ -180,6 +180,9 @@ type Verdict struct {
|
||||
// checked.
|
||||
Checked bool
|
||||
Reason string
|
||||
// Header is what was read, zero when nothing was; its shape sizes the
|
||||
// model's memory.
|
||||
Header Header
|
||||
}
|
||||
|
||||
// LlamaCpp judges an architecture against a llama.cpp build.
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package gguf
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestRealHeaders reads real files when LLMGW_GGUF_URLS names them (space
|
||||
// separated), and prints the shape and memory estimate for a 16384-token pool.
|
||||
// Skipped otherwise: CI has no mirror to read.
|
||||
func TestRealHeaders(t *testing.T) {
|
||||
urls := strings.Fields(os.Getenv("LLMGW_GGUF_URLS"))
|
||||
if len(urls) == 0 {
|
||||
t.Skip("LLMGW_GGUF_URLS not set")
|
||||
}
|
||||
for _, u := range urls {
|
||||
h, err := FromURL(t.Context(), nil, u)
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", u, err)
|
||||
continue
|
||||
}
|
||||
t.Logf("%s: arch=%s layers=%d heads=%d kv_heads=%d embd=%d key_len=%d kv/token=%d B, kv@16k=%.2f GiB",
|
||||
u[strings.LastIndex(u, "/")+1:], h.Architecture, h.BlockCount, h.HeadCount, h.HeadCountKV,
|
||||
h.EmbeddingLength, h.KeyLength, h.KVBytesPerToken(), float64(16384*h.KVBytesPerToken())/(1<<30))
|
||||
if h.KVBytesPerToken() == 0 {
|
||||
t.Errorf("%s: no KV shape read", u)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ package router
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -68,6 +70,7 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err
|
||||
r.m.observeAdmissionRejected(req.Band, reason)
|
||||
writeError(w, status, "server_error", err.Error())
|
||||
r.m.observeRequest(r.metricModel(req.Model), "", status, time.Since(start), nil, 0)
|
||||
r.accessLog(req, dispatchResult{err: err}, status, time.Since(start))
|
||||
}
|
||||
|
||||
// prefixBytes is how much of the prompt is kept for prefix-aware routing.
|
||||
@@ -224,6 +227,12 @@ type parsedRequest struct {
|
||||
UpstreamBody []byte
|
||||
Header http.Header
|
||||
|
||||
// RequestID follows the request through the access log, the usage record
|
||||
// and the engine, and goes back to the client in X-Request-Id.
|
||||
RequestID string
|
||||
// QueueMs is how long admission held it.
|
||||
QueueMs int64
|
||||
|
||||
SessionID string
|
||||
Band string
|
||||
// KeyID, ProjectID and Tenant are empty when authentication is off.
|
||||
@@ -275,8 +284,19 @@ type inferenceBody struct {
|
||||
func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, req *http.Request) {
|
||||
start := time.Now()
|
||||
// First, so even a request refused as malformed can be named when
|
||||
// somebody asks what happened to it.
|
||||
rid := requestID(req.Header.Get("X-Request-Id"))
|
||||
w.Header().Set("X-Request-Id", rid)
|
||||
|
||||
parsed, err := r.parse(req, path)
|
||||
if err == nil {
|
||||
parsed.RequestID = rid
|
||||
if parsed.Header == nil {
|
||||
parsed.Header = http.Header{}
|
||||
}
|
||||
parsed.Header.Set("X-Request-Id", rid)
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
r.m.observeRequest("", "", http.StatusBadRequest, time.Since(start), nil, 0)
|
||||
@@ -323,6 +343,9 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
w.Header().Set("x-llmgw-served-model", parsed.Model)
|
||||
}
|
||||
}
|
||||
// On every answer, refused or not, so a client can pace itself
|
||||
// before it is refused rather than learn the limit from a 429.
|
||||
r.rateLimitHeaders(w, p)
|
||||
if err != nil {
|
||||
r.rejectLimit(w, parsed, err, start)
|
||||
return
|
||||
@@ -339,6 +362,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
}
|
||||
defer lease.Release()
|
||||
queued := time.Since(start)
|
||||
parsed.QueueMs = queued.Milliseconds()
|
||||
r.m.observeAdmission(lease.Band(), r.metricModel(parsed.Model), queued)
|
||||
if queued > time.Millisecond {
|
||||
w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10))
|
||||
@@ -422,6 +446,7 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR
|
||||
|
||||
r.m.observeRequest(r.metricModel(req.Model), res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT)
|
||||
r.account(req, res, status, time.Since(start))
|
||||
r.accessLog(req, res, status, time.Since(start))
|
||||
}
|
||||
|
||||
// account settles a finished request against its key.
|
||||
@@ -437,6 +462,8 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
At: time.Now(), KeyID: req.KeyID, UserID: req.UserID, ProjectID: req.ProjectID,
|
||||
Model: req.Model, ServedModel: res.model, Status: status,
|
||||
LatencyMs: took.Milliseconds(),
|
||||
RequestID: req.RequestID, Replica: res.endpointID,
|
||||
QueueMs: req.QueueMs, TTFTMs: res.outcome.TTFT.Milliseconds(),
|
||||
}
|
||||
if u.ServedModel == "" {
|
||||
u.ServedModel = req.Model
|
||||
@@ -468,6 +495,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro
|
||||
// which is absent when waiting does not help.
|
||||
writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error())
|
||||
r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0)
|
||||
r.accessLog(req, dispatchResult{err: err}, http.StatusTooManyRequests, time.Since(start))
|
||||
}
|
||||
|
||||
// parse reads the body and extracts the routing-relevant fields.
|
||||
@@ -854,3 +882,77 @@ func (r *Router) handleMe(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// requestID is the client's X-Request-Id when it is one worth keeping, or a
|
||||
// new one. Kept only when short and plain: it is written into logs and a
|
||||
// header, and a client-chosen string with newlines or megabytes in it is not
|
||||
// an id.
|
||||
func requestID(sent string) string {
|
||||
if n := len(sent); n > 0 && n <= 128 && strings.IndexFunc(sent, func(c rune) bool {
|
||||
return !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' ||
|
||||
c == '-' || c == '_' || c == '.' || c == ':')
|
||||
}) < 0 {
|
||||
return sent
|
||||
}
|
||||
var b [12]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return "req_" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// accessLog writes one line per finished request: enough to follow a slow or
|
||||
// failed request from the id a client reports to the replica that served it,
|
||||
// and where its time went. Before this the only per-request lines were for
|
||||
// disconnects, early stream ends and upstream retries.
|
||||
func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, took time.Duration) {
|
||||
attrs := []any{
|
||||
"request_id", req.RequestID, "status", status, "model", req.Model,
|
||||
"duration_ms", took.Milliseconds(), "queue_ms", req.QueueMs, "stream", req.Stream,
|
||||
}
|
||||
if req.RequestedModel != "" && req.RequestedModel != req.Model {
|
||||
attrs = append(attrs, "requested_model", req.RequestedModel)
|
||||
}
|
||||
if res.model != "" && res.model != req.Model {
|
||||
attrs = append(attrs, "served_model", res.model)
|
||||
}
|
||||
if res.endpointID != "" {
|
||||
attrs = append(attrs, "replica", res.endpointID)
|
||||
}
|
||||
if res.outcome.TTFT > 0 {
|
||||
attrs = append(attrs, "ttft_ms", res.outcome.TTFT.Milliseconds())
|
||||
}
|
||||
if u := res.outcome.Usage; u != nil {
|
||||
attrs = append(attrs, "prompt_tokens", u.PromptTokens, "completion_tokens", u.CompletionTokens)
|
||||
}
|
||||
if req.KeyID != "" {
|
||||
attrs = append(attrs, "key", req.KeyID, "project", req.ProjectID)
|
||||
}
|
||||
if req.UserID != "" {
|
||||
attrs = append(attrs, "user", req.UserID)
|
||||
}
|
||||
if res.err != nil {
|
||||
attrs = append(attrs, "err", res.err.Error())
|
||||
}
|
||||
r.log.Info("request", attrs...)
|
||||
}
|
||||
|
||||
// rateLimitHeaders sets OpenAI's x-ratelimit-* headers from the tightest
|
||||
// limit that applies. SDKs and editors read these to slow down before they
|
||||
// are refused; only Retry-After, and only on a refusal, was sent before. A
|
||||
// limit that is not set is not advertised.
|
||||
func (r *Router) rateLimitHeaders(w http.ResponseWriter, p auth.Principal) {
|
||||
if r.acct == nil {
|
||||
return
|
||||
}
|
||||
h := r.acct.Headroom(p)
|
||||
reset := strconv.FormatFloat(h.Reset.Seconds(), 'f', 0, 64) + "s"
|
||||
if h.RequestLimit > 0 {
|
||||
w.Header().Set("x-ratelimit-limit-requests", strconv.Itoa(h.RequestLimit))
|
||||
w.Header().Set("x-ratelimit-remaining-requests", strconv.Itoa(h.RequestsLeft))
|
||||
w.Header().Set("x-ratelimit-reset-requests", reset)
|
||||
}
|
||||
if h.TokenLimit > 0 {
|
||||
w.Header().Set("x-ratelimit-limit-tokens", strconv.Itoa(h.TokenLimit))
|
||||
w.Header().Set("x-ratelimit-remaining-tokens", strconv.Itoa(h.TokensLeft))
|
||||
w.Header().Set("x-ratelimit-reset-tokens", reset)
|
||||
}
|
||||
}
|
||||
@@ -140,3 +140,51 @@ func waitForUsage(t *testing.T, rt *Router) store.KeyUsage {
|
||||
t.Fatal("no usage was recorded")
|
||||
return store.KeyUsage{}
|
||||
}
|
||||
|
||||
func TestARequestCanBeFollowedByItsID(t *testing.T) {
|
||||
// "Why was my request slow?" had no answer: the queue time went out in a
|
||||
// header and nowhere else, and nothing tied a request to its replica.
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
|
||||
rec := post(t, rt.Handler(),
|
||||
map[string]any{"model": "q14", "messages": []any{
|
||||
map[string]any{"role": "user", "content": "hello"}}},
|
||||
map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": "client-abc.1"})
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("X-Request-Id"); got != "client-abc.1" {
|
||||
t.Errorf("the client's id came back as %q", got)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
u, ok, err := rt.store.UsageByRequest(t.Context(), "client-abc.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok {
|
||||
if u.Replica == "" || u.Status != 200 {
|
||||
t.Errorf("recorded %+v; expected the replica that answered", u)
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("the request's usage cannot be found by its id")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnIDThatIsNotOneIsReplaced(t *testing.T) {
|
||||
for _, bad := range []string{"", "has space", "new\nline", strings.Repeat("x", 200)} {
|
||||
got := requestID(bad)
|
||||
if got == bad || !strings.HasPrefix(got, "req_") {
|
||||
t.Errorf("%q became %q", bad, got)
|
||||
}
|
||||
}
|
||||
if requestID("req_1") != "req_1" {
|
||||
t.Error("a plain id was not kept")
|
||||
}
|
||||
}
|
||||
@@ -526,7 +526,7 @@ func copyHeaders(dst, src http.Header) {
|
||||
// llmgw, not llmgw to the engine. llmgw's own key for the engine is set by Do.
|
||||
func cloneForUpstream(h http.Header) http.Header {
|
||||
out := make(http.Header, 4)
|
||||
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate"} {
|
||||
for _, k := range []string{"Accept", "User-Agent", "Traceparent", "Tracestate", "X-Request-Id"} {
|
||||
if v := h.Get(k); v != "" {
|
||||
out.Set(k, v)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- What a request went through, beside what it cost: which replica answered,
|
||||
-- how long it queued in admission, and how long the first token took. Without
|
||||
-- them "why was my request slow" had no answer — the queue time went out in a
|
||||
-- response header and nowhere else.
|
||||
--
|
||||
-- request_id is the id the client sent or the gateway made, echoed in the
|
||||
-- X-Request-Id header and the access log, so one request can be followed from
|
||||
-- the client's error to the replica that served it.
|
||||
ALTER TABLE usage ADD COLUMN request_id TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN replica TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN queue_ms BIGINT NOT NULL DEFAULT 0;
|
||||
ALTER TABLE usage ADD COLUMN ttft_ms BIGINT NOT NULL DEFAULT 0;
|
||||
CREATE INDEX usage_by_request ON usage(request_id) WHERE request_id <> '';
|
||||
@@ -0,0 +1,13 @@
|
||||
-- What a request went through, beside what it cost: which replica answered,
|
||||
-- how long it queued in admission, and how long the first token took. Without
|
||||
-- them "why was my request slow" had no answer — the queue time went out in a
|
||||
-- response header and nowhere else.
|
||||
--
|
||||
-- request_id is the id the client sent or the gateway made, echoed in the
|
||||
-- X-Request-Id header and the access log, so one request can be followed from
|
||||
-- the client's error to the replica that served it.
|
||||
ALTER TABLE usage ADD COLUMN request_id TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN replica TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE usage ADD COLUMN queue_ms INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE usage ADD COLUMN ttft_ms INTEGER NOT NULL DEFAULT 0;
|
||||
CREATE INDEX usage_by_request ON usage(request_id) WHERE request_id <> '';
|
||||
@@ -424,8 +424,9 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
return s.inTx(ctx, func(tx conn) error {
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -433,7 +434,8 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
for _, r := range records {
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID); err != nil {
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -828,3 +830,25 @@ func (s *DB) SnapshotTo(ctx context.Context, path string) error {
|
||||
_, err := s.raw.ExecContext(ctx, `VACUUM INTO ?`, path)
|
||||
return err
|
||||
}
|
||||
|
||||
// UsageByRequest finds the record of one request by the id it was given, or
|
||||
// ok=false when none has that id.
|
||||
func (s *DB) UsageByRequest(ctx context.Context, requestID string) (Usage, bool, error) {
|
||||
var u Usage
|
||||
var ts int64
|
||||
err := s.db.QueryRowContext(ctx,
|
||||
`SELECT ts, key_id, project_id, COALESCE(user_id, ''), model, served_model, prompt_tokens,
|
||||
completion_tokens, cost_micros, status, latency_ms, request_id, replica, queue_ms, ttft_ms
|
||||
FROM usage WHERE request_id = ? ORDER BY ts DESC LIMIT 1`, requestID).
|
||||
Scan(&ts, &u.KeyID, &u.ProjectID, &u.UserID, &u.Model, &u.ServedModel, &u.PromptTokens,
|
||||
&u.CompletionTokens, &u.CostMicros, &u.Status, &u.LatencyMs, &u.RequestID, &u.Replica,
|
||||
&u.QueueMs, &u.TTFTMs)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Usage{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Usage{}, false, err
|
||||
}
|
||||
u.At = time.UnixMilli(ts)
|
||||
return u, true, nil
|
||||
}
|
||||
@@ -244,6 +244,15 @@ type Usage struct {
|
||||
CostMicros int64
|
||||
Status int
|
||||
LatencyMs int64
|
||||
|
||||
// RequestID is the id the client sent in X-Request-Id or the gateway made.
|
||||
RequestID string
|
||||
// Replica answered the request; empty when none did.
|
||||
Replica string
|
||||
// QueueMs is time spent waiting in admission; TTFTMs is time to the first
|
||||
// token, 0 when none came.
|
||||
QueueMs int64
|
||||
TTFTMs int64
|
||||
}
|
||||
|
||||
// Store is the persistent half of tenancy and accounting.
|
||||
|
||||
@@ -49,6 +49,11 @@ type serveForm struct {
|
||||
// 3.5 GB for a KV pool. Nothing was wrong with placement — it was never
|
||||
// told what the model needed.
|
||||
MinMemoryGB int
|
||||
// MemoryBasis says how MinMemoryGB was arrived at, for the form to show.
|
||||
MemoryBasis string
|
||||
// CtxPool is the engine context pool the form starts with, and the one
|
||||
// the memory figure assumes.
|
||||
CtxPool int
|
||||
|
||||
// Refusal is set when this version cannot be served as it stands, and says
|
||||
// why. Offered and then failing at placement is worse than not offered.
|
||||
@@ -63,6 +68,7 @@ type serveForm struct {
|
||||
func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
fleet Source, cat CatalogAdmin, check func(source string) gguf.Verdict,
|
||||
) serveForm {
|
||||
var header gguf.Header
|
||||
f := serveForm{
|
||||
Model: st.Model,
|
||||
Version: st.Version,
|
||||
@@ -101,10 +107,12 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
case "llamacpp":
|
||||
f.Profile = "cpu"
|
||||
if check != nil {
|
||||
if v := check(f.Source); !v.Servable {
|
||||
v := check(f.Source)
|
||||
if !v.Servable {
|
||||
f.Refusal = "llama.cpp on the nodes cannot run this: " + v.Reason
|
||||
return f
|
||||
}
|
||||
header = v.Header
|
||||
}
|
||||
default:
|
||||
f.Profile = "gpu"
|
||||
@@ -123,7 +131,27 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
"same model runs on the CPU nodes"
|
||||
}
|
||||
}
|
||||
f.MinMemoryGB = memoryFor(st.BytesTotal, st.BytesDone)
|
||||
f.CtxPool = defaultCtxPool
|
||||
f.MinMemoryGB, f.MemoryBasis = memoryFor(st.BytesTotal, st.BytesDone), "weights plus half again plus two"
|
||||
weights := st.BytesTotal
|
||||
if weights <= 0 {
|
||||
weights = st.BytesDone
|
||||
}
|
||||
if b := header.MemoryBytes(weights, defaultCtxPool); b > 0 {
|
||||
kvHeads := header.HeadCountKV
|
||||
if kvHeads == 0 {
|
||||
kvHeads = header.HeadCount
|
||||
}
|
||||
// From the model's own shape: its KV cache for the pool the form
|
||||
// starts with, which ranges from under half a gigabyte to over five
|
||||
// across the models on the live mirror at the same 16k — a spread the
|
||||
// flat "half again" could not see.
|
||||
f.MinMemoryGB = int((b + 1<<30 - 1) >> 30)
|
||||
f.MemoryBasis = fmt.Sprintf("weights %.1f GB + KV cache %.1f GB for %d tokens (%d layers × %d KV heads) + %.1f GB for the engine",
|
||||
float64(weights)/(1<<30), float64(int64(defaultCtxPool)*header.KVBytesPerToken())/(1<<30),
|
||||
defaultCtxPool, header.BlockCount, kvHeads,
|
||||
float64(int64(1<<30)+weights/20)/(1<<30))
|
||||
}
|
||||
if biggest := biggestNode(fleet); biggest > 0 && f.MinMemoryGB > biggest {
|
||||
f.Warning = strings.TrimSuffix(f.Warning, ".") +
|
||||
fmt.Sprintf("%sthe largest node has %d GB and this needs about %d",
|
||||
@@ -353,3 +381,7 @@ func modeOf(sc catalog.Scaling) string {
|
||||
return modeStopped
|
||||
}
|
||||
}
|
||||
|
||||
// defaultCtxPool is the llama.cpp context pool the serve form starts with.
|
||||
// --parallel divides it among slots; it does not multiply it.
|
||||
const defaultCtxPool = 16384
|
||||
@@ -147,11 +147,11 @@ reachable from this machine.</p>
|
||||
<label>Output limit <span class="opt">optional</span>
|
||||
<input type="text" name="max_output" value="1024"></label>
|
||||
<label>Engine context pool <span class="opt">divided across slots</span>
|
||||
<input type="text" name="ctx_size" value="16384"></label>
|
||||
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
|
||||
</div>
|
||||
<div class="field-row">
|
||||
<label>Profile name <input type="text" name="profile" value="{{.Profile}}"></label>
|
||||
<label>Needs RAM (GB) <span class="opt">estimated from the weights</span>
|
||||
<label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
|
||||
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}"></label>
|
||||
<label>Only nodes labelled <span class="opt">key / value, optional</span>
|
||||
<input type="text" name="selector_key" placeholder="role" size="8">
|
||||
@@ -162,8 +162,8 @@ reachable from this machine.</p>
|
||||
<p class="note">The context pool is what the engine is given; llama.cpp divides
|
||||
it across its parallel slots, so the number one request may use is the smaller
|
||||
one above.</p>
|
||||
<p class="note">The memory figure is the weights plus half again for the KV pool
|
||||
plus two for the engine — an estimate, and editable. It is filled in rather than
|
||||
<p class="note">Memory: {{.MemoryBasis}}. Change the context pool and this changes
|
||||
with it, by the KV cache's share — an estimate, and editable. It is filled in rather than
|
||||
left blank because a CPU profile has no card to be short of: with no figure,
|
||||
every ready node is a candidate and the first one picked is whichever the
|
||||
scheduler reached, not whichever can hold the model. A node label does the same
|
||||
|
||||
Reference in new issue
Block a user