Merge features-c2: the docs portal and its links in English and Turkish
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
e8a62b23b7
201 files changed
+26276
-630
No files matched your search
@@ -0,0 +1,525 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Reusing answers, for a project that asks: the same request to the same
|
||||
// model, from the same project and within its TTL, is answered with what
|
||||
// the model said the first time rather than asking it again.
|
||||
//
|
||||
// Only an answer that would come out the same is reused, unless the project
|
||||
// says otherwise: a chat or a completion asked at temperature 0 or with a
|
||||
// seed, and an embedding, a rerank or a score, which have no sampling at
|
||||
// all. A request sampled above temperature 0 is asked for a new answer, and
|
||||
// handing it an old one would quietly turn variety into repetition.
|
||||
//
|
||||
// The answers are held in this process's memory and nowhere else. A shared
|
||||
// cache in the store would write every answer of every project that turned
|
||||
// this on to the database — the very thing the request log is opt-in for —
|
||||
// and add a round trip to each request to save one; with two control planes
|
||||
// behind a balancer each fills its own, which costs some hits and no
|
||||
// correctness.
|
||||
|
||||
// cacheHeader says what the cache did for a request, hit, miss or bypass,
|
||||
// and is how a client asks it to stay out of the way: "no".
|
||||
const cacheHeader = "x-llmgw-cache"
|
||||
|
||||
// Defaults for router.cache.
|
||||
const (
|
||||
defaultCacheBytes = 128 << 20
|
||||
defaultCacheEntryBytes = 1 << 20
|
||||
)
|
||||
|
||||
// cacheEndpoints are those whose answers can be reused. The chat and
|
||||
// completion endpoints only when the answer is reproducible; the others
|
||||
// always, as they sample nothing. Audio is a form, not JSON, and is not
|
||||
// among them.
|
||||
var cacheEndpoints = map[string]bool{
|
||||
"/v1/chat/completions": true,
|
||||
"/v1/completions": true,
|
||||
"/v1/embeddings": true,
|
||||
"/v1/rerank": true,
|
||||
"/v1/score": true,
|
||||
}
|
||||
|
||||
// sampledEndpoints are those whose answer depends on sampling.
|
||||
var sampledEndpoints = map[string]bool{
|
||||
"/v1/chat/completions": true,
|
||||
"/v1/completions": true,
|
||||
}
|
||||
|
||||
// cacheState is where a request's answer is kept, for one whose answer
|
||||
// may be reused and was not found.
|
||||
type cacheState struct {
|
||||
// key is the answer's place in the cache; empty when it is not kept.
|
||||
key string
|
||||
// project is whose answer it is, and ttl how long it is reused.
|
||||
project string
|
||||
ttl time.Duration
|
||||
}
|
||||
|
||||
// cachedAnswer is one reusable answer.
|
||||
type cachedAnswer struct {
|
||||
key, project string
|
||||
// model is the model that answered, by its catalog name.
|
||||
model string
|
||||
path string
|
||||
// body is the answer as one JSON object — a chat.completion, a
|
||||
// text_completion, a list of embeddings — even when it was streamed.
|
||||
body []byte
|
||||
at time.Time
|
||||
// expires is when the TTL in force when it was kept runs out. The TTL
|
||||
// in force when it is asked for is checked too, so shortening a
|
||||
// project's TTL shortens the life of what it already has.
|
||||
expires time.Time
|
||||
size int64
|
||||
}
|
||||
|
||||
// answerCache is a least-recently-used cache of answers, bounded in bytes.
|
||||
type answerCache struct {
|
||||
maxBytes, maxEntry int64
|
||||
|
||||
mu sync.Mutex
|
||||
lru *list.List // of *cachedAnswer, most recently used at the front
|
||||
by map[string]*list.Element
|
||||
bytes int64
|
||||
// swept is when expired answers were last cleared out.
|
||||
swept time.Time
|
||||
// size reports bytes and answers held, after each change.
|
||||
size func(bytes int64, n int)
|
||||
}
|
||||
|
||||
func newAnswerCache(maxBytes, maxEntry int64, size func(int64, int)) *answerCache {
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = defaultCacheBytes
|
||||
}
|
||||
if maxEntry <= 0 {
|
||||
maxEntry = defaultCacheEntryBytes
|
||||
}
|
||||
maxEntry = min(maxEntry, maxBytes)
|
||||
if size == nil {
|
||||
size = func(int64, int) {}
|
||||
}
|
||||
return &answerCache{maxBytes: maxBytes, maxEntry: maxEntry, lru: list.New(),
|
||||
by: map[string]*list.Element{}, size: size}
|
||||
}
|
||||
|
||||
// get is the answer kept under key, when it is younger than ttl and has
|
||||
// not expired.
|
||||
func (c *answerCache) get(key string, ttl time.Duration, now time.Time) *cachedAnswer {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
el, ok := c.by[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
a := el.Value.(*cachedAnswer)
|
||||
if now.After(a.expires) || now.Sub(a.at) >= ttl {
|
||||
c.remove(el)
|
||||
c.size(c.bytes, len(c.by))
|
||||
return nil
|
||||
}
|
||||
c.lru.MoveToFront(el)
|
||||
return a
|
||||
}
|
||||
|
||||
// put keeps an answer, making room for it by dropping those used longest
|
||||
// ago. One larger than an entry may be is not kept.
|
||||
func (c *answerCache) put(a *cachedAnswer, now time.Time) bool {
|
||||
// What the entry costs besides its body: the key twice, the names, and
|
||||
// the bookkeeping, roughly.
|
||||
a.size = int64(len(a.body) + 2*len(a.key) + len(a.project) + len(a.model) + len(a.path) + 160)
|
||||
if a.size > c.maxEntry {
|
||||
return false
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.by[a.key]; ok {
|
||||
c.remove(el)
|
||||
}
|
||||
// Now and then, those past their time, so a quiet project's answers
|
||||
// do not sit in memory until pushed out by others.
|
||||
if now.Sub(c.swept) > time.Minute {
|
||||
c.swept = now
|
||||
for el := c.lru.Back(); el != nil; {
|
||||
prev := el.Prev()
|
||||
if now.After(el.Value.(*cachedAnswer).expires) {
|
||||
c.remove(el)
|
||||
}
|
||||
el = prev
|
||||
}
|
||||
}
|
||||
for c.bytes+a.size > c.maxBytes && c.lru.Len() > 0 {
|
||||
c.remove(c.lru.Back())
|
||||
}
|
||||
c.by[a.key] = c.lru.PushFront(a)
|
||||
c.bytes += a.size
|
||||
c.size(c.bytes, len(c.by))
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *answerCache) remove(el *list.Element) {
|
||||
a := el.Value.(*cachedAnswer)
|
||||
c.lru.Remove(el)
|
||||
delete(c.by, a.key)
|
||||
c.bytes -= a.size
|
||||
}
|
||||
|
||||
// answerCacheOf is a project's setting, read at most every requestLogTTL
|
||||
// with its other settings. Unreadable, it reuses nothing: asking the model
|
||||
// again is the safe way to be wrong.
|
||||
func (r *Router) answerCacheOf(ctx context.Context, project string) storepkg.AnswerCache {
|
||||
if r.store == nil || project == "" {
|
||||
return storepkg.AnswerCache{}
|
||||
}
|
||||
r.reqLogs.mu.Lock()
|
||||
c, ok := r.reqLogs.by[project]
|
||||
r.reqLogs.mu.Unlock()
|
||||
if ok && time.Since(c.at) < requestLogTTL {
|
||||
return c.cache
|
||||
}
|
||||
c, err := r.readProjectPolicy(ctx, project)
|
||||
if err != nil {
|
||||
r.log.Warn("could not read whether a project reuses answers", "project", project, "err", err)
|
||||
}
|
||||
return c.cache
|
||||
}
|
||||
|
||||
// cacheLookup decides whether a request's answer may come from its
|
||||
// project's cache, says what it decided in x-llmgw-cache, and returns the
|
||||
// answer when there is one. Run after the guardrails have judged the
|
||||
// request, so a cached answer reaches only a request that would have been
|
||||
// sent to the model, and the request it is looked up by is the one the
|
||||
// model would have been sent.
|
||||
func (r *Router) cacheLookup(ctx context.Context, w http.ResponseWriter, req *parsedRequest) *cachedAnswer {
|
||||
if r.answers == nil || req.internal || req.ProjectID == "" {
|
||||
return nil
|
||||
}
|
||||
set := r.answerCacheOf(ctx, req.ProjectID)
|
||||
if set.TTL <= 0 {
|
||||
// Says nothing: there is no cache here to have bypassed.
|
||||
return nil
|
||||
}
|
||||
result := "bypass"
|
||||
defer func() {
|
||||
w.Header().Set(cacheHeader, result)
|
||||
r.m.cacheRequests.WithLabelValues(r.metricModel(req.Model), result).Inc()
|
||||
}()
|
||||
if !reusable(req, set.Any) {
|
||||
return nil
|
||||
}
|
||||
model, _ := r.resolve(req.Model)
|
||||
key, ok := answerKey(req.ProjectID, req.Path, model, req.UpstreamBody)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
now := time.Now()
|
||||
if a := r.answers.get(key, set.TTL, now); a != nil && a.project == req.ProjectID {
|
||||
result = "hit"
|
||||
// Age is HTTP's own way of saying how old a reused answer is.
|
||||
w.Header().Set("Age", strconv.Itoa(int(now.Sub(a.at).Seconds())))
|
||||
return a
|
||||
}
|
||||
result = "miss"
|
||||
req.cache = cacheState{key: key, project: req.ProjectID, ttl: set.TTL}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reusable says whether a request's answer may be reused at all: not one
|
||||
// the client asked to be answered afresh, not one continuing a stored
|
||||
// conversation (each turn is new, and is saved), not one under a policy
|
||||
// that keeps nothing — a cached answer is a kept one — and, for chat and
|
||||
// completions, only one asked at temperature 0 or with a seed unless the
|
||||
// project reuses any answer.
|
||||
func reusable(req *parsedRequest, anyAnswer bool) bool {
|
||||
if req.noStore || req.ThreadID != "" || req.ContentType != "" || !cacheEndpoints[req.Path] {
|
||||
return false
|
||||
}
|
||||
if v := strings.ToLower(strings.TrimSpace(req.Header.Get(cacheHeader))); v == "no" || v == "off" || v == "bypass" {
|
||||
return false
|
||||
}
|
||||
for _, v := range req.Header.Values("Cache-Control") {
|
||||
v = strings.ToLower(v)
|
||||
if strings.Contains(v, "no-cache") || strings.Contains(v, "no-store") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if !sampledEndpoints[req.Path] || anyAnswer {
|
||||
return true
|
||||
}
|
||||
var b struct {
|
||||
Temperature *float64 `json:"temperature"`
|
||||
Seed json.RawMessage `json:"seed"`
|
||||
}
|
||||
if json.Unmarshal(req.UpstreamBody, &b) != nil {
|
||||
return false
|
||||
}
|
||||
seeded := len(b.Seed) > 0 && string(b.Seed) != "null"
|
||||
return seeded || b.Temperature != nil && *b.Temperature == 0
|
||||
}
|
||||
|
||||
// answerFields are taken out of a request before it is looked up: they
|
||||
// change how an answer is delivered or recorded, never what it says.
|
||||
var answerFields = []string{"model", "stream", "stream_options", "user", "metadata", "store", "service_tier"}
|
||||
|
||||
// answerKey is where a request's answer is kept: the project, the endpoint,
|
||||
// the model by its catalog name, and the request with what does not change
|
||||
// the answer taken out and its keys sorted. The project is spelled out
|
||||
// rather than only hashed, so no answer is ever looked up across projects.
|
||||
func answerKey(project, path, model string, body []byte) (string, bool) {
|
||||
d := json.NewDecoder(bytes.NewReader(body))
|
||||
d.UseNumber()
|
||||
var m map[string]any
|
||||
if err := d.Decode(&m); err != nil || m == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, f := range answerFields {
|
||||
delete(m, f)
|
||||
}
|
||||
// encoding/json writes map keys sorted, at every depth.
|
||||
norm, err := json.Marshal(canonicalNumbers(m))
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
h := sha256.New()
|
||||
for _, part := range [][]byte{[]byte(path), []byte(model), norm} {
|
||||
h.Write(part)
|
||||
h.Write([]byte{0})
|
||||
}
|
||||
return project + "\x00" + hex.EncodeToString(h.Sum(nil)), true
|
||||
}
|
||||
|
||||
// canonicalNumbers writes each number one way, so 0, 0.0 and 0e0 are the
|
||||
// same temperature. Whole numbers are left as written: a seed above 2^53
|
||||
// would lose its last digits as a float, and two seeds become one.
|
||||
func canonicalNumbers(v any) any {
|
||||
switch x := v.(type) {
|
||||
case map[string]any:
|
||||
for k, e := range x {
|
||||
x[k] = canonicalNumbers(e)
|
||||
}
|
||||
case []any:
|
||||
for i, e := range x {
|
||||
x[i] = canonicalNumbers(e)
|
||||
}
|
||||
case json.Number:
|
||||
s := string(x)
|
||||
if _, err := strconv.ParseInt(s, 10, 64); err == nil {
|
||||
return x
|
||||
}
|
||||
if f, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
if f == float64(int64(f)) && f > -1<<53 && f < 1<<53 {
|
||||
return json.Number(strconv.FormatInt(int64(f), 10))
|
||||
}
|
||||
return json.Number(strconv.FormatFloat(f, 'g', -1, 64))
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// cacheWriter keeps a copy of what is written to the client, up to a
|
||||
// bound, so an answer can be kept once it is known to be a good one.
|
||||
type cacheWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
buf bytes.Buffer
|
||||
limit int
|
||||
over bool
|
||||
}
|
||||
|
||||
func (c *cacheWriter) WriteHeader(code int) {
|
||||
if c.status == 0 {
|
||||
c.status = code
|
||||
}
|
||||
c.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func (c *cacheWriter) Write(p []byte) (int, error) {
|
||||
if c.status == 0 {
|
||||
c.status = http.StatusOK
|
||||
}
|
||||
if !c.over {
|
||||
if c.buf.Len()+len(p) > c.limit {
|
||||
c.over = true
|
||||
c.buf = bytes.Buffer{}
|
||||
} else {
|
||||
c.buf.Write(p)
|
||||
}
|
||||
}
|
||||
return c.ResponseWriter.Write(p)
|
||||
}
|
||||
|
||||
// Flush passes through, which the proxy needs to stream.
|
||||
func (c *cacheWriter) Flush() {
|
||||
if f, ok := c.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (c *cacheWriter) Unwrap() http.ResponseWriter { return c.ResponseWriter }
|
||||
|
||||
// cacheWriterFor wraps w when a request's answer may be kept.
|
||||
func (r *Router) cacheWriterFor(w http.ResponseWriter, req *parsedRequest) (http.ResponseWriter, *cacheWriter) {
|
||||
if req.cache.key == "" {
|
||||
return w, nil
|
||||
}
|
||||
cw := &cacheWriter{ResponseWriter: w, limit: int(r.answers.maxEntry)}
|
||||
return cw, cw
|
||||
}
|
||||
|
||||
// cacheKeep keeps an answer that may be reused: a whole 200 from the model
|
||||
// asked for — not a fallback, which is another model, nor a canary, which
|
||||
// is weights on trial — that no guard changed or stopped.
|
||||
func (r *Router) cacheKeep(req *parsedRequest, res dispatchResult, status int, cw *cacheWriter) {
|
||||
if cw == nil || cw.over || status != http.StatusOK || cw.status != http.StatusOK || res.err != nil ||
|
||||
res.outcome.Status != http.StatusOK || res.outcome.Stopped || res.fallback || res.onCanary {
|
||||
return
|
||||
}
|
||||
if res.judged && res.guard.Action != guard.Allow && !res.guard.Shadow {
|
||||
return
|
||||
}
|
||||
var body []byte
|
||||
captured := bytes.TrimSpace(cw.buf.Bytes())
|
||||
if strings.Contains(cw.Header().Get("Content-Type"), "text/event-stream") ||
|
||||
bytes.HasPrefix(captured, []byte("data:")) {
|
||||
b, ok := assembleStream(req.Path, captured, res.outcome.Usage)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
body = b
|
||||
} else {
|
||||
if len(captured) == 0 || captured[0] != '{' || !json.Valid(captured) {
|
||||
return
|
||||
}
|
||||
body = bytes.Clone(captured)
|
||||
}
|
||||
model := res.model
|
||||
if model == "" {
|
||||
model, _ = r.resolve(req.Model)
|
||||
}
|
||||
now := time.Now()
|
||||
r.answers.put(&cachedAnswer{key: req.cache.key, project: req.cache.project, model: model,
|
||||
path: req.Path, body: body, at: now, expires: now.Add(req.cache.ttl)}, now)
|
||||
}
|
||||
|
||||
// serveCached answers a request from the cache, as the model would have:
|
||||
// as one JSON object, or as a stream of chunks to a client that asked for
|
||||
// one. The guardrails' output policy judges it as it judges any answer —
|
||||
// the policy in force now, not the one when it was first given — and the
|
||||
// usage is recorded as a request that generated nothing: no tokens, no
|
||||
// cost, marked as a cache hit.
|
||||
func (r *Router) serveCached(ctx context.Context, w http.ResponseWriter, req *parsedRequest, a *cachedAnswer, start time.Time) {
|
||||
body := a.body
|
||||
res := dispatchResult{servedModel: servedModel{model: a.model}, cached: true}
|
||||
if r.guard != nil {
|
||||
if f := r.guard.Output(ctx, r.subject(req, a.model)); f != nil {
|
||||
// A copy: the kept answer is shared by every request reading it,
|
||||
// and a filter is free to rewrite what it is given.
|
||||
body = f.Buffered(bytes.Clone(body))
|
||||
res.guard, res.judged = f.Verdict(), true
|
||||
if res.guard.Stricter(req.guardIn) {
|
||||
setGuardHeaders(w.Header(), res.guard)
|
||||
}
|
||||
}
|
||||
}
|
||||
var werr error
|
||||
if req.Stream && sampledEndpoints[req.Path] {
|
||||
werr = writeCachedStream(w, req, body)
|
||||
} else {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, werr = w.Write(body)
|
||||
}
|
||||
status := http.StatusOK
|
||||
if werr != nil {
|
||||
status = 499
|
||||
}
|
||||
res.outcome = proxy.Outcome{Status: http.StatusOK, Wrote: true, Usage: &proxy.Usage{},
|
||||
Reply: replyText(body)}
|
||||
took := time.Since(start)
|
||||
r.m.observeRequest(r.metricModel(req.Model), "cache", status, took, nil, 0)
|
||||
r.account(req, res, status, took)
|
||||
r.accessLog(req, res, status, took)
|
||||
r.observeOutput(req, res, res.judged)
|
||||
if req.logRetain > 0 {
|
||||
go r.keepRequest(context.WithoutCancel(ctx), req, res, status)
|
||||
}
|
||||
}
|
||||
|
||||
// writeCachedStream sends a kept answer as the chunks a model streams, with
|
||||
// the usage chunk only to a client that asked for one.
|
||||
func writeCachedStream(w http.ResponseWriter, req *parsedRequest, body []byte) error {
|
||||
events, err := streamFromAnswer(req.Path, body, clientWantsUsage(req))
|
||||
if err != nil {
|
||||
// Not an answer that can be streamed — a guard's replacement, say.
|
||||
// Sent whole, which a client reads as a model that does not stream.
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, err = w.Write(body)
|
||||
return err
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.Header().Set("Cache-Control", "no-cache")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
for _, e := range append(events, "[DONE]") {
|
||||
if _, err := io.WriteString(w, "data: "+e+"\n\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if f, ok := w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// clientWantsUsage says whether the client asked for the usage chunk
|
||||
// itself, rather than the gateway asking for it on its behalf.
|
||||
func clientWantsUsage(req *parsedRequest) bool {
|
||||
if req.SuppressUsageChunk {
|
||||
return false
|
||||
}
|
||||
var b struct {
|
||||
StreamOptions *struct {
|
||||
IncludeUsage bool `json:"include_usage"`
|
||||
} `json:"stream_options"`
|
||||
}
|
||||
return json.Unmarshal(req.UpstreamBody, &b) == nil && b.StreamOptions != nil && b.StreamOptions.IncludeUsage
|
||||
}
|
||||
|
||||
// replyText is an answer's first choice as text, for the request log.
|
||||
func replyText(body []byte) string {
|
||||
var c struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content *string `json:"content"`
|
||||
} `json:"message"`
|
||||
Text *string `json:"text"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if json.Unmarshal(body, &c) != nil || len(c.Choices) == 0 {
|
||||
return ""
|
||||
}
|
||||
if s := c.Choices[0].Message.Content; s != nil {
|
||||
return *s
|
||||
}
|
||||
if s := c.Choices[0].Text; s != nil {
|
||||
return *s
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,615 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// cachingRouter is pricedRouter with its project reusing answers for an
|
||||
// hour, reproducible ones only unless anyAnswer.
|
||||
func cachingRouter(t *testing.T, engineURL string, anyAnswer bool) (*Router, string) {
|
||||
t.Helper()
|
||||
rt, secret := pricedRouter(t, engineURL)
|
||||
if err := rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), time.Hour, anyAnswer); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rt, secret
|
||||
}
|
||||
|
||||
func ask(temperature any, text string) map[string]any {
|
||||
b := map[string]any{"model": "q14", "messages": []any{map[string]any{"role": "user", "content": text}}}
|
||||
if temperature != nil {
|
||||
b["temperature"] = temperature
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// usageOf waits for the record of one request.
|
||||
func usageOf(t *testing.T, rt *Router, id string) store.Usage {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if u, ok, err := rt.store.UsageByRequest(t.Context(), id); err == nil && ok {
|
||||
return u
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("no usage was recorded for %s", id)
|
||||
return store.Usage{}
|
||||
}
|
||||
|
||||
func TestAProjectThatReusesAnswersAnswersTheSameRequestWithoutTheModel(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := cachingRouter(t, e.srv.URL, false)
|
||||
h := rt.Handler()
|
||||
auth := func(id string) map[string]string {
|
||||
return map[string]string{"Authorization": "Bearer " + secret, "X-Request-Id": id}
|
||||
}
|
||||
|
||||
first := post(t, h, ask(0, "what is 2+2"), auth("first"))
|
||||
if first.Code != 200 || first.Header().Get(cacheHeader) != "miss" {
|
||||
t.Fatalf("first: %d %q %s", first.Code, first.Header().Get(cacheHeader), first.Body.String())
|
||||
}
|
||||
// The same request, said differently: keys in another order, another
|
||||
// end user, the temperature written as a float, by the canonical name.
|
||||
again := map[string]any{"messages": []any{map[string]any{"content": "what is 2+2", "role": "user"}},
|
||||
"temperature": 0.0, "model": "qwen2.5-14b-instruct", "user": "someone-else"}
|
||||
second := post(t, h, again, auth("second"))
|
||||
if second.Code != 200 || second.Header().Get(cacheHeader) != "hit" {
|
||||
t.Fatalf("second: %d %q %s", second.Code, second.Header().Get(cacheHeader), second.Body.String())
|
||||
}
|
||||
if e.calls.Load() != 1 {
|
||||
t.Errorf("the engine was asked %d times; the second answer should have been reused", e.calls.Load())
|
||||
}
|
||||
if second.Body.String() != first.Body.String() {
|
||||
t.Errorf("reused %s, first was %s", second.Body.String(), first.Body.String())
|
||||
}
|
||||
if second.Header().Get("Age") == "" {
|
||||
t.Error("a reused answer did not say its age")
|
||||
}
|
||||
|
||||
// Recorded, as a request that generated nothing.
|
||||
miss, hit := usageOf(t, rt, "first"), usageOf(t, rt, "second")
|
||||
if miss.CacheHit || miss.CostMicros == 0 || miss.PromptTokens == 0 {
|
||||
t.Errorf("the answer the model gave: %+v", miss)
|
||||
}
|
||||
if !hit.CacheHit || hit.CostMicros != 0 || hit.PromptTokens != 0 || hit.CompletionTokens != 0 ||
|
||||
hit.Status != 200 || hit.ServedModel != "qwen2.5-14b-instruct" {
|
||||
t.Errorf("the reused answer: %+v", hit)
|
||||
}
|
||||
|
||||
// A sampled answer is asked for each time, and says so.
|
||||
for i := range 2 {
|
||||
rec := post(t, h, ask(0.7, "write a poem"), auth(fmt.Sprintf("poem-%d", i)))
|
||||
if rec.Header().Get(cacheHeader) != "bypass" {
|
||||
t.Errorf("a sampled request: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
}
|
||||
if e.calls.Load() != 3 {
|
||||
t.Errorf("the engine was asked %d times, expected each sampled request to reach it", e.calls.Load())
|
||||
}
|
||||
// A seed makes it reproducible.
|
||||
seeded := ask(0.7, "write a poem")
|
||||
seeded["seed"] = 42
|
||||
post(t, h, seeded, auth("seed-1"))
|
||||
if rec := post(t, h, seeded, auth("seed-2")); rec.Header().Get(cacheHeader) != "hit" {
|
||||
t.Errorf("a seeded request: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
// A client that asks for a fresh answer gets one.
|
||||
hdr := auth("fresh")
|
||||
hdr["Cache-Control"] = "no-cache"
|
||||
if rec := post(t, h, ask(0, "what is 2+2"), hdr); rec.Header().Get(cacheHeader) != "bypass" {
|
||||
t.Errorf("Cache-Control: no-cache: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
hdr = auth("fresh-2")
|
||||
hdr[cacheHeader] = "no"
|
||||
if rec := post(t, h, ask(0, "what is 2+2"), hdr); rec.Header().Get(cacheHeader) != "bypass" {
|
||||
t.Errorf("x-llmgw-cache: no: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
|
||||
if n := testutil.ToFloat64(rt.m.cacheRequests.WithLabelValues("qwen2.5-14b-instruct", "hit")); n != 2 {
|
||||
t.Errorf("%v hits counted", n)
|
||||
}
|
||||
if n := testutil.ToFloat64(rt.m.cacheRequests.WithLabelValues("qwen2.5-14b-instruct", "miss")); n != 2 {
|
||||
t.Errorf("%v misses counted", n)
|
||||
}
|
||||
if n := testutil.ToFloat64(rt.m.cacheBytes); n <= 0 {
|
||||
t.Errorf("the cache holds %v bytes", n)
|
||||
}
|
||||
|
||||
// Turned off, nothing is reused, and nothing is said about it.
|
||||
_ = rt.store.SetAnswerCache(t.Context(), mustProjectOf(t, rt, "key-1"), 0, false)
|
||||
rt.reqLogs.by = nil
|
||||
calls := e.calls.Load()
|
||||
if rec := post(t, h, ask(0, "what is 2+2"), auth("off")); rec.Header().Get(cacheHeader) != "" || e.calls.Load() != calls+1 {
|
||||
t.Errorf("a project that reuses nothing: %q, %d calls", rec.Header().Get(cacheHeader), e.calls.Load()-calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProjectThatReusesAnyAnswerReusesSampledOnes(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := cachingRouter(t, e.srv.URL, true)
|
||||
auth := map[string]string{"Authorization": "Bearer " + secret}
|
||||
post(t, rt.Handler(), ask(nil, "write a poem"), auth)
|
||||
if rec := post(t, rt.Handler(), ask(nil, "write a poem"), auth); rec.Header().Get(cacheHeader) != "hit" {
|
||||
t.Errorf("%q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeysOfOneProjectNeverSeeAnotherProjectsAnswers(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := cachingRouter(t, e.srv.URL, false)
|
||||
team, err := rt.store.CreateTeam(t.Context(), mustOrgOf(t, rt), "other")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, err := rt.store.CreateProject(t.Context(), team, "elsewhere")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = rt.store.SetAnswerCache(t.Context(), other, time.Hour, true)
|
||||
secret2 := store.KeyPrefix + strings.Repeat("b", 43)
|
||||
sum := sha256.Sum256([]byte(secret2))
|
||||
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",
|
||||
ProjectID: other, CreatedAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
post(t, rt.Handler(), ask(0, "the secret plan"), map[string]string{"Authorization": "Bearer " + secret})
|
||||
rec := post(t, rt.Handler(), ask(0, "the secret plan"), map[string]string{"Authorization": "Bearer " + secret2})
|
||||
if rec.Header().Get(cacheHeader) != "miss" || e.calls.Load() != 2 {
|
||||
t.Errorf("another project's key: %q, the engine asked %d times", rec.Header().Get(cacheHeader), e.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func mustOrgOf(t *testing.T, rt *Router) string {
|
||||
t.Helper()
|
||||
orgs, err := rt.store.Tree(t.Context())
|
||||
if err != nil || len(orgs) == 0 {
|
||||
t.Fatalf("no organization: %v", err)
|
||||
}
|
||||
return orgs[0].ID
|
||||
}
|
||||
|
||||
func TestAFailedAnswerIsNotReused(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := cachingRouter(t, e.srv.URL, false)
|
||||
auth := map[string]string{"Authorization": "Bearer " + secret}
|
||||
e.status.Store(http.StatusBadRequest)
|
||||
if rec := post(t, rt.Handler(), ask(0, "hello"), auth); rec.Code == 200 {
|
||||
t.Fatalf("%d", rec.Code)
|
||||
}
|
||||
e.status.Store(0)
|
||||
if rec := post(t, rt.Handler(), ask(0, "hello"), auth); rec.Code != 200 || rec.Header().Get(cacheHeader) != "miss" {
|
||||
t.Errorf("after a failure: %d %q", rec.Code, rec.Header().Get(cacheHeader))
|
||||
}
|
||||
}
|
||||
|
||||
// switchGuard blocks every request, or puts every one under no_store, when
|
||||
// told to.
|
||||
type switchGuard struct {
|
||||
block, noStore atomic.Bool
|
||||
// answers, when set, stops answers saying "hi".
|
||||
answers atomic.Bool
|
||||
}
|
||||
|
||||
func (g *switchGuard) Input(context.Context, guard.Subject, []byte) (guard.Verdict, error) {
|
||||
if g.block.Load() {
|
||||
return guard.Verdict{Action: guard.Block, Policy: "nothing"}, nil
|
||||
}
|
||||
return guard.Verdict{}, nil
|
||||
}
|
||||
|
||||
func (g *switchGuard) Output(context.Context, guard.Subject) guard.OutputFilter {
|
||||
if !g.answers.Load() {
|
||||
return nil
|
||||
}
|
||||
return &stopHi{}
|
||||
}
|
||||
|
||||
func (g *switchGuard) Directives(guard.Subject) guard.Directives {
|
||||
return guard.Directives{NoStore: g.noStore.Load()}
|
||||
}
|
||||
|
||||
func TestTheGuardrailsStillDecideAboutAnAnswerFromTheCache(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := cachingRouter(t, e.srv.URL, false)
|
||||
g := &switchGuard{}
|
||||
rt.SetGuard(g)
|
||||
auth := map[string]string{"Authorization": "Bearer " + secret}
|
||||
if rec := post(t, rt.Handler(), ask(0, "hello"), auth); rec.Header().Get(cacheHeader) != "miss" {
|
||||
t.Fatalf("%q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
|
||||
// A policy that now refuses the request refuses it, answer kept or not.
|
||||
g.block.Store(true)
|
||||
if rec := post(t, rt.Handler(), ask(0, "hello"), auth); rec.Code != http.StatusBadRequest || rec.Header().Get(cacheHeader) != "" {
|
||||
t.Errorf("a blocked request: %d %q %s", rec.Code, rec.Header().Get(cacheHeader), rec.Body.String())
|
||||
}
|
||||
g.block.Store(false)
|
||||
|
||||
// A policy that keeps nothing neither reads the cache nor fills it.
|
||||
g.noStore.Store(true)
|
||||
if rec := post(t, rt.Handler(), ask(0, "hello"), auth); rec.Code != 200 || rec.Header().Get(cacheHeader) != "bypass" {
|
||||
t.Errorf("under no_store: %d %q", rec.Code, rec.Header().Get(cacheHeader))
|
||||
}
|
||||
if rec := post(t, rt.Handler(), ask(0, "under no_store"), auth); rec.Header().Get(cacheHeader) != "bypass" {
|
||||
t.Errorf("under no_store: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
g.noStore.Store(false)
|
||||
if rec := post(t, rt.Handler(), ask(0, "under no_store"), auth); rec.Header().Get(cacheHeader) != "miss" {
|
||||
t.Errorf("an answer given under no_store was kept: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
|
||||
// An output policy added since judges the reused answer too.
|
||||
g.answers.Store(true)
|
||||
rec := post(t, rt.Handler(), ask(0, "hello"), auth)
|
||||
if rec.Header().Get(cacheHeader) != "hit" || !strings.Contains(rec.Body.String(), "content_filter") ||
|
||||
rec.Header().Get(guardHeader) != "blocked" {
|
||||
t.Errorf("an answer the output policy stops: %q %q %s", rec.Header().Get(cacheHeader),
|
||||
rec.Header().Get(guardHeader), rec.Body.String())
|
||||
}
|
||||
// And an answer it stopped is not kept.
|
||||
if rec := post(t, rt.Handler(), ask(0, "new question"), auth); rec.Header().Get(cacheHeader) != "miss" {
|
||||
t.Fatalf("%q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
g.answers.Store(false)
|
||||
if rec := post(t, rt.Handler(), ask(0, "new question"), auth); rec.Header().Get(cacheHeader) != "miss" {
|
||||
t.Errorf("an answer a guard stopped was kept: %q", rec.Header().Get(cacheHeader))
|
||||
}
|
||||
}
|
||||
|
||||
// greetingEngine streams "Hello world" to a request that asks for a
|
||||
// stream, with usage when asked, and answers whole otherwise.
|
||||
func greetingEngine(t *testing.T) (*httptest.Server, *atomic.Int64) {
|
||||
t.Helper()
|
||||
var calls atomic.Int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/metrics" {
|
||||
return
|
||||
}
|
||||
calls.Add(1)
|
||||
var b struct {
|
||||
Stream bool `json:"stream"`
|
||||
StreamOptions struct {
|
||||
IncludeUsage bool `json:"include_usage"`
|
||||
} `json:"stream_options"`
|
||||
}
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
_ = json.Unmarshal(body, &b)
|
||||
if !b.Stream {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = io.WriteString(w, `{"id":"c1","object":"chat.completion","created":1,"model":"qwen2.5-14b-instruct",`+
|
||||
`"choices":[{"index":0,"message":{"role":"assistant","content":"Hello world"},"finish_reason":"stop"}],`+
|
||||
`"usage":{"prompt_tokens":5,"completion_tokens":2,"total_tokens":7}}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
head := `{"id":"c1","object":"chat.completion.chunk","created":1,"model":"qwen2.5-14b-instruct","choices":`
|
||||
for _, c := range []string{
|
||||
`[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]}`,
|
||||
`[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}`,
|
||||
`[{"index":0,"delta":{"content":" world"},"finish_reason":null}]}`,
|
||||
`[{"index":0,"delta":{},"finish_reason":"stop"}]}`,
|
||||
} {
|
||||
fmt.Fprintf(w, "data: %s%s\n\n", head, c)
|
||||
}
|
||||
if b.StreamOptions.IncludeUsage {
|
||||
fmt.Fprintf(w, "data: %s[],\"usage\":{\"prompt_tokens\":5,\"completion_tokens\":2,\"total_tokens\":7}}\n\n", head)
|
||||
}
|
||||
_, _ = io.WriteString(w, "data: [DONE]\n\n")
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, &calls
|
||||
}
|
||||
|
||||
// streamedText is the content of a streamed chat answer, and whether it
|
||||
// carried usage and ended with [DONE].
|
||||
func streamedText(t *testing.T, sse string) (text string, usage, done bool) {
|
||||
t.Helper()
|
||||
for _, line := range strings.Split(sse, "\n") {
|
||||
data, ok := strings.CutPrefix(line, "data: ")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if data == "[DONE]" {
|
||||
done = true
|
||||
continue
|
||||
}
|
||||
var c struct {
|
||||
Object string `json:"object"`
|
||||
Choices []struct {
|
||||
Delta struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"delta"`
|
||||
} `json:"choices"`
|
||||
Usage *struct {
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(data), &c); err != nil || c.Object != "chat.completion.chunk" {
|
||||
t.Fatalf("not a chunk: %q", data)
|
||||
}
|
||||
for _, ch := range c.Choices {
|
||||
text += ch.Delta.Content
|
||||
}
|
||||
if c.Usage != nil && c.Usage.CompletionTokens == 2 {
|
||||
usage = true
|
||||
}
|
||||
}
|
||||
return text, usage, done
|
||||
}
|
||||
|
||||
func TestAStreamedAnswerIsReusedForClientsThatStreamAndThatDoNot(t *testing.T) {
|
||||
srv, calls := greetingEngine(t)
|
||||
rt, secret := cachingRouter(t, srv.URL, false)
|
||||
auth := map[string]string{"Authorization": "Bearer " + secret}
|
||||
streamed := func(includeUsage bool) map[string]any {
|
||||
b := ask(0, "greet me")
|
||||
b["stream"] = true
|
||||
if includeUsage {
|
||||
b["stream_options"] = map[string]any{"include_usage": true}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
first := post(t, rt.Handler(), streamed(false), auth)
|
||||
if first.Header().Get(cacheHeader) != "miss" {
|
||||
t.Fatalf("%q %s", first.Header().Get(cacheHeader), first.Body.String())
|
||||
}
|
||||
if text, _, _ := streamedText(t, first.Body.String()); text != "Hello world" {
|
||||
t.Fatalf("streamed %q", text)
|
||||
}
|
||||
|
||||
// Whole, to a client that does not stream.
|
||||
whole := post(t, rt.Handler(), ask(0, "greet me"), auth)
|
||||
var c struct {
|
||||
Object string `json:"object"`
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Role, Content string
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
PromptTokens, CompletionTokens int
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(whole.Body.Bytes(), &c); err != nil || whole.Header().Get(cacheHeader) != "hit" {
|
||||
t.Fatalf("%q %s %v", whole.Header().Get(cacheHeader), whole.Body.String(), err)
|
||||
}
|
||||
if c.Object != "chat.completion" || len(c.Choices) != 1 || c.Choices[0].Message.Content != "Hello world" ||
|
||||
c.Choices[0].Message.Role != "assistant" || c.Choices[0].FinishReason != "stop" {
|
||||
t.Errorf("put back together as %s", whole.Body.String())
|
||||
}
|
||||
|
||||
// Streamed again, with the usage only to a client that asks for it.
|
||||
for _, withUsage := range []bool{false, true} {
|
||||
rec := post(t, rt.Handler(), streamed(withUsage), auth)
|
||||
text, usage, done := streamedText(t, rec.Body.String())
|
||||
if rec.Header().Get(cacheHeader) != "hit" || text != "Hello world" || !done || usage != withUsage ||
|
||||
!strings.HasPrefix(rec.Header().Get("Content-Type"), "text/event-stream") {
|
||||
t.Errorf("streamed from the cache (usage %v): %q %q usage=%v done=%v\n%s", withUsage,
|
||||
rec.Header().Get(cacheHeader), text, usage, done, rec.Body.String())
|
||||
}
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Errorf("the engine was asked %d times", calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestManyClientsReadingAndFillingTheCacheAtOnce(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := cachingRouter(t, e.srv.URL, false)
|
||||
h := rt.Handler()
|
||||
auth := map[string]string{"Authorization": "Bearer " + secret}
|
||||
var wg sync.WaitGroup
|
||||
for i := range 40 {
|
||||
wg.Go(func() {
|
||||
rec := post(t, h, ask(0, fmt.Sprintf("question %d", i%5)), auth)
|
||||
if rec.Code != 200 {
|
||||
t.Errorf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
for i := range 5 {
|
||||
if rec := post(t, h, ask(0, fmt.Sprintf("question %d", i)), auth); rec.Header().Get(cacheHeader) != "hit" {
|
||||
t.Errorf("question %d: %q", i, rec.Header().Get(cacheHeader))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAnswerCacheDropsWhatWasUsedLongestAgoPastItsBudget(t *testing.T) {
|
||||
var held int64
|
||||
c := newAnswerCache(1000, 400, func(b int64, _ int) { held = b })
|
||||
now := time.Now()
|
||||
put := func(key string) bool {
|
||||
return c.put(&cachedAnswer{key: key, project: "p", body: bytes.Repeat([]byte("x"), 100),
|
||||
at: now, expires: now.Add(time.Hour)}, now)
|
||||
}
|
||||
for _, k := range []string{"a", "b", "c"} {
|
||||
put(k)
|
||||
}
|
||||
c.get("a", time.Hour, now) // a is now the most recently used
|
||||
put("d")
|
||||
put("e")
|
||||
if c.get("b", time.Hour, now) != nil {
|
||||
t.Error("b, used longest ago, was kept past the budget")
|
||||
}
|
||||
if c.get("a", time.Hour, now) == nil {
|
||||
t.Error("a, used recently, was dropped")
|
||||
}
|
||||
if held > 1000 {
|
||||
t.Errorf("holding %d bytes of 1000", held)
|
||||
}
|
||||
if c.put(&cachedAnswer{key: "big", body: bytes.Repeat([]byte("x"), 500)}, now) {
|
||||
t.Error("an answer larger than an entry may be was kept")
|
||||
}
|
||||
// Past its TTL, or past a TTL shortened since, it is gone.
|
||||
if c.get("a", time.Hour, now.Add(2*time.Hour)) != nil {
|
||||
t.Error("an expired answer was returned")
|
||||
}
|
||||
if c.get("e", time.Minute, now.Add(2*time.Minute)) != nil {
|
||||
t.Error("an answer older than the project's TTL now was returned")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerIsLookedUpByWhatChangesIt(t *testing.T) {
|
||||
key := func(project, model, body string) string {
|
||||
k, ok := answerKey(project, "/v1/chat/completions", model, []byte(body))
|
||||
if !ok {
|
||||
t.Fatalf("no key for %s", body)
|
||||
}
|
||||
return k
|
||||
}
|
||||
base := key("p", "m", `{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0}`)
|
||||
same := []string{
|
||||
`{"messages":[{"content":"hi","role":"user"}],"temperature":0.0,"model":"alias"}`,
|
||||
`{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0,"stream":true,"stream_options":{"include_usage":true}}`,
|
||||
`{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0e0,"user":"u","metadata":{"a":"b"}}`,
|
||||
}
|
||||
for _, b := range same {
|
||||
if key("p", "m", b) != base {
|
||||
t.Errorf("%s was looked up apart", b)
|
||||
}
|
||||
}
|
||||
different := []string{
|
||||
`{"model":"m","messages":[{"role":"user","content":"hi!"}],"temperature":0}`,
|
||||
`{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0,"max_tokens":5}`,
|
||||
`{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0,"seed":9007199254740993}`,
|
||||
}
|
||||
for _, b := range different {
|
||||
if key("p", "m", b) == base {
|
||||
t.Errorf("%s was looked up as the same", b)
|
||||
}
|
||||
}
|
||||
if key("p", "m", `{"seed":9007199254740993}`) == key("p", "m", `{"seed":9007199254740992}`) {
|
||||
t.Error("two large seeds were taken for one")
|
||||
}
|
||||
if key("other", "m", `{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0}`) == base {
|
||||
t.Error("another project's request was looked up as the same")
|
||||
}
|
||||
if key("p", "other", `{"model":"m","messages":[{"role":"user","content":"hi"}],"temperature":0}`) == base {
|
||||
t.Error("another model's request was looked up as the same")
|
||||
}
|
||||
if _, ok := answerKey("p", "/v1/chat/completions", "m", []byte(`not json`)); ok {
|
||||
t.Error("a body that is not JSON was given a key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStreamedAnswerIsPutBackTogetherAndTakenApartAgain(t *testing.T) {
|
||||
sse := strings.Join([]string{
|
||||
`data: {"id":"c","object":"chat.completion.chunk","created":7,"model":"m","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}`,
|
||||
`data: {"id":"c","object":"chat.completion.chunk","created":7,"model":"m","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"get_weather","arguments":""}}]},"finish_reason":null}]}`,
|
||||
`data: {"id":"c","object":"chat.completion.chunk","created":7,"model":"m","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"city\":"}}]},"finish_reason":null}]}`,
|
||||
`data: {"id":"c","object":"chat.completion.chunk","created":7,"model":"m","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Paris\"}"}}]},"finish_reason":null}]}`,
|
||||
`data: {"id":"c","object":"chat.completion.chunk","created":7,"model":"m","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}`,
|
||||
`data: [DONE]`,
|
||||
}, "\n\n") + "\n\n"
|
||||
body, ok := assembleStream("/v1/chat/completions", []byte(sse), &proxy.Usage{PromptTokens: 3, CompletionTokens: 4})
|
||||
if !ok {
|
||||
t.Fatal("not put back together")
|
||||
}
|
||||
var c struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content *string `json:"content"`
|
||||
ToolCalls []toolCall `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
} `json:"choices"`
|
||||
Usage struct {
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(c.Choices) != 1 || c.Choices[0].Message.Content != nil || len(c.Choices[0].Message.ToolCalls) != 1 ||
|
||||
c.Choices[0].Message.ToolCalls[0].Function.Arguments != `{"city":"Paris"}` ||
|
||||
c.Choices[0].Message.ToolCalls[0].ID != "call_1" || c.Choices[0].FinishReason != "tool_calls" ||
|
||||
c.Usage.TotalTokens != 7 {
|
||||
t.Errorf("put back together as %s", body)
|
||||
}
|
||||
|
||||
// Taken apart, and put back together again, it is the same answer.
|
||||
events, err := streamFromAnswer("/v1/chat/completions", body, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, ok := assembleStream("/v1/chat/completions",
|
||||
[]byte("data: "+strings.Join(append(events, "[DONE]"), "\n\ndata: ")+"\n\n"), &proxy.Usage{PromptTokens: 3, CompletionTokens: 4})
|
||||
if !ok || !bytes.Equal(again, body) {
|
||||
t.Errorf("round trip:\n%s\n%s", body, again)
|
||||
}
|
||||
|
||||
// What it does not understand, and a stream cut short, are not kept.
|
||||
for _, bad := range []string{
|
||||
strings.Replace(sse, `"delta":{}`, `"delta":{"audio":{"id":"a"}}`, 1),
|
||||
strings.Replace(sse, `"finish_reason":null}]}`, `"finish_reason":null,"logprobs":{"content":[]}}]}`, 1),
|
||||
strings.Replace(sse, "data: [DONE]", "", 1),
|
||||
} {
|
||||
if _, ok := assembleStream("/v1/chat/completions", []byte(bad), nil); ok {
|
||||
t.Errorf("kept: %s", bad)
|
||||
}
|
||||
}
|
||||
|
||||
// A completion, too.
|
||||
text := `data: {"id":"t","object":"text_completion","created":1,"model":"m","choices":[{"index":0,"text":"Hel","finish_reason":null}]}` + "\n\n" +
|
||||
`data: {"id":"t","object":"text_completion","created":1,"model":"m","choices":[{"index":0,"text":"lo","finish_reason":"length"}]}` + "\n\n" +
|
||||
"data: [DONE]\n\n"
|
||||
tb, ok := assembleStream("/v1/completions", []byte(text), nil)
|
||||
if !ok || !strings.Contains(string(tb), `"text":"Hello"`) || !strings.Contains(string(tb), `"object":"text_completion"`) {
|
||||
t.Errorf("a completion: %s", tb)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerFromTheCacheReachesAnthropicClientsInTheirShape(t *testing.T) {
|
||||
srv, calls := greetingEngine(t)
|
||||
rt, secret := cachingRouter(t, srv.URL, false)
|
||||
send := func(stream bool) *httptest.ResponseRecorder {
|
||||
b, _ := json.Marshal(map[string]any{"model": "q14", "max_tokens": 50, "temperature": 0, "stream": stream,
|
||||
"messages": []any{map[string]any{"role": "user", "content": "greet me"}}})
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/messages", bytes.NewReader(b))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("x-api-key", secret)
|
||||
rec := httptest.NewRecorder()
|
||||
rt.Handler().ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
first := send(true)
|
||||
if first.Code != 200 || !strings.Contains(first.Body.String(), "Hello") {
|
||||
t.Fatalf("%d %s", first.Code, first.Body.String())
|
||||
}
|
||||
whole := send(false)
|
||||
if whole.Code != 200 || !strings.Contains(whole.Body.String(), `"text":"Hello world"`) {
|
||||
t.Errorf("whole: %d %s", whole.Code, whole.Body.String())
|
||||
}
|
||||
streamed := send(true)
|
||||
if !strings.Contains(streamed.Body.String(), "content_block_delta") ||
|
||||
!strings.Contains(streamed.Body.String(), "Hello world") ||
|
||||
!strings.Contains(streamed.Body.String(), "message_stop") {
|
||||
t.Errorf("streamed: %s", streamed.Body.String())
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Errorf("the engine was asked %d times", calls.Load())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
||||
)
|
||||
|
||||
// A cached answer is kept as one JSON object whichever way it was asked
|
||||
// for, so a client that streams and one that does not share it. A streamed
|
||||
// answer is put back together into the object a model would have sent
|
||||
// whole, and an answer is taken apart into chunks for a client that
|
||||
// streams.
|
||||
|
||||
// streamChunk is one streamed chunk of a chat or a completion.
|
||||
type streamChunk struct {
|
||||
ID string `json:"id"`
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
SystemFingerprint string `json:"system_fingerprint"`
|
||||
Choices []struct {
|
||||
Index int `json:"index"`
|
||||
// Delta is a chat chunk's, as fields: those not understood here
|
||||
// leave the answer unkept rather than kept without them.
|
||||
Delta map[string]json.RawMessage `json:"delta"`
|
||||
Text *string `json:"text"`
|
||||
FinishReason *string `json:"finish_reason"`
|
||||
Logprobs json.RawMessage `json:"logprobs"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
|
||||
// toolCallDelta is part of one tool call, as streamed.
|
||||
type toolCallDelta struct {
|
||||
Index int `json:"index"`
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Function struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
} `json:"function"`
|
||||
}
|
||||
|
||||
type toolCall struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Function struct {
|
||||
Name string `json:"name"`
|
||||
Arguments string `json:"arguments"`
|
||||
} `json:"function"`
|
||||
}
|
||||
|
||||
// assembledChoice is one choice being put back together.
|
||||
type assembledChoice struct {
|
||||
role string
|
||||
content, reasoning, text strings.Builder
|
||||
hasContent, hasReasoning bool
|
||||
tools []*toolCall
|
||||
finish *string
|
||||
}
|
||||
|
||||
// assembleStream puts a streamed chat or completion back together as the
|
||||
// object the model would have answered without streaming, with usage as the
|
||||
// engine reported it. ok is false for anything it does not fully
|
||||
// understand — logprobs, audio, a chunk that is not JSON — which is then
|
||||
// not kept: an answer kept with something missing would be served short.
|
||||
func assembleStream(path string, sse []byte, usage *proxy.Usage) ([]byte, bool) {
|
||||
chat := path == "/v1/chat/completions"
|
||||
if !chat && path != "/v1/completions" {
|
||||
return nil, false
|
||||
}
|
||||
var head streamChunk
|
||||
choices := map[int]*assembledChoice{}
|
||||
maxIndex := -1
|
||||
done := false
|
||||
sc := bufio.NewScanner(bytes.NewReader(sse))
|
||||
sc.Buffer(make([]byte, 64<<10), len(sse)+1)
|
||||
for sc.Scan() {
|
||||
data, ok := strings.CutPrefix(sc.Text(), "data:")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
data = strings.TrimSpace(data)
|
||||
if data == "[DONE]" {
|
||||
done = true
|
||||
continue
|
||||
}
|
||||
var c streamChunk
|
||||
if json.Unmarshal([]byte(data), &c) != nil {
|
||||
return nil, false
|
||||
}
|
||||
if head.ID == "" {
|
||||
head.ID, head.Created, head.Model = c.ID, c.Created, c.Model
|
||||
}
|
||||
if c.SystemFingerprint != "" {
|
||||
head.SystemFingerprint = c.SystemFingerprint
|
||||
}
|
||||
for _, ch := range c.Choices {
|
||||
if len(ch.Logprobs) > 0 && string(ch.Logprobs) != "null" {
|
||||
return nil, false
|
||||
}
|
||||
a := choices[ch.Index]
|
||||
if a == nil {
|
||||
a = &assembledChoice{}
|
||||
choices[ch.Index] = a
|
||||
maxIndex = max(maxIndex, ch.Index)
|
||||
}
|
||||
if ch.FinishReason != nil {
|
||||
a.finish = ch.FinishReason
|
||||
}
|
||||
if ch.Text != nil {
|
||||
a.text.WriteString(*ch.Text)
|
||||
}
|
||||
if !addDelta(a, ch.Delta) {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
}
|
||||
if sc.Err() != nil || !done || len(choices) == 0 || len(choices) != maxIndex+1 {
|
||||
// A stream that never said it was done may have been cut short.
|
||||
return nil, false
|
||||
}
|
||||
out := map[string]any{"id": head.ID, "created": head.Created, "model": head.Model}
|
||||
if head.SystemFingerprint != "" {
|
||||
out["system_fingerprint"] = head.SystemFingerprint
|
||||
}
|
||||
list := make([]map[string]any, 0, len(choices))
|
||||
for i := 0; i <= maxIndex; i++ {
|
||||
a := choices[i]
|
||||
if a.finish == nil {
|
||||
return nil, false
|
||||
}
|
||||
c := map[string]any{"index": i, "finish_reason": *a.finish, "logprobs": nil}
|
||||
if chat {
|
||||
role := a.role
|
||||
if role == "" {
|
||||
role = "assistant"
|
||||
}
|
||||
msg := map[string]any{"role": role, "content": nil}
|
||||
if a.hasContent || len(a.tools) == 0 {
|
||||
msg["content"] = a.content.String()
|
||||
}
|
||||
if a.hasReasoning {
|
||||
msg["reasoning_content"] = a.reasoning.String()
|
||||
}
|
||||
if len(a.tools) > 0 {
|
||||
msg["tool_calls"] = a.tools
|
||||
}
|
||||
c["message"] = msg
|
||||
} else {
|
||||
c["text"] = a.text.String()
|
||||
}
|
||||
list = append(list, c)
|
||||
}
|
||||
out["choices"] = list
|
||||
if chat {
|
||||
out["object"] = "chat.completion"
|
||||
} else {
|
||||
out["object"] = "text_completion"
|
||||
}
|
||||
if usage != nil {
|
||||
out["usage"] = map[string]int{"prompt_tokens": usage.PromptTokens,
|
||||
"completion_tokens": usage.CompletionTokens,
|
||||
"total_tokens": usage.PromptTokens + usage.CompletionTokens}
|
||||
}
|
||||
b, err := json.Marshal(out)
|
||||
return b, err == nil
|
||||
}
|
||||
|
||||
// addDelta adds one chat chunk's delta to its choice, and says whether it
|
||||
// understood every field of it.
|
||||
func addDelta(a *assembledChoice, delta map[string]json.RawMessage) bool {
|
||||
for k, raw := range delta {
|
||||
if string(raw) == "null" {
|
||||
continue
|
||||
}
|
||||
switch k {
|
||||
case "role":
|
||||
if json.Unmarshal(raw, &a.role) != nil {
|
||||
return false
|
||||
}
|
||||
case "content", "reasoning_content":
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) != nil {
|
||||
return false
|
||||
}
|
||||
// Empty says nothing: the first chunk carries the role with an
|
||||
// empty content even when the answer is a tool call.
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if k == "content" {
|
||||
a.content.WriteString(s)
|
||||
a.hasContent = true
|
||||
} else {
|
||||
a.reasoning.WriteString(s)
|
||||
a.hasReasoning = true
|
||||
}
|
||||
case "refusal":
|
||||
// An empty refusal is a field some engines always send.
|
||||
var s string
|
||||
if json.Unmarshal(raw, &s) != nil || s != "" {
|
||||
return false
|
||||
}
|
||||
case "tool_calls":
|
||||
var parts []toolCallDelta
|
||||
if json.Unmarshal(raw, &parts) != nil {
|
||||
return false
|
||||
}
|
||||
for _, p := range parts {
|
||||
for len(a.tools) <= p.Index {
|
||||
a.tools = append(a.tools, &toolCall{})
|
||||
}
|
||||
t := a.tools[p.Index]
|
||||
if p.ID != "" {
|
||||
t.ID = p.ID
|
||||
}
|
||||
if p.Type != "" {
|
||||
t.Type = p.Type
|
||||
}
|
||||
t.Function.Name += p.Function.Name
|
||||
t.Function.Arguments += p.Function.Arguments
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// streamFromAnswer is a kept chat or completion as the chunks a model would
|
||||
// have streamed: per choice, its whole text in one chunk and then its
|
||||
// finish reason, and the usage last for a client that asks for it.
|
||||
func streamFromAnswer(path string, body []byte, withUsage bool) ([]string, error) {
|
||||
var a struct {
|
||||
ID string `json:"id"`
|
||||
Created int64 `json:"created"`
|
||||
Model string `json:"model"`
|
||||
SystemFingerprint string `json:"system_fingerprint"`
|
||||
Usage json.RawMessage `json:"usage"`
|
||||
Choices []struct {
|
||||
Index int `json:"index"`
|
||||
FinishReason json.RawMessage `json:"finish_reason"`
|
||||
Logprobs json.RawMessage `json:"logprobs"`
|
||||
Text *string `json:"text"`
|
||||
Message *struct {
|
||||
Role string `json:"role"`
|
||||
Content *string `json:"content"`
|
||||
ReasoningContent *string `json:"reasoning_content"`
|
||||
ToolCalls []json.RawMessage `json:"tool_calls"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &a); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(a.Choices) == 0 {
|
||||
return nil, errors.New("an answer with no choices")
|
||||
}
|
||||
chat := path == "/v1/chat/completions"
|
||||
object := "text_completion"
|
||||
if chat {
|
||||
object = "chat.completion.chunk"
|
||||
}
|
||||
chunk := func(choices []map[string]any) (string, error) {
|
||||
c := map[string]any{"id": a.ID, "object": object, "created": a.Created, "model": a.Model,
|
||||
"choices": choices}
|
||||
if a.SystemFingerprint != "" {
|
||||
c["system_fingerprint"] = a.SystemFingerprint
|
||||
}
|
||||
b, err := json.Marshal(c)
|
||||
return string(b), err
|
||||
}
|
||||
var events []string
|
||||
add := func(choices []map[string]any) error {
|
||||
e, err := chunk(choices)
|
||||
events = append(events, e)
|
||||
return err
|
||||
}
|
||||
for _, ch := range a.Choices {
|
||||
var logprobs any
|
||||
if len(ch.Logprobs) > 0 {
|
||||
logprobs = ch.Logprobs
|
||||
}
|
||||
finish := any(json.RawMessage("null"))
|
||||
if len(ch.FinishReason) > 0 {
|
||||
finish = ch.FinishReason
|
||||
}
|
||||
if !chat {
|
||||
if ch.Text == nil {
|
||||
return nil, errors.New("a completion with no text")
|
||||
}
|
||||
if err := add([]map[string]any{{"index": ch.Index, "text": *ch.Text, "logprobs": logprobs,
|
||||
"finish_reason": nil}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := add([]map[string]any{{"index": ch.Index, "text": "", "finish_reason": finish}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
m := ch.Message
|
||||
if m == nil {
|
||||
return nil, errors.New("a chat answer with no message")
|
||||
}
|
||||
role := m.Role
|
||||
if role == "" {
|
||||
role = "assistant"
|
||||
}
|
||||
if err := add([]map[string]any{{"index": ch.Index, "delta": map[string]any{"role": role, "content": ""},
|
||||
"finish_reason": nil}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if m.ReasoningContent != nil && *m.ReasoningContent != "" {
|
||||
if err := add([]map[string]any{{"index": ch.Index,
|
||||
"delta": map[string]any{"reasoning_content": *m.ReasoningContent}, "finish_reason": nil}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if m.Content != nil && *m.Content != "" || logprobs != nil {
|
||||
content := ""
|
||||
if m.Content != nil {
|
||||
content = *m.Content
|
||||
}
|
||||
if err := add([]map[string]any{{"index": ch.Index, "delta": map[string]any{"content": content},
|
||||
"logprobs": logprobs, "finish_reason": nil}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(m.ToolCalls) > 0 {
|
||||
calls := make([]map[string]any, 0, len(m.ToolCalls))
|
||||
for i, raw := range m.ToolCalls {
|
||||
var tc map[string]any
|
||||
if err := json.Unmarshal(raw, &tc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tc["index"] = i
|
||||
calls = append(calls, tc)
|
||||
}
|
||||
if err := add([]map[string]any{{"index": ch.Index, "delta": map[string]any{"tool_calls": calls},
|
||||
"finish_reason": nil}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if err := add([]map[string]any{{"index": ch.Index, "delta": map[string]any{},
|
||||
"finish_reason": finish}}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if withUsage && len(a.Usage) > 0 && string(a.Usage) != "null" {
|
||||
c := map[string]any{"id": a.ID, "object": object, "created": a.Created, "model": a.Model,
|
||||
"choices": []any{}, "usage": a.Usage}
|
||||
b, err := json.Marshal(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
events = append(events, string(b))
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
)
|
||||
|
||||
func docsRouter(t *testing.T, signIn bool) http.Handler {
|
||||
t.Helper()
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: "http://127.0.0.1:1", Models: []string{"m"}})
|
||||
cfg.Auth.Enabled = true
|
||||
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
||||
cfg.Landing.DocsSignIn = signIn
|
||||
return newTestRouter(t, cfg).Handler()
|
||||
}
|
||||
|
||||
// The docs are public by default on a gateway with keys: the client guide is
|
||||
// for the person who has none yet, so asking for one first defeats it.
|
||||
func TestTheDocsAreServedWithoutAKey(t *testing.T) {
|
||||
h := docsRouter(t, false)
|
||||
for _, path := range []string{"/docs", "/docs/clients"} {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), "llmgw docs") {
|
||||
t.Errorf("%s: %d %.200s", path, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDocsSignInSendsAVisitorToSignIn(t *testing.T) {
|
||||
h := docsRouter(t, true)
|
||||
for _, path := range []string{"/docs", "/docs/install"} {
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil))
|
||||
// The session guard, not the key check: a browser has no key.
|
||||
if strings.Contains(w.Body.String(), "api key") {
|
||||
t.Errorf("%s asked a browser for an API key", path)
|
||||
}
|
||||
if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/login" {
|
||||
t.Errorf("%s: %d to %q, want a redirect to /login", path, w.Code, w.Header().Get("Location"))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -234,6 +234,11 @@ func (r *Router) Handler() http.Handler {
|
||||
pages.RegisterAdminGuard(mux, r.store, r.guardLibrary)
|
||||
}
|
||||
pages.Register(mux)
|
||||
// The docs portal, with the menu of whoever is signed in, and only to
|
||||
// them when the site asked for that.
|
||||
docs := pages.DocsPage(web.Docs(version.Version, true), r.landing.DocsSignIn)
|
||||
mux.HandleFunc("GET /docs", docs)
|
||||
mux.HandleFunc("GET /docs/{doc}", docs)
|
||||
r.registerChat(mux, pages)
|
||||
// An organization's own page, for the people who manage it.
|
||||
pages.RegisterOrgPages(mux, r.store, r.acct)
|
||||
@@ -285,18 +290,29 @@ func (r *Router) Handler() http.Handler {
|
||||
Catalog: r.catalogAdmin, Fleet: r.clusterView,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
// No accounts, so nobody to show a menu to or to send to sign in.
|
||||
docs := web.Docs(version.Version, false)
|
||||
mux.HandleFunc("GET /docs", docs)
|
||||
mux.HandleFunc("GET /docs/{doc}", docs)
|
||||
}
|
||||
|
||||
if r.auth == nil {
|
||||
return mux
|
||||
}
|
||||
exempt := []string{"/health", "/ready", "/leader", "/", "/about", "/_static/", "/_lang/", "/claim/",
|
||||
"/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/",
|
||||
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/"}
|
||||
// The docs are public unless the site said otherwise, and then guarded by
|
||||
// the session check in front of them — or, on a gateway with keys and no
|
||||
// accounts to sign in to, by the key like any other path.
|
||||
if !r.landing.DocsSignIn || r.accounts != nil {
|
||||
exempt = append(exempt, "/docs", "/docs/")
|
||||
}
|
||||
// Wrapping the whole mux rather than checking inside each handler: adding a
|
||||
// route must not be able to add an unauthenticated one by omission. Health
|
||||
// stays open because a load balancer has no key.
|
||||
return auth.Middleware(r.auth, r.log,
|
||||
"/health", "/ready", "/leader", "/", "/about", "/_static/", "/_lang/", "/claim/",
|
||||
"/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/",
|
||||
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux)
|
||||
return auth.Middleware(r.auth, r.log, exempt...)(mux)
|
||||
}
|
||||
|
||||
// parsedRequest is what the router needs from a request. Everything else in the
|
||||
@@ -369,6 +385,9 @@ type parsedRequest struct {
|
||||
// internal is a request the gateway makes of itself — a judge's — which
|
||||
// no guard filters.
|
||||
internal bool
|
||||
// cache is where its answer is kept, when its project reuses answers
|
||||
// and this one may be.
|
||||
cache cacheState
|
||||
}
|
||||
|
||||
// inferenceBody covers the fields the router reads across all inference
|
||||
@@ -534,8 +553,17 @@ func (r *Router) handleInference(path string) http.HandlerFunc {
|
||||
}
|
||||
|
||||
r.decideRequestLog(req.Context(), parsed)
|
||||
res := r.dispatch(req.Context(), w, parsed)
|
||||
// After the guardrails, so a policy still decides whether a request is
|
||||
// answered, and the request looked up is the one the model would be
|
||||
// sent.
|
||||
if a := r.cacheLookup(req.Context(), w, parsed); a != nil {
|
||||
r.serveCached(req.Context(), w, parsed, a, start)
|
||||
return
|
||||
}
|
||||
dw, cw := r.cacheWriterFor(w, parsed)
|
||||
res := r.dispatch(req.Context(), dw, parsed)
|
||||
status := r.finish(w, parsed, res, start)
|
||||
r.cacheKeep(parsed, res, status, cw)
|
||||
r.observeCanary(parsed, res, status, time.Since(start))
|
||||
if parsed.logRetain > 0 {
|
||||
go r.keepRequest(context.WithoutCancel(req.Context()), parsed, res, status)
|
||||
@@ -648,6 +676,9 @@ func (r *Router) account(req *parsedRequest, res dispatchResult, status int, too
|
||||
EndUser: req.EndUser, Tags: req.Tags,
|
||||
}
|
||||
guardFields(&u, req.guardIn, res.guard)
|
||||
// An answer reused generates nothing: its usage is the zero the hit
|
||||
// carries, and it costs nothing against a budget or a token limit.
|
||||
u.CacheHit = res.cached
|
||||
if u.ServedModel == "" {
|
||||
u.ServedModel = req.Model
|
||||
}
|
||||
@@ -1172,6 +1203,9 @@ func (r *Router) accessLog(req *parsedRequest, res dispatchResult, status int, t
|
||||
if res.outcome.Stopped {
|
||||
attrs = append(attrs, "stopped_by_guard", true)
|
||||
}
|
||||
if res.cached {
|
||||
attrs = append(attrs, "cache", "hit")
|
||||
}
|
||||
r.log.Info("request", attrs...)
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,12 @@ type metrics struct {
|
||||
|
||||
prefixDecisions *prometheus.CounterVec
|
||||
|
||||
// cacheRequests counts what the answer cache did, cacheBytes and
|
||||
// cacheEntries what it holds.
|
||||
cacheRequests *prometheus.CounterVec
|
||||
cacheBytes prometheus.Gauge
|
||||
cacheEntries prometheus.Gauge
|
||||
|
||||
tenant *tenantMetrics
|
||||
guard *guardMetrics
|
||||
token *tokenMetrics
|
||||
@@ -91,6 +97,20 @@ func newMetrics() *metrics {
|
||||
Name: "llmgw_admission_rejected_total",
|
||||
Help: "Requests refused by flow control, by reason.",
|
||||
}, []string{"band", "reason"}),
|
||||
cacheRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "llmgw_cache_requests_total",
|
||||
Help: "Requests of projects that reuse answers, by what the answer cache did: " +
|
||||
"hit (answered from it), miss (asked the model, and kept the answer if it could), " +
|
||||
"bypass (not one whose answer is reused).",
|
||||
}, []string{"model", "result"}),
|
||||
cacheBytes: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "llmgw_cache_bytes",
|
||||
Help: "Memory the answers kept for reuse take in this process, roughly.",
|
||||
}),
|
||||
cacheEntries: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "llmgw_cache_entries",
|
||||
Help: "Answers kept for reuse in this process.",
|
||||
}),
|
||||
tenant: newTenantMetrics(),
|
||||
guard: newGuardMetrics(),
|
||||
token: newTokenMetrics(),
|
||||
@@ -103,6 +123,7 @@ func (m *metrics) collectors() []prometheus.Collector {
|
||||
m.requests, m.duration, m.ttft, m.tokens,
|
||||
m.scorerValue, m.filtered, m.considered,
|
||||
m.admissionWait, m.admissionRejected, m.prefixDecisions,
|
||||
m.cacheRequests, m.cacheBytes, m.cacheEntries,
|
||||
}, slices.Concat(m.tenant.collectors(), m.guard.collectors(), m.token.collectors())...)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// answeringEngine is an engine that answers a few questions right, one
|
||||
// wrong, and grades as a judge would.
|
||||
func answeringEngine(t *testing.T, served string) (*httptest.Server, *atomic.Int32) {
|
||||
t.Helper()
|
||||
var calls atomic.Int32
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
|
||||
fmt.Fprintf(w, "# TYPE vllm:num_requests_running gauge\nvllm:num_requests_running 0\n")
|
||||
})
|
||||
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
var req struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
Stream bool `json:"stream"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Stream || len(req.Messages) == 0 {
|
||||
http.Error(w, "bad request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
q := req.Messages[len(req.Messages)-1].Content
|
||||
answer := "I am not sure."
|
||||
switch {
|
||||
case served == "grader" && strings.Contains(q, "Answer:\nParis"):
|
||||
answer = `{"score": 1, "reason": "names Paris"}`
|
||||
case served == "grader":
|
||||
answer = `{"score": 0, "reason": "does not name Paris"}`
|
||||
case strings.Contains(q, "17 + 25"):
|
||||
answer = "<think>17 and 25</think>42"
|
||||
case strings.Contains(q, "6 * 7"):
|
||||
answer = "6 * 7 = 43"
|
||||
case strings.Contains(q, "JSON"):
|
||||
answer = "```json\n{\"ok\": true}\n```"
|
||||
case strings.Contains(q, "capital"):
|
||||
answer = "Paris"
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"model": req.Model, "choices": []any{map[string]any{
|
||||
"message": map[string]any{"role": "assistant", "content": answer}, "finish_reason": "stop"}},
|
||||
"usage": map[string]any{"prompt_tokens": 5, "completion_tokens": 3}})
|
||||
})
|
||||
srv := httptest.NewServer(mux)
|
||||
t.Cleanup(srv.Close)
|
||||
return srv, &calls
|
||||
}
|
||||
|
||||
const qualitySuite = `
|
||||
name: smoke
|
||||
threshold: 0.7
|
||||
judge: grader
|
||||
cases:
|
||||
- id: add
|
||||
prompt: What is 17 + 25?
|
||||
number: 42
|
||||
- id: mul
|
||||
prompt: What is 6 * 7?
|
||||
number: 42
|
||||
- id: json
|
||||
prompt: Give a JSON object with ok true.
|
||||
json_schema: {type: object, required: [ok], properties: {ok: {const: true}}}
|
||||
- id: capital
|
||||
prompt: What is the capital of France?
|
||||
judge: It names Paris.
|
||||
weight: 2
|
||||
`
|
||||
|
||||
func TestASuiteIsAskedThroughTheRouterAndScoredAndKept(t *testing.T) {
|
||||
model, modelCalls := answeringEngine(t, "qwen")
|
||||
grader, graderCalls := answeringEngine(t, "grader")
|
||||
rt := newTestRouter(t, baseConfig(
|
||||
config.Endpoint{ID: "a", URL: model.URL, Models: []string{"qwen"}},
|
||||
config.Endpoint{ID: "j", URL: grader.URL, Models: []string{"grader"}},
|
||||
))
|
||||
db, err := store.OpenSQLite(t.Context(), filepath.Join(t.TempDir(), "store.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
svc := eval.NewService(eval.Deps{Call: rt.Replay, Store: db, Log: slog.New(slog.DiscardHandler),
|
||||
Builtin: fstest.MapFS{"smoke.yaml": {Data: []byte(qualitySuite)}}})
|
||||
|
||||
live, err := svc.Start("qwen", eval.Request{Suite: "smoke", Concurrency: 2}, "tester")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for live.State == eval.Running && time.Now().Before(deadline) {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
live, _ = svc.Get(t.Context(), live.ID)
|
||||
}
|
||||
// add 1, mul 0, json 1, capital 1 at weight 2: 4 of 5.
|
||||
if live.State != eval.Done || live.Score != 0.8 || !live.Passed || live.Below != 1 || live.Errors != 0 {
|
||||
t.Fatalf("%+v", live)
|
||||
}
|
||||
if modelCalls.Load() != 4 || graderCalls.Load() != 1 {
|
||||
t.Errorf("the model was asked %d times and the judge %d", modelCalls.Load(), graderCalls.Load())
|
||||
}
|
||||
kept, err := db.EvalRunByID(t.Context(), live.ID)
|
||||
if err != nil || len(kept.Results) != 4 || kept.Results[0].Answer != "42" ||
|
||||
!strings.Contains(kept.Results[1].Detail, "read 43") || kept.Results[3].Detail != "the judge: names Paris" {
|
||||
t.Fatalf("kept %+v %v", kept, err)
|
||||
}
|
||||
|
||||
// A model nothing serves scores nothing, and says why.
|
||||
run, err := svc.Run(t.Context(), "nothing", eval.Request{Suite: "smoke"}, "tester", "")
|
||||
if err != nil || run.Passed || run.Errors != 4 || !strings.Contains(run.Error, "no case was answered") {
|
||||
t.Fatalf("%+v %v", run, err)
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,9 @@ type requestLogs struct {
|
||||
type cachedLog struct {
|
||||
l storepkg.RequestLog
|
||||
inCluster bool
|
||||
at time.Time
|
||||
// cache is how the project reuses answers.
|
||||
cache storepkg.AnswerCache
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// requestLogOf is a project's setting, read at most every requestLogTTL.
|
||||
@@ -49,7 +51,8 @@ func (r *Router) requestLogOf(ctx context.Context, project string) storepkg.Requ
|
||||
return c.l
|
||||
}
|
||||
|
||||
// readProjectPolicy reads a project's keeping and residency, and caches them.
|
||||
// readProjectPolicy reads a project's keeping, residency and reuse of
|
||||
// answers, and caches them.
|
||||
func (r *Router) readProjectPolicy(ctx context.Context, project string) (cachedLog, error) {
|
||||
l, err := r.store.RequestLogOf(ctx, project)
|
||||
if err != nil {
|
||||
@@ -59,7 +62,11 @@ func (r *Router) readProjectPolicy(ctx context.Context, project string) (cachedL
|
||||
if err != nil {
|
||||
return cachedLog{}, err
|
||||
}
|
||||
c := cachedLog{l: l, inCluster: in, at: time.Now()}
|
||||
ac, err := r.store.AnswerCacheOf(ctx, project)
|
||||
if err != nil {
|
||||
return cachedLog{}, err
|
||||
}
|
||||
c := cachedLog{l: l, inCluster: in, cache: ac, at: time.Now()}
|
||||
r.reqLogs.mu.Lock()
|
||||
if r.reqLogs.by == nil {
|
||||
r.reqLogs.by = map[string]cachedLog{}
|
||||
|
||||
@@ -74,6 +74,8 @@ type Router struct {
|
||||
batchWake chan struct{}
|
||||
// reqLogs is each project's keeping of requests, as last read.
|
||||
reqLogs requestLogs
|
||||
// answers are the answers projects reuse; nil reuses none.
|
||||
answers *answerCache
|
||||
|
||||
// extraMetrics are collectors other components publish on /metrics.
|
||||
extraMetrics []prometheus.Collector
|
||||
@@ -271,6 +273,11 @@ func New(cfg config.Server, log *slog.Logger) (*Router, error) {
|
||||
MaxResponseBytes: cfg.Router.MaxBodyBytes,
|
||||
}),
|
||||
}
|
||||
r.answers = newAnswerCache(cfg.Router.Cache.MaxBytes, cfg.Router.Cache.MaxEntryBytes,
|
||||
func(bytes int64, n int) {
|
||||
metrics.cacheBytes.Set(float64(bytes))
|
||||
metrics.cacheEntries.Set(float64(n))
|
||||
})
|
||||
r.poller = scrape.NewPoller(reg, scrape.Config{
|
||||
Interval: cfg.Router.ScrapeInterval,
|
||||
Timeout: cfg.Router.ScrapeTimeout,
|
||||
@@ -772,6 +779,9 @@ type dispatchResult struct {
|
||||
// the candidate, and onCanary whether the candidate served it.
|
||||
canaryOf, canary string
|
||||
onCanary bool
|
||||
// cached is an answer from the project's answer cache, which no model
|
||||
// was asked for.
|
||||
cached bool
|
||||
}
|
||||
|
||||
type servedModel struct {
|
||||
|
||||
@@ -28,6 +28,11 @@ type Mapping struct {
|
||||
PromptTokens string
|
||||
GenerationTokens string
|
||||
SuccessTotal string
|
||||
|
||||
// Path is where the engine serves these metrics, when it is not the
|
||||
// poller's configured path: trtllm-serve's /metrics is JSON, and its
|
||||
// Prometheus text is under /prometheus/metrics.
|
||||
Path string
|
||||
}
|
||||
|
||||
// MappingFor returns the metric names for an engine.
|
||||
@@ -41,6 +46,8 @@ func MappingFor(engine string) Mapping {
|
||||
switch engine {
|
||||
case "llamacpp":
|
||||
return LlamaCppMapping()
|
||||
case "trtllm":
|
||||
return TRTLLMMapping()
|
||||
default:
|
||||
return VLLMMapping()
|
||||
}
|
||||
@@ -72,6 +79,33 @@ func LlamaCppMapping() Mapping {
|
||||
}
|
||||
}
|
||||
|
||||
// TRTLLMMapping returns TensorRT-LLM's metric names, as trtllm-serve
|
||||
// publishes them under /prometheus/metrics.
|
||||
//
|
||||
// They are there only when the server runs with return_perf_metrics, and
|
||||
// running and waiting move only with enable_iter_perf_stats; llmgw's launcher
|
||||
// turns both on. The gauges are set from each iteration's statistics, so
|
||||
// they say what the engine was doing when it last iterated: an engine that
|
||||
// has gone idle keeps showing its last batch until the next request moves it.
|
||||
// Running and waiting first appeared in the 1.3 release candidates; an older
|
||||
// server serves none of the three and is reported as breaking the contract,
|
||||
// which is what it does.
|
||||
//
|
||||
// KV cache utilization is used blocks over the pool, a fraction like vLLM's.
|
||||
// The cache's configuration is published under other label names than the
|
||||
// block counts read from vLLM, so it is left unread.
|
||||
func TRTLLMMapping() Mapping {
|
||||
return Mapping{
|
||||
Running: "trtllm_num_requests_running",
|
||||
Waiting: "trtllm_num_requests_waiting",
|
||||
KVCacheUsage: "trtllm_kv_cache_utilization",
|
||||
PromptTokens: "trtllm_prompt_tokens_total",
|
||||
GenerationTokens: "trtllm_generation_tokens_total",
|
||||
SuccessTotal: "trtllm_request_success_total",
|
||||
Path: "/prometheus/metrics",
|
||||
}
|
||||
}
|
||||
|
||||
// VLLMMapping returns vLLM's current metric names.
|
||||
func VLLMMapping() Mapping {
|
||||
return Mapping{
|
||||
|
||||
@@ -208,11 +208,16 @@ func (p *Poller) fetch(ctx context.Context, base, apiKey string, mapping Mapping
|
||||
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+p.cfg.Path, nil)
|
||||
path := p.cfg.Path
|
||||
if mapping.Path != "" {
|
||||
path = mapping.Path
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+path, nil)
|
||||
if err != nil {
|
||||
return endpoint.Stats{}, err
|
||||
}
|
||||
// llama.cpp guards /metrics with its key; vLLM does not, and ignores one.
|
||||
// llama.cpp guards /metrics with its key, and llmgw's TensorRT-LLM
|
||||
// launcher guards everything but /health; vLLM does not, and ignores one.
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package scrape
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
||||
)
|
||||
|
||||
// trtllmMetrics is what trtllm-serve's /prometheus/metrics serves, in the
|
||||
// shape prometheus_client's multiprocess collector writes it: every family
|
||||
// labelled with the model and the backend, gauges with the pid of the
|
||||
// process that set them, counters named with their _total. Trimmed to the
|
||||
// families that matter and one histogram, which the parser has to step over.
|
||||
const trtllmMetrics = `# HELP trtllm_request_success_total Count of successfully processed requests.
|
||||
# TYPE trtllm_request_success_total counter
|
||||
trtllm_request_success_total{engine_type="pytorch",finished_reason="stop",model_name="llama-3.1-70b"} 40.0
|
||||
trtllm_request_success_total{engine_type="pytorch",finished_reason="length",model_name="llama-3.1-70b"} 2.0
|
||||
# HELP trtllm_e2e_request_latency_seconds Histogram of end to end request latency in seconds.
|
||||
# TYPE trtllm_e2e_request_latency_seconds histogram
|
||||
trtllm_e2e_request_latency_seconds_bucket{engine_type="pytorch",le="0.5",model_name="llama-3.1-70b"} 10.0
|
||||
trtllm_e2e_request_latency_seconds_bucket{engine_type="pytorch",le="+Inf",model_name="llama-3.1-70b"} 42.0
|
||||
trtllm_e2e_request_latency_seconds_count{engine_type="pytorch",model_name="llama-3.1-70b"} 42.0
|
||||
trtllm_e2e_request_latency_seconds_sum{engine_type="pytorch",model_name="llama-3.1-70b"} 61.5
|
||||
# HELP trtllm_prompt_tokens_total Number of prefill tokens processed.
|
||||
# TYPE trtllm_prompt_tokens_total counter
|
||||
trtllm_prompt_tokens_total{engine_type="pytorch",model_name="llama-3.1-70b"} 18234.0
|
||||
# HELP trtllm_generation_tokens_total Number of generation tokens processed.
|
||||
# TYPE trtllm_generation_tokens_total counter
|
||||
trtllm_generation_tokens_total{engine_type="pytorch",model_name="llama-3.1-70b"} 9120.0
|
||||
# HELP trtllm_kv_cache_utilization KV cache utilization
|
||||
# TYPE trtllm_kv_cache_utilization gauge
|
||||
trtllm_kv_cache_utilization{engine_type="pytorch",model_name="llama-3.1-70b",pid="4121"} 0.375
|
||||
# HELP trtllm_kv_cache_config_info KV cache configuration info
|
||||
# TYPE trtllm_kv_cache_config_info gauge
|
||||
trtllm_kv_cache_config_info{enable_block_reuse="True",engine_type="pytorch",model_name="llama-3.1-70b",page_size="32",pid="4121"} 1.0
|
||||
# HELP trtllm_num_requests_running Number of active requests
|
||||
# TYPE trtllm_num_requests_running gauge
|
||||
trtllm_num_requests_running{engine_type="pytorch",model_name="llama-3.1-70b",pid="4121"} 5.0
|
||||
# HELP trtllm_num_requests_waiting Number of queued requests
|
||||
# TYPE trtllm_num_requests_waiting gauge
|
||||
trtllm_num_requests_waiting{engine_type="pytorch",model_name="llama-3.1-70b",pid="4121"} 2.0
|
||||
`
|
||||
|
||||
func TestTRTLLMMetricsSatisfyTheContract(t *testing.T) {
|
||||
s, err := Parse(strings.NewReader(trtllmMetrics), MappingFor("trtllm"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.Running != 5 || s.Waiting != 2 {
|
||||
t.Errorf("running=%d waiting=%d", s.Running, s.Waiting)
|
||||
}
|
||||
// A fraction of the pool, as vLLM's is: the scorer compares the two.
|
||||
if s.KVCacheUsage != 0.375 {
|
||||
t.Errorf("kv=%v", s.KVCacheUsage)
|
||||
}
|
||||
if s.PromptTokens != 18234 || s.GenerationTokens != 9120 {
|
||||
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
|
||||
}
|
||||
// Split by finished_reason, summed.
|
||||
if s.SuccessTotal != 42 {
|
||||
t.Errorf("success=%v", s.SuccessTotal)
|
||||
}
|
||||
// Its cache configuration carries no block counts to read.
|
||||
if s.BlockSize != 0 || s.NumGPUBlocks != 0 {
|
||||
t.Errorf("block size %d and blocks %d read from labels that do not hold them", s.BlockSize, s.NumGPUBlocks)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATRTLLMWithoutItsQueueGaugesIsReportedNotIdle(t *testing.T) {
|
||||
// TensorRT-LLM 1.2 publishes the cache's utilization and none of the queue
|
||||
// gauges. Read as zeros, it would look like the emptiest endpoint in the
|
||||
// cluster and take every request.
|
||||
only := `# TYPE trtllm_kv_cache_utilization gauge
|
||||
trtllm_kv_cache_utilization{engine_type="pytorch",model_name="m"} 0.1
|
||||
`
|
||||
var missing *MissingError
|
||||
_, err := Parse(strings.NewReader(only), TRTLLMMapping())
|
||||
if !errors.As(err, &missing) || len(missing.Missing) != 2 {
|
||||
t.Fatalf("expected running and waiting reported missing, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTRTLLMIsScrapedWhereItPublishesPrometheusText(t *testing.T) {
|
||||
// trtllm-serve's /metrics answers JSON — iteration statistics, drained by
|
||||
// each read — and the launcher puts every path but /health behind the key.
|
||||
var gotKey atomic.Value
|
||||
gotKey.Store("")
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/prometheus/metrics":
|
||||
gotKey.Store(r.Header.Get("Authorization"))
|
||||
if r.Header.Get("Authorization") != "Bearer k1" {
|
||||
http.Error(w, "a valid API key is required", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(trtllmMetrics))
|
||||
case "/metrics":
|
||||
_, _ = w.Write([]byte(`[{"iter": 154, "numActiveRequests": 1}]`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
reg := endpoint.NewRegistry(endpoint.Config{})
|
||||
reg.Replace([]endpoint.Info{{ID: "t1", URL: srv.URL, Engine: "trtllm", APIKey: "k1", Models: []string{"m"}}})
|
||||
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 2 * time.Second}, nil)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
p.Sync(ctx)
|
||||
defer p.Stop()
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
for _, snap := range reg.Snapshots("m") {
|
||||
if snap.ID == "t1" && snap.Stats.Running == 5 {
|
||||
if snap.Stats.Waiting != 2 || snap.Stats.KVCacheUsage != 0.375 {
|
||||
t.Errorf("stats read wrong: %+v", snap.Stats)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("the endpoint was never measured (key sent: %q)", gotKey.Load())
|
||||
}
|
||||
Reference in new issue
Block a user