From 91fe24a1058aa7a8db71add310df8c82e1df0d65 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 30 Sep 2026 02:25:07 +0200 Subject: [PATCH] Close what a second security review found still open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review of v0.10.4-rc.4, and a look at the live network, found: - Alertmanager listening on every interface: from any node its API took silences, with no login. Bound to loopback on the monitoring host, and the monitoring README now installs it that way; Prometheus beside it is its only client, and one Alertmanager needs no cluster port. - Login and signup unbounded in memory. Each attempt runs argon2 with 64 MiB, and nothing limited how many ran at once; the per-address throttle kept one entry per address ever tried. Hashing now takes one of a few slots and waits for one otherwise. The throttle expires entries whose window has passed when it fills, and if it is still full it refuses new addresses: evicting live ones instead would let a flood of made-up addresses reset the throttle on the address actually being guessed. - Metric labels taken from the request. A refused request counted the model name it asked for, so any key holder could mint a series per request. Requests are counted under the catalog's name, or "unknown". - Two in code written this week. A SQLite backup made its snapshot with the default mode in the listable backup directory, a full copy of the store readable while the backup ran; it is now made in a private directory, and the unit sets UMask=0077. And archive links were checked by name only, so "a -> ." then "a/a/b -> ../.." passed and a file written through them landed above the install directory, as root. Archives now unpack through os.Root, and a link may only point at a file beside it — which every llama.cpp b10991 archive, CPU and CUDA, was checked to follow. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 22 ++++++++++++ deploy/monitoring/README.md | 11 ++++-- deploy/systemd/llmgw-backup.service | 3 ++ internal/account/password.go | 18 ++++++++-- internal/account/service.go | 20 +++++++++++ internal/account/throttle_bound_test.go | 33 +++++++++++++++++ internal/agent/provision/install.go | 42 +++++++++++++--------- internal/agent/provision/install_test.go | 46 ++++++++++++++++++++++++ internal/cli/backup.go | 14 ++++---- internal/cli/backup_test.go | 23 ++++++++++++ internal/router/handlers.go | 16 ++++----- internal/router/metrics.go | 14 ++++++++ internal/router/router_test.go | 30 ++++++++++++++++ 13 files changed, 258 insertions(+), 34 deletions(-) create mode 100644 internal/account/throttle_bound_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da36a1..f33ec42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,28 @@ more than 26 hours old or there has never been one. docs/install.md now says how to restore, including why running engines survive it: the engine-key secret comes back with the store. +### Security + +- **Alertmanager answered anyone on the network**, including requests to + silence alerts. The monitoring README now binds it to loopback, where + Prometheus beside it is its only client. +- **Login and signup could be made to use gigabytes.** Each attempt hashes with + 64 MiB of argon2, with no bound on how many ran at once, and the per-address + throttle grew by one entry per address tried. Hashes now run a few at a time + and the rest wait; the throttle expires old entries, and when it is still + full it refuses new addresses rather than dropping live ones — which would + have let a flood of made-up addresses reset the throttle on a real one. +- **Any key holder could grow Prometheus without bound** by asking for made-up + model names, each of which became a metric label. Requests are counted under + the catalog's name for their model, or "unknown". +- **A SQLite backup left a full copy of the store readable** by local users + while it ran. The snapshot is now made in a private directory, and + llmgw-backup.service sets `UMask=0077`. +- **An engine archive could write outside its directory through chained + links.** Archives are unpacked through `os.Root`, and a link may only point + at a file beside it. Every llama.cpp b10991 archive, CPU and CUDA, already + follows that rule. + ### Fixes - Tests that slept a fixed time and then asserted now wait for their diff --git a/deploy/monitoring/README.md b/deploy/monitoring/README.md index 3947d74..ea7a752 100644 --- a/deploy/monitoring/README.md +++ b/deploy/monitoring/README.md @@ -46,6 +46,12 @@ apt-get install prometheus prometheus-alertmanager install -m 0644 prometheus.yml /etc/prometheus/prometheus.yml install -m 0644 llmgw-rules.yml /etc/prometheus/rules/llmgw.yml install -m 0644 alertmanager.yml /etc/prometheus/alertmanager.yml +# Alertmanager answers on loopback only. Its API takes silences from anyone who +# can reach it, with no login, and silencing BackupTooOld or ModelParked is how +# something that should be noticed is not. Prometheus beside it is its only +# client; one Alertmanager needs no cluster port. +echo 'ARGS="--web.listen-address=127.0.0.1:9093 --cluster.listen-address="' \ + > /etc/default/prometheus-alertmanager promtool check config /etc/prometheus/prometheus.yml systemctl restart prometheus prometheus-alertmanager @@ -75,7 +81,8 @@ Alertmanager groups and holds alerts and has no receiver. Where an alert should go — an address, a Slack workspace, a webhook — is a decision with credentials attached, and a monitoring system that quietly starts mailing people is worse than one that is visibly not wired up. Alerts are -visible at `:9093` and in Prometheus; add a receiver to `alertmanager.yml` to +visible in Prometheus, and at `:9093` on the monitoring host itself (`ssh -L +9093:127.0.0.1:9093 ` to open its page); add a receiver to `alertmanager.yml` to make them travel. `critical` already routes separately so it can be pointed somewhere else without touching the rest. @@ -86,7 +93,7 @@ An alerting system nobody has seen fire is an assumption. ``` llmgw ctl cordon # work already there keeps running # ~2 minutes later -curl -s http://:9093/api/v2/alerts | jq '.[].labels' +ssh curl -s http://127.0.0.1:9093/api/v2/alerts | jq '.[].labels' llmgw ctl uncordon # the alert resolves on its own ``` diff --git a/deploy/systemd/llmgw-backup.service b/deploy/systemd/llmgw-backup.service index c830afa..c918ee7 100644 --- a/deploy/systemd/llmgw-backup.service +++ b/deploy/systemd/llmgw-backup.service @@ -4,6 +4,9 @@ After=llmgw-server.service postgresql.service [Service] Type=oneshot +# Nothing it writes is for anyone else: the archive holds the CA's key and a +# copy of the store. +UMask=0077 # Root, because the archive includes /etc/llmgw/secrets and the certificate # authority's key, which the llmgw user deliberately cannot read all of. ExecStart=/usr/local/bin/llmgw server backup --config /etc/llmgw/server.yaml --keep 14 diff --git a/internal/account/password.go b/internal/account/password.go index e6e0c17..f19be13 100644 --- a/internal/account/password.go +++ b/internal/account/password.go @@ -15,6 +15,7 @@ import ( "encoding/base64" "errors" "fmt" + "runtime" "strings" "golang.org/x/crypto/argon2" @@ -56,7 +57,7 @@ func HashPassword(password string) (string, error) { if _, err := rand.Read(salt); err != nil { return "", err } - sum := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) + sum := idKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) b64 := base64.RawStdEncoding.EncodeToString return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", @@ -93,7 +94,7 @@ func VerifyPassword(password, encoded string) (bool, error) { return false, ErrBadHash } - got := argon2.IDKey([]byte(password), salt, time, memory, threads, uint32(len(want))) + got := idKey([]byte(password), salt, time, memory, threads, uint32(len(want))) return subtle.ConstantTimeCompare(got, want) == 1, nil } @@ -110,3 +111,16 @@ func init() { } dummyHash = h } + +// argonSlots bounds how many hashes run at once. Each takes 64 MiB, and the +// login and signup forms are on the client-facing port: without a bound, a +// few hundred parallel attempts at random addresses asked for gigabytes at +// once, and the per-address throttle does not see them — each address is new. +// Beyond the bound, attempts wait their turn instead. +var argonSlots = make(chan struct{}, max(2, runtime.NumCPU()/2)) + +func idKey(password, salt []byte, time, memory uint32, threads uint8, keyLen uint32) []byte { + argonSlots <- struct{}{} + defer func() { <-argonSlots }() + return argon2.IDKey(password, salt, time, memory, threads, keyLen) +} diff --git a/internal/account/service.go b/internal/account/service.go index 2bfc3d4..2c5e345 100644 --- a/internal/account/service.go +++ b/internal/account/service.go @@ -230,6 +230,22 @@ func (s *Service) allowAttempt(email string) bool { defer s.mu.Unlock() w, ok := s.attempts[email] + if !ok && len(s.attempts) >= maxTrackedAddresses { + // Full of addresses nobody has signed in as for a while: forget the + // ones whose window has passed. Only expiring, never evicting early — + // dropping a live window is how an attacker spraying random addresses + // would buy unlimited guesses at the one they care about. + for k, v := range s.attempts { + if now.Sub(v.since) > s.cfg.AttemptWindow { + delete(s.attempts, k) + } + } + if len(s.attempts) >= maxTrackedAddresses { + // Still full, so somebody is spraying addresses faster than they + // expire. New ones wait; those already tracked are unaffected. + return false + } + } if !ok || now.Sub(w.since) > s.cfg.AttemptWindow { s.attempts[email] = &attemptWindow{count: 1, since: now} return true @@ -238,6 +254,10 @@ func (s *Service) allowAttempt(email string) bool { return w.count <= s.cfg.MaxAttempts } +// maxTrackedAddresses bounds the throttle's memory. Unbounded, it grew by one +// entry per address tried and shrank only on a successful sign-in. +const maxTrackedAddresses = 50000 + func (s *Service) clearAttempts(email string) { s.mu.Lock() delete(s.attempts, email) diff --git a/internal/account/throttle_bound_test.go b/internal/account/throttle_bound_test.go new file mode 100644 index 0000000..11796b1 --- /dev/null +++ b/internal/account/throttle_bound_test.go @@ -0,0 +1,33 @@ +package account + +import ( + "fmt" + "testing" + "time" +) + +func TestSprayingAddressesCannotGrowTheThrottleOrEscapeIt(t *testing.T) { + now := time.Unix(0, 0) + s := &Service{cfg: DefaultConfig(), attempts: map[string]*attemptWindow{}, now: func() time.Time { return now }} + + // The target is being guessed at. + for i := 0; i < s.cfg.MaxAttempts; i++ { + s.allowAttempt("victim@example.com") + } + // Then the table is flooded with made-up addresses. + for i := 0; i < maxTrackedAddresses+100; i++ { + s.allowAttempt(fmt.Sprintf("x%d@example.com", i)) + } + if n := len(s.attempts); n > maxTrackedAddresses { + t.Errorf("the table grew to %d", n) + } + if s.allowAttempt("victim@example.com") { + t.Error("flooding the table reset the target's throttle") + } + + // Once the window passes, everything expires and sign-ins work again. + now = now.Add(s.cfg.AttemptWindow + time.Second) + if !s.allowAttempt("new@example.com") { + t.Error("a new address was refused after the flood expired") + } +} diff --git a/internal/agent/provision/install.go b/internal/agent/provision/install.go index e843d7d..c319118 100644 --- a/internal/agent/provision/install.go +++ b/internal/agent/provision/install.go @@ -523,6 +523,11 @@ func (in *Installer) download(ctx context.Context, repo, tag, asset, dst, want s // untarGz unpacks an archive under dir, refusing anything that would land // outside it. A release archive is somebody else's file, and a path of // "../../etc/cron.d/x" in it would otherwise be written as root. +// +// Every write goes through an os.Root, which refuses to leave dir however the +// path gets there. Checking names alone was not enough: "a -> ." followed by +// "a/a/b -> ../.." reads as inside and resolves two levels above it, and a +// file written through that lands wherever the links point. func untarGz(archive, dir string) error { f, err := os.Open(archive) //nolint:gosec // inside the install root if err != nil { @@ -533,6 +538,11 @@ func untarGz(archive, dir string) error { if err != nil { return err } + root, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer root.Close() tr := tar.NewReader(gz) for { h, err := tr.Next() @@ -542,22 +552,22 @@ func untarGz(archive, dir string) error { if err != nil { return err } - target, err := within(dir, h.Name) - if err != nil { + name := filepath.Clean(h.Name) + if _, err := within(dir, name); err != nil { return err } switch h.Typeflag { case tar.TypeDir: - if err := os.MkdirAll(target, 0o755); err != nil { + if err := root.MkdirAll(name, 0o755); err != nil { return err } case tar.TypeReg: - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil { return err } - out, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) //nolint:gosec // checked above + out, err := root.OpenFile(name, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, fs.FileMode(h.Mode)&0o755) if err != nil { - return err + return fmt.Errorf("%s: %w", h.Name, err) } if _, err := io.Copy(out, tr); err != nil { //nolint:gosec // a release archive, size bounded by its checksum out.Close() @@ -567,19 +577,19 @@ func untarGz(archive, dir string) error { return err } case tar.TypeSymlink: - // Libraries are versioned through symlinks, and those are - // relative. One pointing anywhere else is refused. - if filepath.IsAbs(h.Linkname) { - return fmt.Errorf("%s links outside the archive", h.Name) + // Libraries are versioned through symlinks to a file beside them — + // libllama.so -> libllama.so.0 — and that is all a release needs. + // A target with a separator in it could point anywhere once other + // links are followed, and a binary found through one would be run + // as root, so it is refused rather than reasoned about. + if h.Linkname == "" || strings.ContainsAny(h.Linkname, `/\`) || h.Linkname == ".." || h.Linkname == "." { + return fmt.Errorf("%s links to %q; only a file beside it is allowed", h.Name, h.Linkname) } - if _, err := within(dir, filepath.Join(filepath.Dir(h.Name), h.Linkname)); err != nil { - return fmt.Errorf("%s links outside the archive", h.Name) - } - if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + if err := root.MkdirAll(filepath.Dir(name), 0o755); err != nil { return err } - _ = os.Remove(target) - if err := os.Symlink(h.Linkname, target); err != nil { + _ = root.Remove(name) + if err := root.Symlink(h.Linkname, name); err != nil { return err } default: diff --git a/internal/agent/provision/install_test.go b/internal/agent/provision/install_test.go index f8453f2..5aefcb4 100644 --- a/internal/agent/provision/install_test.go +++ b/internal/agent/provision/install_test.go @@ -342,3 +342,49 @@ func TestAVersionThatIsAPathIsRefused(t *testing.T) { } } } + +func TestChainedLinksCannotCarryAFileOutOfTheInstall(t *testing.T) { + // Each link reads as inside the directory; together they point above it. + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + _ = tw.WriteHeader(&tar.Header{Name: "a", Linkname: ".", Typeflag: tar.TypeSymlink}) + _ = tw.WriteHeader(&tar.Header{Name: "a/a/b", Linkname: "../..", Typeflag: tar.TypeSymlink}) + _ = tw.WriteHeader(&tar.Header{Name: "a/a/b/escaped", Mode: 0o644, Size: 1, Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte("x")) + _ = tw.Close() + _ = gz.Close() + + parent := t.TempDir() + dir := filepath.Join(parent, "install") + _ = os.MkdirAll(dir, 0o755) + archive := filepath.Join(parent, "evil.tar.gz") + _ = os.WriteFile(archive, buf.Bytes(), 0o644) + + if err := untarGz(archive, dir); err == nil { + t.Error("an archive of chained links was unpacked") + } + if _, err := os.Stat(filepath.Join(parent, "escaped")); err == nil { + t.Fatal("a file was written outside the install directory") + } +} + +func TestALibrarysVersionLinkIsStillUnpacked(t *testing.T) { + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + _ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so.0", Mode: 0o755, Size: 1, Typeflag: tar.TypeReg}) + _, _ = tw.Write([]byte("x")) + _ = tw.WriteHeader(&tar.Header{Name: "lib/libllama.so", Linkname: "libllama.so.0", Typeflag: tar.TypeSymlink}) + _ = tw.Close() + _ = gz.Close() + dir := t.TempDir() + archive := filepath.Join(t.TempDir(), "ok.tar.gz") + _ = os.WriteFile(archive, buf.Bytes(), 0o644) + if err := untarGz(archive, dir); err != nil { + t.Fatal(err) + } + if target, err := os.Readlink(filepath.Join(dir, "lib", "libllama.so")); err != nil || target != "libllama.so.0" { + t.Errorf("link %q %v", target, err) + } +} diff --git a/internal/cli/backup.go b/internal/cli/backup.go index 25fbdef..1a0f354 100644 --- a/internal/cli/backup.go +++ b/internal/cli/backup.go @@ -141,13 +141,17 @@ func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error { if err != nil { return err } - tmp, err := os.CreateTemp(scratch, ".store-*") + // Inside a directory only its owner can enter. The archive directory is + // listable on purpose, and VACUUM INTO creates its file with the default + // mode: made there directly, a full copy of the store — password hashes, + // the engine-key secret — sat readable by any local user for as long as + // the backup took. + tmpDir, err := os.MkdirTemp(scratch, ".store-") if err != nil { return err } - tmpPath := tmp.Name() - _ = tmp.Close() - defer os.Remove(tmpPath) + defer os.RemoveAll(tmpDir) + tmpPath := filepath.Join(tmpDir, "store") name := "store.sqlite" if store.IsPostgres(addr) { @@ -156,8 +160,6 @@ func addStore(ctx context.Context, tw *tar.Writer, addr, scratch string) error { return err } } else { - // VACUUM INTO refuses a file that exists. - _ = os.Remove(tmpPath) db, err := store.Open(ctx, addr) if err != nil { return err diff --git a/internal/cli/backup_test.go b/internal/cli/backup_test.go index 0ec1a18..d1f0a7a 100644 --- a/internal/cli/backup_test.go +++ b/internal/cli/backup_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "time" @@ -116,3 +117,25 @@ func TestABackupDirectoryTheControlPlaneCannotListIsSaid(t *testing.T) { t.Errorf("a listable directory: %s", why) } } + +func TestNoScratchCopyOfTheStoreIsLeftBehind(t *testing.T) { + // The snapshot is made before it is archived. Made straight into the + // listable archive directory, it was readable by anyone for that time. + tmp := t.TempDir() + dbPath := filepath.Join(tmp, "store.db") + db, err := store.Open(t.Context(), dbPath) + if err != nil { + t.Fatal(err) + } + _ = db.Close() + out := filepath.Join(tmp, "backups") + if _, err := writeBackup(t.Context(), dbPath, t.TempDir(), out); err != nil { + t.Fatal(err) + } + entries, _ := os.ReadDir(out) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".store") { + t.Errorf("a scratch copy of the store was left behind: %s", e.Name()) + } + } +} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index f71a526..241356c 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -51,7 +51,7 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): // The client gave up while queued; nobody is listening. r.m.observeAdmissionRejected(req.Band, "client_gone") - r.m.observeRequest(req.Model, "", 499, time.Since(start), nil, 0) + r.m.observeRequest(r.metricModel(req.Model), "", 499, time.Since(start), nil, 0) return case errors.Is(err, admission.ErrQueueTimeout): status = http.StatusGatewayTimeout @@ -67,7 +67,7 @@ func (r *Router) rejectAdmission(w http.ResponseWriter, req *parsedRequest, err } r.m.observeAdmissionRejected(req.Band, reason) writeError(w, status, "server_error", err.Error()) - r.m.observeRequest(req.Model, "", status, time.Since(start), nil, 0) + r.m.observeRequest(r.metricModel(req.Model), "", status, time.Since(start), nil, 0) } // prefixBytes is how much of the prompt is kept for prefix-aware routing. @@ -289,7 +289,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { if _, entry := r.resolve(parsed.Model); entry.ContextLength > 0 { if why := checkContextFits(entry, parsed.PromptTokens, parsed.MaxTokens); why != "" { writeError(w, http.StatusBadRequest, "invalid_request_error", why) - r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0) + r.m.observeRequest(r.metricModel(parsed.Model), "", http.StatusBadRequest, time.Since(start), nil, 0) return } } @@ -297,7 +297,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { if p, ok := auth.FromContext(req.Context()); ok { if !r.permits(p, parsed.Model) { auth.WriteForbiddenModel(w, parsed.Model) - r.m.observeRequest(parsed.Model, "", http.StatusForbidden, time.Since(start), nil, 0) + r.m.observeRequest(r.metricModel(parsed.Model), "", http.StatusForbidden, time.Since(start), nil, 0) return } // Limits are checked before admission, not after: a request that is @@ -339,7 +339,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { } defer lease.Release() queued := time.Since(start) - r.m.observeAdmission(lease.Band(), parsed.Model, queued) + r.m.observeAdmission(lease.Band(), r.metricModel(parsed.Model), queued) if queued > time.Millisecond { w.Header().Set("x-llmgw-queue-ms", strconv.FormatInt(queued.Milliseconds(), 10)) } @@ -349,7 +349,7 @@ func (r *Router) handleInference(path string) http.HandlerFunc { // model that will actually serve, not the one that was asked for. if dropped, err := r.loadThread(req.Context(), parsed); err != nil { writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error()) - r.m.observeRequest(parsed.Model, "", http.StatusBadRequest, time.Since(start), nil, 0) + r.m.observeRequest(r.metricModel(parsed.Model), "", http.StatusBadRequest, time.Since(start), nil, 0) return } else if dropped > 0 { // Dropping turns is the right answer and must not be a silent one. @@ -420,7 +420,7 @@ func (r *Router) finish(w http.ResponseWriter, req *parsedRequest, res dispatchR writeError(w, status, "server_error", res.err.Error()) } - r.m.observeRequest(req.Model, res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) + r.m.observeRequest(r.metricModel(req.Model), res.pool, status, time.Since(start), res.outcome.Usage, res.outcome.TTFT) r.account(req, res, status, time.Since(start)) } @@ -467,7 +467,7 @@ func (r *Router) rejectLimit(w http.ResponseWriter, req *parsedRequest, err erro // already know how to surface that code. What differs is Retry-After, // which is absent when waiting does not help. writeError(w, http.StatusTooManyRequests, "rate_limit_exceeded", le.Error()) - r.m.observeRequest(req.Model, "", http.StatusTooManyRequests, time.Since(start), nil, 0) + r.m.observeRequest(r.metricModel(req.Model), "", http.StatusTooManyRequests, time.Since(start), nil, 0) } // parse reads the body and extracts the routing-relevant fields. diff --git a/internal/router/metrics.go b/internal/router/metrics.go index 9565e8c..79ef1e1 100644 --- a/internal/router/metrics.go +++ b/internal/router/metrics.go @@ -151,3 +151,17 @@ func (m *metrics) observeScores(d scheduler.Decision) { } m.considered.Observe(float64(d.Considered)) } + +// metricModel is the model label a request is counted under: the catalog's +// name for it, or "unknown". Taken as sent, any key holder could mint a new +// series per request — a refused request counted its made-up model name — +// and grow Prometheus without bound. +func (r *Router) metricModel(name string) string { + if name == "" { + return "" + } + if m, ok := r.modelMap()[name]; ok { + return m.Name + } + return "unknown" +} diff --git a/internal/router/router_test.go b/internal/router/router_test.go index 62285dc..b245539 100644 --- a/internal/router/router_test.go +++ b/internal/router/router_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/prometheus/client_golang/prometheus" "io" "log/slog" "net/http" @@ -655,3 +656,32 @@ func TestAModelThatCannotComeUpIsSaidSoAtOnce(t *testing.T) { t.Errorf("the reason was not given: %s", rec.Body.String()) } } + +func TestAMadeUpModelNameDoesNotBecomeAMetricLabel(t *testing.T) { + // Counted as sent, every request for a nonsense model minted a new series. + e := newFakeEngine(t, "m") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"m"}}) + cfg.Models = []catalog.Model{{Name: "m", Aliases: []string{"mm"}}} + rt := newTestRouter(t, cfg) + + if got := rt.metricModel("mm"); got != "m" { + t.Errorf("an alias is counted as %q, not the model it names", got) + } + for i := 0; i < 20; i++ { + post(t, rt.Handler(), map[string]any{"model": fmt.Sprintf("junk-%d", i), "messages": []any{}}, nil) + } + // Twenty names, one series: all of them "unknown". + reg := prometheus.NewRegistry() + reg.MustRegister(rt.m.requests) + mfs, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + series := 0 + for _, mf := range mfs { + series += len(mf.GetMetric()) + } + if series != 1 { + t.Errorf("%d series after twenty made-up names", series) + } +}