A judge the leader serves by itself is sized as init's is
CI / test (push) Successful in 5m13s

A judge a policy names and the catalog lacked was served with 3 GB, no
cores and llama.cpp's prompt cache; on a busy node its judgments missed
their budget and fail: closed refused the requests. It now gets init's
2 GB and cores, and every judge modelserve serves runs with
--cache-ram 0.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 16:32:04 +02:00
1 parent 99c34fd68f
commit 8b122aae09
8 files changed
+70 -3

No files matched your search

+13
View File
@@ -163,6 +163,10 @@ type Request struct {
// CPUCores is what a replica computes on; zero takes half the smallest
// node it fits on.
CPUCores int `json:"cpu_cores,omitempty"`
// MemoryGB is what a replica on CPUs reserves, in place of the figure
// worked out from its weights and context: a judge's, whose own sizing
// is known.
MemoryGB int `json:"memory_gb,omitempty"`
// By is who asked, for the catalog's record.
By string `json:"-"`
}
@@ -518,6 +522,9 @@ func (s *Server) run(j *Job) {
memGB = int(gb(set.Bytes)*1.5) + 2
basis = "weights plus half again plus two; the header gave no shape"
}
if req.MemoryGB > 0 {
memGB, basis = req.MemoryGB, "as asked, in place of "+basis
}
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
cores := req.CPUCores
@@ -631,6 +638,12 @@ func llamaArgs(req Request) []string {
n := strconv.Itoa(req.ContextLength)
args = append([]string{"--embedding"}, append(args, "--batch-size", n, "--ubatch-size", n)...)
}
if req.GuardFormat != "" {
// No prompt cache: a judge's prompts, each a different text, are
// not asked again, and llama.cpp's default of up to 8 GB grew one
// judge to 3.2 GB over a day.
args = append(args, "--cache-ram", "0")
}
return args
}