A judge the leader serves by itself is sized as init's is
A judge a policy names and the catalog lacked was served with 3 GB, no cores and llama.cpp's prompt cache; on a busy node its judgments missed their budget and fail: closed refused the requests. It now gets init's 2 GB and cores, and every judge modelserve serves runs with --cache-ram 0. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
99c34fd68f
commit
8b122aae09
8 files changed
+70
-3
No files matched your search
@@ -163,6 +163,10 @@ type Request struct {
|
||||
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||
// node it fits on.
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
// MemoryGB is what a replica on CPUs reserves, in place of the figure
|
||||
// worked out from its weights and context: a judge's, whose own sizing
|
||||
// is known.
|
||||
MemoryGB int `json:"memory_gb,omitempty"`
|
||||
// By is who asked, for the catalog's record.
|
||||
By string `json:"-"`
|
||||
}
|
||||
@@ -518,6 +522,9 @@ func (s *Server) run(j *Job) {
|
||||
memGB = int(gb(set.Bytes)*1.5) + 2
|
||||
basis = "weights plus half again plus two; the header gave no shape"
|
||||
}
|
||||
if req.MemoryGB > 0 {
|
||||
memGB, basis = req.MemoryGB, "as asked, in place of "+basis
|
||||
}
|
||||
s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
|
||||
|
||||
cores := req.CPUCores
|
||||
@@ -631,6 +638,12 @@ func llamaArgs(req Request) []string {
|
||||
n := strconv.Itoa(req.ContextLength)
|
||||
args = append([]string{"--embedding"}, append(args, "--batch-size", n, "--ubatch-size", n)...)
|
||||
}
|
||||
if req.GuardFormat != "" {
|
||||
// No prompt cache: a judge's prompts, each a different text, are
|
||||
// not asked again, and llama.cpp's default of up to 8 GB grew one
|
||||
// judge to 3.2 GB over a day.
|
||||
args = append(args, "--cache-ram", "0")
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
|
||||
@@ -428,3 +428,27 @@ func TestAPoolTheCardCannotHoldIsShrunkNotStarted(t *testing.T) {
|
||||
t.Errorf("a pool that fits was changed: %+v %q", got, why)
|
||||
}
|
||||
}
|
||||
|
||||
// A judge is sized as asked, not from its weights, and runs with no prompt
|
||||
// cache: what `server init` gives its own.
|
||||
func TestAJudgeIsServedWithTheMemoryAskedAndNoPromptCache(t *testing.T) {
|
||||
files := []hf.File{{Path: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", Bytes: 600 << 20, SHA256: "aa"}}
|
||||
s, _, c, _ := setup(t, files, nil)
|
||||
job, err := s.Start(Request{Repo: "example/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B-Q8_0.gguf",
|
||||
GuardFormat: "qwen3guard", CPU: true, CPUCores: 2, MemoryGB: 2, Replicas: 2, CtxPool: 8192, ContextLength: 4096})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wait(t, s, job.ID)
|
||||
if len(c.added) != 1 {
|
||||
t.Fatalf("added %+v", c.added)
|
||||
}
|
||||
p := c.added[0].Profiles[0]
|
||||
if p.MinMemoryGB != 2 || p.CPUCores != 2 || !strings.Contains(strings.Join(p.Args, " "), "--cache-ram 0") {
|
||||
t.Errorf("profile %+v", p)
|
||||
}
|
||||
// Any other model keeps llama.cpp's cache.
|
||||
if strings.Contains(strings.Join(llamaArgs(Request{CtxPool: 8192, ContextLength: 4096}), " "), "cache-ram") {
|
||||
t.Error("a model that is not a judge lost its prompt cache")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user