Outside services read answers redacted and whole; a card's own lanes

From the second review.
- An output service reads the answer with the values restore put back
  taken out again (guard.Subject.Restored), as an input service reads the
  request redacted.
- With n above 1, the service reads the first choice once it is whole,
  not at whichever choice finished first.
- Restored values go into an answer asked for as JSON (response_format)
  escaped for it, as into tool-call arguments.
- The embedding failure is logged once a minute through an outage, not
  once.
- A card's own lane count and fastest generation come from the kernel
  (max_link_width, max_link_speed): the driver's width.max is what card
  and slot allow together, so an x16 card in an x8 slot read x8 of x8 and
  was never said.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 13:22:39 +01:00
1 parent 1948a787ae
commit c09d5ad4ee
12 files changed
+220 -30

No files matched your search

+1
View File
@@ -35,6 +35,7 @@ func (p *parsedRequest) guardSubject(model string) guard.Subject {
// can do.
func (r *Router) subject(req *parsedRequest, model string) guard.Subject {
s := req.guardSubject(model)
s.Restored = req.guardIn.Restore
if _, entry := r.resolve(model); entry.Name != "" {
s.Tools, s.Vision = entry.Capabilities.Tools, entry.Capabilities.Vision
}
+19 -3
View File
@@ -22,6 +22,10 @@ import (
type restorer struct {
values map[string]string
longest int
// jsonContent is an answer asked for as JSON (response_format): its
// content is JSON, and a value goes into it escaped, as into a tool
// call's arguments.
jsonContent bool
// held is, per choice and field, the tail of the text so far that may
// be the start of a placeholder the next event completes: "[EMA" before
// "IL_1]". Keyed by heldKey.
@@ -110,7 +114,7 @@ func (r *restorer) walk(v any, key string) any {
x[i] = r.walk(e, key)
}
case string:
return r.text(x, key == "arguments")
return r.text(x, key == "arguments" || (r.jsonContent && key == "content"))
}
return v
}
@@ -164,7 +168,7 @@ func (r *restorer) OnEvent(data string) ([]string, bool) {
}
}
if delta != nil {
swap(delta, "content", heldKey{idx, "content", -1}, false)
swap(delta, "content", heldKey{idx, "content", -1}, r.jsonContent)
swap(delta, "reasoning_content", heldKey{idx, "reasoning_content", -1}, false)
calls, _ := delta["tool_calls"].([]any)
for j, tc := range calls {
@@ -260,11 +264,12 @@ func (c streamChain) OnEnd() []string {
// restoreInto puts the input's placeholders back in this attempt's answer,
// ahead of whatever output filter the attempt already has.
func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
func restoreInto(values map[string]string, stream, jsonContent bool, preq *proxy.Request) {
if len(values) == 0 {
return
}
rs := newRestorer(values)
rs.jsonContent = jsonContent
if f := preq.Filter; f != nil {
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
} else {
@@ -279,3 +284,14 @@ func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
preq.StreamFilter = rs
}
}
// jsonAnswer reports whether a request asks for its answer as JSON.
func jsonAnswer(body []byte) bool {
var b struct {
ResponseFormat *struct {
Type string `json:"type"`
} `json:"response_format"`
}
return json.Unmarshal(body, &b) == nil && b.ResponseFormat != nil &&
(b.ResponseFormat.Type == "json_object" || b.ResponseFormat.Type == "json_schema")
}
+28
View File
@@ -10,6 +10,7 @@ import (
"regexp"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
@@ -218,3 +219,30 @@ func TestAKeptAnswerHoldsNoValueItsRequestHadRedacted(t *testing.T) {
t.Errorf("%q", got)
}
}
func TestAnAnswerAskedForAsJSONGetsItsValuesEscapedForIt(t *testing.T) {
r := newRestorer(map[string]string{"[NAME_1]": `O"Brien`})
r.jsonContent = true
body := r.body([]byte(`{"choices":[{"message":{"content":"{\"name\":\"[NAME_1]\"}"}}]}`))
var whole struct {
Choices []struct{ Message struct{ Content string } }
}
if err := json.Unmarshal(body, &whole); err != nil {
t.Fatal(err)
}
var obj map[string]string
if err := json.Unmarshal([]byte(whole.Choices[0].Message.Content), &obj); err != nil || obj["name"] != `O"Brien` {
t.Errorf("content %q (%v)", whole.Choices[0].Message.Content, err)
}
if !jsonAnswer([]byte(`{"response_format":{"type":"json_schema"}}`)) || jsonAnswer([]byte(`{"model":"m"}`)) {
t.Error("jsonAnswer")
}
}
func TestAFailureIsWarnedOfOnceAMinuteNotOnce(t *testing.T) {
var w lastWarned
now := time.Now()
if !w.Due(now, time.Minute) || w.Due(now.Add(30*time.Second), time.Minute) || !w.Due(now.Add(61*time.Second), time.Minute) {
t.Error("a failure every 30 s is warned of once a minute")
}
}
+8 -5
View File
@@ -1256,7 +1256,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
}
// A filter per attempt, for the model this attempt is served by.
filter := r.outputFilter(uctx, req, model, &preq)
restoreInto(req.guardIn.Restore, req.Stream, &preq)
restoreInto(req.guardIn.Restore, req.Stream, jsonAnswer(req.UpstreamBody), &preq)
var out proxy.Outcome
var perr error
if dec.Endpoint.Engine == endpoint.EngineComfyUI {
@@ -1387,11 +1387,14 @@ type lastWarned struct {
at time.Time
}
// Swap records now and says when it was before.
func (w *lastWarned) Swap(now time.Time) time.Time {
// Due says whether every has passed since it was last said, and if so
// records now as that time: once a minute through an outage, not once.
func (w *lastWarned) Due(now time.Time, every time.Duration) bool {
w.mu.Lock()
defer w.mu.Unlock()
was := w.at
if now.Sub(w.at) < every {
return false
}
w.at = now
return was
return true
}
+1 -1
View File
@@ -54,7 +54,7 @@ func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model st
vec, err := r.embedQuestion(ctx, question, req.inCluster)
if err != nil {
// Once a minute at most: down, the model fails every miss.
if now := time.Now(); now.Sub(r.embedWarned.Swap(now)) > time.Minute {
if r.embedWarned.Due(time.Now(), time.Minute) {
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
"model", r.cacheEmbedModel, "err", err)
}