Outside services read answers redacted and whole; a card's own lanes
CI / test (push) Successful in 9m16s

From the second review.
- An output service reads the answer with the values restore put back
  taken out again (guard.Subject.Restored), as an input service reads the
  request redacted.
- With n above 1, the service reads the first choice once it is whole,
  not at whichever choice finished first.
- Restored values go into an answer asked for as JSON (response_format)
  escaped for it, as into tool-call arguments.
- The embedding failure is logged once a minute through an outage, not
  once.
- A card's own lane count and fastest generation come from the kernel
  (max_link_width, max_link_speed): the driver's width.max is what card
  and slot allow together, so an x16 card in an x8 slot read x8 of x8 and
  was never said.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-29 13:22:39 +01:00
1 parent 1948a787ae
commit c09d5ad4ee
12 files changed
+220 -30

No files matched your search

+4
View File
@@ -54,6 +54,10 @@ type Subject struct {
Stream bool
// Tools and Vision are what the model's catalog entry says it can do.
Tools, Vision bool
// Restored are the values the request's redaction took out, by
// placeholder, which restore puts back in the answer: outside services
// reading the answer are not told them.
Restored map[string]string
}
// Verdict is a judgement on one stage of one request.
+37 -19
View File
@@ -68,7 +68,7 @@ func (g *Gate) Output(ctx context.Context, s Subject) OutputFilter {
return nil
}
memo := g.tools.get(s.RequestID)
return &outFilter{ctx: ctx, g: g, p: p, names: names, shadow: p.Mode == "shadow",
return &outFilter{ctx: ctx, g: g, p: p, names: names, shadow: p.Mode == "shadow", restored: s.Restored,
schemas: memo.schemas, answer: memo.answer, user: memo.user, choices: map[int]*choiceState{}}
}
@@ -76,21 +76,24 @@ func (g *Gate) Output(ctx context.Context, s Subject) OutputFilter {
type outFilter struct {
// servicesAsked says the outside services have read the answer.
servicesAsked bool
ctx context.Context
g *Gate
p Policy
names string
shadow bool
schemas map[string]json.RawMessage
answer json.RawMessage
full strings.Builder // the whole answer, when it must fit a schema
user string // the request's last user turn, for the judge
sofar strings.Builder // the answer so far, for the judge
served strings.Builder // the answer so far, for outside services, asked once at its end
judged int // how much of it the judge has read
v Verdict
took time.Duration
stopped bool
// restored are the request's redacted values, by placeholder, put back
// in the answer before this filter reads it.
restored map[string]string
ctx context.Context
g *Gate
p Policy
names string
shadow bool
schemas map[string]json.RawMessage
answer json.RawMessage
full strings.Builder // the whole answer, when it must fit a schema
user string // the request's last user turn, for the judge
sofar strings.Builder // the answer so far, for the judge
served strings.Builder // the answer so far, for outside services, asked once at its end
judged int // how much of it the judge has read
v Verdict
took time.Duration
stopped bool
held []string // events not yet released
sent bool // whether a chunk this filter made has gone out
@@ -105,6 +108,8 @@ type choiceState struct {
// replaced: it reaches the client only in a chunk this filter makes.
carried bool
calls map[int]*call
// done says the choice has finished: its text is all here.
done bool
}
type call struct {
@@ -214,6 +219,13 @@ func (f *outFilter) askServices(text string) finding {
if !slices.ContainsFunc(f.p.Services, Service.onOutput) || text == "" {
return finding{}
}
// As the model wrote it: with restore, the values put back for the
// client are taken out again, as redact took them out of the request.
for ph, v := range f.restored {
if v != "" {
text = strings.ReplaceAll(text, v, ph)
}
}
start := time.Now()
fs, err := f.g.askServices(f.ctx, f.p, false, text)
f.took += time.Since(start)
@@ -381,6 +393,7 @@ func (f *outFilter) OnEvent(data string) ([]string, bool) {
}
if ch.FinishReason != nil && *ch.FinishReason != "" {
finished = true
st.done = true
}
}
if f.shadow {
@@ -447,9 +460,10 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
}
if i == 0 {
f.served.WriteString(head)
// Once: the finish chunk is final, and so is the end of the
// stream after it, which would ask again of the same answer.
if final && !f.servicesAsked {
// Once, when the first choice is whole — not when another
// choice finishes first — and not again at the end of the
// stream after its finish.
if final && st.done && !f.servicesAsked {
f.servicesAsked = true
if f.askServices(f.served.String()).action == Block {
blocked = true
@@ -573,6 +587,10 @@ func withoutContent(data string) string {
}
func (f *outFilter) OnEnd() []string {
// The stream is over: whatever a choice has is all it will have.
for _, st := range f.choices {
st.done = true
}
switch {
case f.stopped:
return nil
+36
View File
@@ -148,3 +148,39 @@ func TestAServiceNamedByTwoPoliciesIsAskedTheStricterWay(t *testing.T) {
t.Errorf("%+v", got)
}
}
func TestAnOutputServiceReadsTheAnswerWithoutTheRestoredValues(t *testing.T) {
srv, seen := recorder(t)
g := gate(t, lib(map[string]Policy{"p": {
Services: []Service{{Name: "cls", URL: srv.URL, Blocked: "flagged", On: "output"}},
}}, "p"))
f := g.Output(context.Background(), Subject{Restored: map[string]string{"[EMAIL_1]": "ayse@example.com"}})
f.Buffered([]byte(`{"choices":[{"index":0,"message":{"role":"assistant","content":"Sent to ayse@example.com."},"finish_reason":"stop"}]}`))
if len(*seen) != 1 || strings.Contains((*seen)[0], "ayse@") || !strings.Contains((*seen)[0], "[EMAIL_1]") {
t.Errorf("the service read %q", *seen)
}
}
func TestOfTwoChoicesTheServiceReadsTheFirstWhole(t *testing.T) {
// The second finishes first; the first is still arriving then.
srv, seen := recorder(t)
g := gate(t, lib(map[string]Policy{"p": {
Services: []Service{{Name: "cls", URL: srv.URL, Blocked: "flagged", On: "output"}},
}}, "p"))
f := g.Output(context.Background(), Subject{Stream: true})
ev := func(idx int, text string, finish bool) string {
c := map[string]any{"index": idx, "delta": map[string]any{"content": text}, "finish_reason": nil}
if finish {
c["finish_reason"] = "stop"
}
b, _ := json.Marshal(map[string]any{"choices": []any{c}})
return string(b)
}
for _, e := range []string{ev(0, "The first ", false), ev(1, "Second.", true), ev(0, "answer.", true)} {
f.OnEvent(e)
}
f.OnEnd()
if len(*seen) != 1 || (*seen)[0] != "The first answer." {
t.Errorf("the service read %q", *seen)
}
}