Outside services read answers redacted and whole; a card's own lanes
From the second review. - An output service reads the answer with the values restore put back taken out again (guard.Subject.Restored), as an input service reads the request redacted. - With n above 1, the service reads the first choice once it is whole, not at whichever choice finished first. - Restored values go into an answer asked for as JSON (response_format) escaped for it, as into tool-call arguments. - The embedding failure is logged once a minute through an outage, not once. - A card's own lane count and fastest generation come from the kernel (max_link_width, max_link_speed): the driver's width.max is what card and slot allow together, so an x16 card in an x8 slot read x8 of x8 and was never said. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1948a787ae
commit
c09d5ad4ee
12 files changed
+220
-30
No files matched your search
@@ -54,6 +54,10 @@ type Subject struct {
|
||||
Stream bool
|
||||
// Tools and Vision are what the model's catalog entry says it can do.
|
||||
Tools, Vision bool
|
||||
// Restored are the values the request's redaction took out, by
|
||||
// placeholder, which restore puts back in the answer: outside services
|
||||
// reading the answer are not told them.
|
||||
Restored map[string]string
|
||||
}
|
||||
|
||||
// Verdict is a judgement on one stage of one request.
|
||||
|
||||
+37
-19
@@ -68,7 +68,7 @@ func (g *Gate) Output(ctx context.Context, s Subject) OutputFilter {
|
||||
return nil
|
||||
}
|
||||
memo := g.tools.get(s.RequestID)
|
||||
return &outFilter{ctx: ctx, g: g, p: p, names: names, shadow: p.Mode == "shadow",
|
||||
return &outFilter{ctx: ctx, g: g, p: p, names: names, shadow: p.Mode == "shadow", restored: s.Restored,
|
||||
schemas: memo.schemas, answer: memo.answer, user: memo.user, choices: map[int]*choiceState{}}
|
||||
}
|
||||
|
||||
@@ -76,21 +76,24 @@ func (g *Gate) Output(ctx context.Context, s Subject) OutputFilter {
|
||||
type outFilter struct {
|
||||
// servicesAsked says the outside services have read the answer.
|
||||
servicesAsked bool
|
||||
ctx context.Context
|
||||
g *Gate
|
||||
p Policy
|
||||
names string
|
||||
shadow bool
|
||||
schemas map[string]json.RawMessage
|
||||
answer json.RawMessage
|
||||
full strings.Builder // the whole answer, when it must fit a schema
|
||||
user string // the request's last user turn, for the judge
|
||||
sofar strings.Builder // the answer so far, for the judge
|
||||
served strings.Builder // the answer so far, for outside services, asked once at its end
|
||||
judged int // how much of it the judge has read
|
||||
v Verdict
|
||||
took time.Duration
|
||||
stopped bool
|
||||
// restored are the request's redacted values, by placeholder, put back
|
||||
// in the answer before this filter reads it.
|
||||
restored map[string]string
|
||||
ctx context.Context
|
||||
g *Gate
|
||||
p Policy
|
||||
names string
|
||||
shadow bool
|
||||
schemas map[string]json.RawMessage
|
||||
answer json.RawMessage
|
||||
full strings.Builder // the whole answer, when it must fit a schema
|
||||
user string // the request's last user turn, for the judge
|
||||
sofar strings.Builder // the answer so far, for the judge
|
||||
served strings.Builder // the answer so far, for outside services, asked once at its end
|
||||
judged int // how much of it the judge has read
|
||||
v Verdict
|
||||
took time.Duration
|
||||
stopped bool
|
||||
|
||||
held []string // events not yet released
|
||||
sent bool // whether a chunk this filter made has gone out
|
||||
@@ -105,6 +108,8 @@ type choiceState struct {
|
||||
// replaced: it reaches the client only in a chunk this filter makes.
|
||||
carried bool
|
||||
calls map[int]*call
|
||||
// done says the choice has finished: its text is all here.
|
||||
done bool
|
||||
}
|
||||
|
||||
type call struct {
|
||||
@@ -214,6 +219,13 @@ func (f *outFilter) askServices(text string) finding {
|
||||
if !slices.ContainsFunc(f.p.Services, Service.onOutput) || text == "" {
|
||||
return finding{}
|
||||
}
|
||||
// As the model wrote it: with restore, the values put back for the
|
||||
// client are taken out again, as redact took them out of the request.
|
||||
for ph, v := range f.restored {
|
||||
if v != "" {
|
||||
text = strings.ReplaceAll(text, v, ph)
|
||||
}
|
||||
}
|
||||
start := time.Now()
|
||||
fs, err := f.g.askServices(f.ctx, f.p, false, text)
|
||||
f.took += time.Since(start)
|
||||
@@ -381,6 +393,7 @@ func (f *outFilter) OnEvent(data string) ([]string, bool) {
|
||||
}
|
||||
if ch.FinishReason != nil && *ch.FinishReason != "" {
|
||||
finished = true
|
||||
st.done = true
|
||||
}
|
||||
}
|
||||
if f.shadow {
|
||||
@@ -447,9 +460,10 @@ func (f *outFilter) flush(final bool) (bool, map[int]string, bool) {
|
||||
}
|
||||
if i == 0 {
|
||||
f.served.WriteString(head)
|
||||
// Once: the finish chunk is final, and so is the end of the
|
||||
// stream after it, which would ask again of the same answer.
|
||||
if final && !f.servicesAsked {
|
||||
// Once, when the first choice is whole — not when another
|
||||
// choice finishes first — and not again at the end of the
|
||||
// stream after its finish.
|
||||
if final && st.done && !f.servicesAsked {
|
||||
f.servicesAsked = true
|
||||
if f.askServices(f.served.String()).action == Block {
|
||||
blocked = true
|
||||
@@ -573,6 +587,10 @@ func withoutContent(data string) string {
|
||||
}
|
||||
|
||||
func (f *outFilter) OnEnd() []string {
|
||||
// The stream is over: whatever a choice has is all it will have.
|
||||
for _, st := range f.choices {
|
||||
st.done = true
|
||||
}
|
||||
switch {
|
||||
case f.stopped:
|
||||
return nil
|
||||
|
||||
@@ -148,3 +148,39 @@ func TestAServiceNamedByTwoPoliciesIsAskedTheStricterWay(t *testing.T) {
|
||||
t.Errorf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnOutputServiceReadsTheAnswerWithoutTheRestoredValues(t *testing.T) {
|
||||
srv, seen := recorder(t)
|
||||
g := gate(t, lib(map[string]Policy{"p": {
|
||||
Services: []Service{{Name: "cls", URL: srv.URL, Blocked: "flagged", On: "output"}},
|
||||
}}, "p"))
|
||||
f := g.Output(context.Background(), Subject{Restored: map[string]string{"[EMAIL_1]": "ayse@example.com"}})
|
||||
f.Buffered([]byte(`{"choices":[{"index":0,"message":{"role":"assistant","content":"Sent to ayse@example.com."},"finish_reason":"stop"}]}`))
|
||||
if len(*seen) != 1 || strings.Contains((*seen)[0], "ayse@") || !strings.Contains((*seen)[0], "[EMAIL_1]") {
|
||||
t.Errorf("the service read %q", *seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOfTwoChoicesTheServiceReadsTheFirstWhole(t *testing.T) {
|
||||
// The second finishes first; the first is still arriving then.
|
||||
srv, seen := recorder(t)
|
||||
g := gate(t, lib(map[string]Policy{"p": {
|
||||
Services: []Service{{Name: "cls", URL: srv.URL, Blocked: "flagged", On: "output"}},
|
||||
}}, "p"))
|
||||
f := g.Output(context.Background(), Subject{Stream: true})
|
||||
ev := func(idx int, text string, finish bool) string {
|
||||
c := map[string]any{"index": idx, "delta": map[string]any{"content": text}, "finish_reason": nil}
|
||||
if finish {
|
||||
c["finish_reason"] = "stop"
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"choices": []any{c}})
|
||||
return string(b)
|
||||
}
|
||||
for _, e := range []string{ev(0, "The first ", false), ev(1, "Second.", true), ev(0, "answer.", true)} {
|
||||
f.OnEvent(e)
|
||||
}
|
||||
f.OnEnd()
|
||||
if len(*seen) != 1 || (*seen)[0] != "The first answer." {
|
||||
t.Errorf("the service read %q", *seen)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user