Outside services read answers redacted and whole; a card's own lanes
CI / test (push) Successful in 9m16s
CI / test (push) Successful in 9m16s
From the second review. - An output service reads the answer with the values restore put back taken out again (guard.Subject.Restored), as an input service reads the request redacted. - With n above 1, the service reads the first choice once it is whole, not at whichever choice finished first. - Restored values go into an answer asked for as JSON (response_format) escaped for it, as into tool-call arguments. - The embedding failure is logged once a minute through an outage, not once. - A card's own lane count and fastest generation come from the kernel (max_link_width, max_link_speed): the driver's width.max is what card and slot allow together, so an x16 card in an x8 slot read x8 of x8 and was never said. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
1948a787ae
commit
c09d5ad4ee
12 files changed
+220
-30
No files matched your search
@@ -35,6 +35,7 @@ func (p *parsedRequest) guardSubject(model string) guard.Subject {
|
||||
// can do.
|
||||
func (r *Router) subject(req *parsedRequest, model string) guard.Subject {
|
||||
s := req.guardSubject(model)
|
||||
s.Restored = req.guardIn.Restore
|
||||
if _, entry := r.resolve(model); entry.Name != "" {
|
||||
s.Tools, s.Vision = entry.Capabilities.Tools, entry.Capabilities.Vision
|
||||
}
|
||||
|
||||
@@ -22,6 +22,10 @@ import (
|
||||
type restorer struct {
|
||||
values map[string]string
|
||||
longest int
|
||||
// jsonContent is an answer asked for as JSON (response_format): its
|
||||
// content is JSON, and a value goes into it escaped, as into a tool
|
||||
// call's arguments.
|
||||
jsonContent bool
|
||||
// held is, per choice and field, the tail of the text so far that may
|
||||
// be the start of a placeholder the next event completes: "[EMA" before
|
||||
// "IL_1]". Keyed by heldKey.
|
||||
@@ -110,7 +114,7 @@ func (r *restorer) walk(v any, key string) any {
|
||||
x[i] = r.walk(e, key)
|
||||
}
|
||||
case string:
|
||||
return r.text(x, key == "arguments")
|
||||
return r.text(x, key == "arguments" || (r.jsonContent && key == "content"))
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -164,7 +168,7 @@ func (r *restorer) OnEvent(data string) ([]string, bool) {
|
||||
}
|
||||
}
|
||||
if delta != nil {
|
||||
swap(delta, "content", heldKey{idx, "content", -1}, false)
|
||||
swap(delta, "content", heldKey{idx, "content", -1}, r.jsonContent)
|
||||
swap(delta, "reasoning_content", heldKey{idx, "reasoning_content", -1}, false)
|
||||
calls, _ := delta["tool_calls"].([]any)
|
||||
for j, tc := range calls {
|
||||
@@ -260,11 +264,12 @@ func (c streamChain) OnEnd() []string {
|
||||
|
||||
// restoreInto puts the input's placeholders back in this attempt's answer,
|
||||
// ahead of whatever output filter the attempt already has.
|
||||
func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
|
||||
func restoreInto(values map[string]string, stream, jsonContent bool, preq *proxy.Request) {
|
||||
if len(values) == 0 {
|
||||
return
|
||||
}
|
||||
rs := newRestorer(values)
|
||||
rs.jsonContent = jsonContent
|
||||
if f := preq.Filter; f != nil {
|
||||
preq.Filter = func(b []byte, h http.Header) []byte { return f(rs.body(b), h) }
|
||||
} else {
|
||||
@@ -279,3 +284,14 @@ func restoreInto(values map[string]string, stream bool, preq *proxy.Request) {
|
||||
preq.StreamFilter = rs
|
||||
}
|
||||
}
|
||||
|
||||
// jsonAnswer reports whether a request asks for its answer as JSON.
|
||||
func jsonAnswer(body []byte) bool {
|
||||
var b struct {
|
||||
ResponseFormat *struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"response_format"`
|
||||
}
|
||||
return json.Unmarshal(body, &b) == nil && b.ResponseFormat != nil &&
|
||||
(b.ResponseFormat.Type == "json_object" || b.ResponseFormat.Type == "json_schema")
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
@@ -218,3 +219,30 @@ func TestAKeptAnswerHoldsNoValueItsRequestHadRedacted(t *testing.T) {
|
||||
t.Errorf("%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAnswerAskedForAsJSONGetsItsValuesEscapedForIt(t *testing.T) {
|
||||
r := newRestorer(map[string]string{"[NAME_1]": `O"Brien`})
|
||||
r.jsonContent = true
|
||||
body := r.body([]byte(`{"choices":[{"message":{"content":"{\"name\":\"[NAME_1]\"}"}}]}`))
|
||||
var whole struct {
|
||||
Choices []struct{ Message struct{ Content string } }
|
||||
}
|
||||
if err := json.Unmarshal(body, &whole); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var obj map[string]string
|
||||
if err := json.Unmarshal([]byte(whole.Choices[0].Message.Content), &obj); err != nil || obj["name"] != `O"Brien` {
|
||||
t.Errorf("content %q (%v)", whole.Choices[0].Message.Content, err)
|
||||
}
|
||||
if !jsonAnswer([]byte(`{"response_format":{"type":"json_schema"}}`)) || jsonAnswer([]byte(`{"model":"m"}`)) {
|
||||
t.Error("jsonAnswer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailureIsWarnedOfOnceAMinuteNotOnce(t *testing.T) {
|
||||
var w lastWarned
|
||||
now := time.Now()
|
||||
if !w.Due(now, time.Minute) || w.Due(now.Add(30*time.Second), time.Minute) || !w.Due(now.Add(61*time.Second), time.Minute) {
|
||||
t.Error("a failure every 30 s is warned of once a minute")
|
||||
}
|
||||
}
|
||||
@@ -1256,7 +1256,7 @@ func (r *Router) dispatchOne(ctx context.Context, w http.ResponseWriter, req *pa
|
||||
}
|
||||
// A filter per attempt, for the model this attempt is served by.
|
||||
filter := r.outputFilter(uctx, req, model, &preq)
|
||||
restoreInto(req.guardIn.Restore, req.Stream, &preq)
|
||||
restoreInto(req.guardIn.Restore, req.Stream, jsonAnswer(req.UpstreamBody), &preq)
|
||||
var out proxy.Outcome
|
||||
var perr error
|
||||
if dec.Endpoint.Engine == endpoint.EngineComfyUI {
|
||||
@@ -1387,11 +1387,14 @@ type lastWarned struct {
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// Swap records now and says when it was before.
|
||||
func (w *lastWarned) Swap(now time.Time) time.Time {
|
||||
// Due says whether every has passed since it was last said, and if so
|
||||
// records now as that time: once a minute through an outage, not once.
|
||||
func (w *lastWarned) Due(now time.Time, every time.Duration) bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
was := w.at
|
||||
if now.Sub(w.at) < every {
|
||||
return false
|
||||
}
|
||||
w.at = now
|
||||
return was
|
||||
return true
|
||||
}
|
||||
@@ -54,7 +54,7 @@ func (r *Router) similarLookup(ctx context.Context, req *parsedRequest, model st
|
||||
vec, err := r.embedQuestion(ctx, question, req.inCluster)
|
||||
if err != nil {
|
||||
// Once a minute at most: down, the model fails every miss.
|
||||
if now := time.Now(); now.Sub(r.embedWarned.Swap(now)) > time.Minute {
|
||||
if r.embedWarned.Due(time.Now(), time.Minute) {
|
||||
r.log.Warn("a question could not be embedded for the cache; only the same request is reused",
|
||||
"model", r.cacheEmbedModel, "err", err)
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user