CI / test (push) Successful in 9m16s
From the second review. - An output service reads the answer with the values restore put back taken out again (guard.Subject.Restored), as an input service reads the request redacted. - With n above 1, the service reads the first choice once it is whole, not at whichever choice finished first. - Restored values go into an answer asked for as JSON (response_format) escaped for it, as into tool-call arguments. - The embedding failure is logged once a minute through an outage, not once. - A card's own lane count and fastest generation come from the kernel (max_link_width, max_link_speed): the driver's width.max is what card and slot allow together, so an x16 card in an x8 slot read x8 of x8 and was never said. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
249 lines
9.9 KiB
Go
249 lines
9.9 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
// restoreGuard redacts alice's address as the gate does, and asks for it
|
|
// back in the answer.
|
|
type restoreGuard struct{}
|
|
|
|
func (restoreGuard) Input(_ context.Context, _ guard.Subject, body []byte) (guard.Verdict, error) {
|
|
if !bytes.Contains(body, []byte("alice@example.com")) {
|
|
return guard.Verdict{}, nil
|
|
}
|
|
return guard.Verdict{Action: guard.Redact, Policy: "pii", Category: "email",
|
|
Body: bytes.ReplaceAll(body, []byte("alice@example.com"), []byte("[EMAIL_1]")),
|
|
Restore: map[string]string{"[EMAIL_1]": `alice@example.com`}}, nil
|
|
}
|
|
func (restoreGuard) Output(context.Context, guard.Subject) guard.OutputFilter { return nil }
|
|
func (restoreGuard) Directives(guard.Subject) guard.Directives { return guard.Directives{} }
|
|
|
|
// echoEngine answers with the placeholder, in one piece or, streamed, split
|
|
// across two events.
|
|
func echoEngine(t *testing.T) *httptest.Server {
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, _ *http.Request) {
|
|
fmt.Fprint(w, "vllm:num_requests_running 0\nvllm:num_requests_waiting 0\nvllm:kv_cache_usage_perc 0.1\n")
|
|
})
|
|
mux.HandleFunc("POST /v1/chat/completions", func(w http.ResponseWriter, r *http.Request) {
|
|
b := new(bytes.Buffer)
|
|
_, _ = b.ReadFrom(r.Body)
|
|
if bytes.Contains(b.Bytes(), []byte("alice@example.com")) {
|
|
t.Errorf("the engine saw the address: %s", b)
|
|
}
|
|
if !bytes.Contains(b.Bytes(), []byte(`"stream":true`)) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprint(w, `{"model":"m","choices":[{"index":0,"message":{"role":"assistant","content":"Sent to [EMAIL_1]."},"finish_reason":"stop"}],"usage":{"prompt_tokens":4,"completion_tokens":4}}`)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/event-stream")
|
|
for _, piece := range []string{"Sent to [EMA", "IL_1]", "."} {
|
|
fmt.Fprintf(w, "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{\"content\":%q},\"finish_reason\":null}]}\n\n", piece)
|
|
}
|
|
fmt.Fprint(w, "data: {\"model\":\"m\",\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\ndata: [DONE]\n\n")
|
|
})
|
|
srv := httptest.NewServer(mux)
|
|
t.Cleanup(srv.Close)
|
|
return srv
|
|
}
|
|
|
|
func TestTheAnswerGetsBackTheValuesItsRequestHadRedacted(t *testing.T) {
|
|
srv := echoEngine(t)
|
|
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}}))
|
|
rt.SetGuard(restoreGuard{})
|
|
|
|
rec := post(t, rt.Handler(), say("write to alice@example.com"), nil)
|
|
if !strings.Contains(rec.Body.String(), "Sent to alice@example.com.") || strings.Contains(rec.Body.String(), "[EMAIL_1]") {
|
|
t.Errorf("one-shot answer: %s", rec.Body.String())
|
|
}
|
|
|
|
streamed := say("write to alice@example.com")
|
|
streamed["stream"] = true
|
|
rec = post(t, rt.Handler(), streamed, nil)
|
|
var text strings.Builder
|
|
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
|
if i := strings.Index(line, `"content":"`); i >= 0 {
|
|
rest := line[i+len(`"content":"`):]
|
|
text.WriteString(rest[:strings.Index(rest, `"`)])
|
|
}
|
|
}
|
|
if text.String() != "Sent to alice@example.com." {
|
|
t.Errorf("streamed answer reads %q\n%s", text.String(), rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestARestorerHoldsOnlyWhatCouldBeAPlaceholder(t *testing.T) {
|
|
r := newRestorer(map[string]string{"[EMAIL_1]": "a@b.c", "[PHONE_1]": "+46 70"})
|
|
for _, c := range []struct{ in, send, hold string }{
|
|
{"Dear [EMA", "Dear ", "[EMA"},
|
|
{"a [1] b", "a [1] b", ""},
|
|
{"x [EMAIL_1] y", "x a@b.c y", ""},
|
|
{"end [", "end ", "["},
|
|
{"[X", "[X", ""},
|
|
} {
|
|
send, hold := r.split(c.in, false)
|
|
if send != c.send || hold != c.hold {
|
|
t.Errorf("split(%q) = %q, %q; want %q, %q", c.in, send, hold, c.send, c.hold)
|
|
}
|
|
}
|
|
// A value with a quote stays valid JSON in a whole answer.
|
|
q := newRestorer(map[string]string{"[PERSON_1]": `O"Brien`})
|
|
if got := string(q.body([]byte(`{"c":"hi [PERSON_1]"}`))); got != `{"c":"hi O\"Brien"}` {
|
|
t.Errorf("body: %s", got)
|
|
}
|
|
// What is held at the end is still sent.
|
|
e := newRestorer(map[string]string{"[EMAIL_1]": "a@b.c"})
|
|
e.OnEvent(`{"choices":[{"index":0,"delta":{"content":"to [EM"}}]}`)
|
|
if out := e.OnEnd(); len(out) != 1 || !strings.Contains(out[0], `"content":"[EM"`) {
|
|
t.Errorf("OnEnd: %v", out)
|
|
}
|
|
}
|
|
|
|
// anyEmailGuard redacts whichever address a request has to [EMAIL_1], and
|
|
// asks for it back: two callers' requests then read the same.
|
|
type anyEmailGuard struct{}
|
|
|
|
var anEmail = regexp.MustCompile(`[a-z]+@example\.com`)
|
|
|
|
func (anyEmailGuard) Input(_ context.Context, _ guard.Subject, body []byte) (guard.Verdict, error) {
|
|
addr := anEmail.Find(body)
|
|
if addr == nil {
|
|
return guard.Verdict{}, nil
|
|
}
|
|
return guard.Verdict{Action: guard.Redact, Policy: "pii", Category: "email",
|
|
Body: anEmail.ReplaceAll(body, []byte("[EMAIL_1]")),
|
|
Restore: map[string]string{"[EMAIL_1]": string(addr)}}, nil
|
|
}
|
|
func (anyEmailGuard) Output(context.Context, guard.Subject) guard.OutputFilter { return nil }
|
|
func (anyEmailGuard) Directives(guard.Subject) guard.Directives { return guard.Directives{} }
|
|
|
|
func TestAnAnswerWithACallersValuesPutBackIsNeverReusedForAnother(t *testing.T) {
|
|
// Redacted, alice's request and bob's read the same; the answer alice
|
|
// got has her address in it again, and must not be bob's.
|
|
srv := echoEngine(t)
|
|
rt, secret := cachingRouter(t, srv.URL, false)
|
|
rt.SetGuard(anyEmailGuard{})
|
|
hdr := map[string]string{"Authorization": "Bearer " + secret}
|
|
|
|
first := post(t, rt.Handler(), ask(0, "write to alice@example.com"), hdr)
|
|
second := post(t, rt.Handler(), ask(0, "write to bob@example.com"), hdr)
|
|
if strings.Contains(second.Body.String(), "alice") {
|
|
t.Fatalf("bob was given alice's answer: %s", second.Body.String())
|
|
}
|
|
if !strings.Contains(second.Body.String(), "bob@example.com") {
|
|
t.Errorf("bob's answer lacks his own address: %s", second.Body.String())
|
|
}
|
|
if got := first.Header().Get(cacheHeader) + "/" + second.Header().Get(cacheHeader); got != "bypass/bypass" {
|
|
t.Errorf("x-llmgw-cache %s, want bypass for both", got)
|
|
}
|
|
}
|
|
|
|
func TestAToolCallsArgumentsGetTheirValuesBackStreamedOrNot(t *testing.T) {
|
|
r := newRestorer(map[string]string{"[NAME_1]": `O"Brien`, "[EMAIL_1]": "a@example.com"})
|
|
// Whole: the value goes into the arguments escaped for the JSON they are.
|
|
body := r.body([]byte(`{"choices":[{"message":{"content":"to [NAME_1]","tool_calls":[{"function":{"name":"send",` +
|
|
`"arguments":"{\"to\":\"[EMAIL_1]\",\"name\":\"[NAME_1]\"}"}}]}}]}`))
|
|
var whole struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string
|
|
ToolCalls []struct{ Function struct{ Arguments string } } `json:"tool_calls"`
|
|
}
|
|
}
|
|
}
|
|
if err := json.Unmarshal(body, &whole); err != nil {
|
|
t.Fatalf("%v: %s", err, body)
|
|
}
|
|
var args map[string]string
|
|
if err := json.Unmarshal([]byte(whole.Choices[0].Message.ToolCalls[0].Function.Arguments), &args); err != nil ||
|
|
args["to"] != "a@example.com" || args["name"] != `O"Brien` || whole.Choices[0].Message.Content != `to O"Brien` {
|
|
t.Fatalf("arguments %q (%v), content %q", whole.Choices[0].Message.ToolCalls[0].Function.Arguments, err, whole.Choices[0].Message.Content)
|
|
}
|
|
|
|
// Streamed, the placeholder split across two events.
|
|
var argsText strings.Builder
|
|
for _, piece := range []string{`{"to":"[EMA`, `IL_1]"}`} {
|
|
ev, _ := json.Marshal(map[string]any{"choices": []any{map[string]any{"index": 0, "finish_reason": nil,
|
|
"delta": map[string]any{"tool_calls": []any{map[string]any{"index": 0, "function": map[string]any{"arguments": piece}}}}}}})
|
|
out, _ := r.OnEvent(string(ev))
|
|
for _, o := range out {
|
|
var e struct {
|
|
Choices []struct {
|
|
Delta struct {
|
|
ToolCalls []struct{ Function struct{ Arguments string } } `json:"tool_calls"`
|
|
}
|
|
}
|
|
}
|
|
_ = json.Unmarshal([]byte(o), &e)
|
|
for _, c := range e.Choices {
|
|
for _, tc := range c.Delta.ToolCalls {
|
|
argsText.WriteString(tc.Function.Arguments)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if argsText.String() != `{"to":"a@example.com"}` {
|
|
t.Errorf("streamed arguments read %q", argsText.String())
|
|
}
|
|
}
|
|
|
|
func TestHeldTextGoesOutWithTheFinishEvent(t *testing.T) {
|
|
// The answer ends on what could have begun a placeholder; the finish
|
|
// event, whose delta is empty, carries it rather than an event after.
|
|
r := newRestorer(map[string]string{"[EMAIL_1]": "a@example.com"})
|
|
out1, _ := r.OnEvent(`{"choices":[{"index":0,"delta":{"content":"see [EMA"},"finish_reason":null}]}`)
|
|
out2, _ := r.OnEvent(`{"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}`)
|
|
if len(r.OnEnd()) != 0 {
|
|
t.Error("held text was released after the finish event")
|
|
}
|
|
if !strings.Contains(out1[0], `"see "`) || !strings.Contains(out2[0], `"content":"[EMA"`) {
|
|
t.Errorf("%s / %s", out1, out2)
|
|
}
|
|
}
|
|
|
|
func TestAKeptAnswerHoldsNoValueItsRequestHadRedacted(t *testing.T) {
|
|
if got := withoutRestored("Sent to alice@example.com.", map[string]string{"[EMAIL_1]": "alice@example.com"}); got != "Sent to [EMAIL_1]." {
|
|
t.Errorf("%q", got)
|
|
}
|
|
}
|
|
|
|
func TestAnAnswerAskedForAsJSONGetsItsValuesEscapedForIt(t *testing.T) {
|
|
r := newRestorer(map[string]string{"[NAME_1]": `O"Brien`})
|
|
r.jsonContent = true
|
|
body := r.body([]byte(`{"choices":[{"message":{"content":"{\"name\":\"[NAME_1]\"}"}}]}`))
|
|
var whole struct {
|
|
Choices []struct{ Message struct{ Content string } }
|
|
}
|
|
if err := json.Unmarshal(body, &whole); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var obj map[string]string
|
|
if err := json.Unmarshal([]byte(whole.Choices[0].Message.Content), &obj); err != nil || obj["name"] != `O"Brien` {
|
|
t.Errorf("content %q (%v)", whole.Choices[0].Message.Content, err)
|
|
}
|
|
if !jsonAnswer([]byte(`{"response_format":{"type":"json_schema"}}`)) || jsonAnswer([]byte(`{"model":"m"}`)) {
|
|
t.Error("jsonAnswer")
|
|
}
|
|
}
|
|
|
|
func TestAFailureIsWarnedOfOnceAMinuteNotOnce(t *testing.T) {
|
|
var w lastWarned
|
|
now := time.Now()
|
|
if !w.Due(now, time.Minute) || w.Due(now.Add(30*time.Second), time.Minute) || !w.Due(now.Add(61*time.Second), time.Minute) {
|
|
t.Error("a failure every 30 s is warned of once a minute")
|
|
}
|
|
}
|