Guard a GPU-less node's problems, and bound thread ids
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a56ad8e4c4
commit
f9493aadb9
6 files changed
+64
-1
No files matched your search
@@ -179,6 +179,10 @@ APIs change between minors without a deprecation period.
|
||||
which refuse private addresses too, now also see through IPv6 addresses
|
||||
carrying an IPv4 one (NAT64, 6to4, IPv4-compatible), and through a proxy:
|
||||
the target's addresses are checked before the request is handed to it.
|
||||
- **A CPU-only node reporting a problem no longer panics the agent
|
||||
handler**, and **thread ids are bounded** to 128 letters, digits and
|
||||
`- _ . :`. A thread is readable by every key of its project that knows
|
||||
the id; the docs now say to use unguessable ids.
|
||||
|
||||
### Everything that can be changed on a running cluster, from the console
|
||||
|
||||
|
||||
@@ -1004,6 +1004,10 @@ curl $BASE/v1/chat/completions \
|
||||
-d '{"model": "<model-name>", "messages": [{"role":"user","content":"and then?"}]}'
|
||||
```
|
||||
|
||||
A thread belongs to the project, not to the key: **any key of the project that
|
||||
knows the id reads the conversation.** Use ids nobody can guess — a UUID, not
|
||||
`my-conversation` — of at most 128 letters, digits and `- _ . :`.
|
||||
|
||||
llmgw puts the stored history in front of that turn and sends it to whichever
|
||||
model is serving — **including a different one from the one the conversation
|
||||
started on**. That is the point: a thread begun on a large model can continue
|
||||
|
||||
@@ -316,3 +316,12 @@ func TestAnotherTenantOnACardIsJudgedEveryHeartbeat(t *testing.T) {
|
||||
t.Errorf("a heartbeat with no foreign reading cleared the card: %q", n.GPUs[0].Health.Problems)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAProblemOnANodeWithoutGPUsDoesNotPanic(t *testing.T) {
|
||||
s := newTestService()
|
||||
nodeWith(s, inventory.Node{ID: "cpu1", State: inventory.Ready})
|
||||
s.applyHeartbeat("cpu1", &llmgwv1.Heartbeat{NodeId: "cpu1", NewProblems: []string{"disk read errors"}})
|
||||
if _, ok := s.inv.Get("cpu1"); !ok {
|
||||
t.Fatal("the node is gone")
|
||||
}
|
||||
}
|
||||
@@ -454,7 +454,7 @@ func (s *Service) applyHeartbeat(nodeID string, hb *llmgwv1.Heartbeat) {
|
||||
n.GPUs[i].Health.Problems = ps
|
||||
}
|
||||
}
|
||||
if len(problems) > 0 {
|
||||
if len(problems) > 0 && len(n.GPUs) > 0 {
|
||||
// Problems are reported per node, not per GPU, so attach them to the
|
||||
// node's first GPU rather than guessing which one is affected. Any
|
||||
// problem is enough to keep that GPU out of placement, which is the
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAThreadIDIsANameNotAPayload(t *testing.T) {
|
||||
for _, ok := range []string{"9b2f6c1e-4a0d-4b8e-9f0a-1c2d3e4f5a6b", "chat_1.v2:x"} {
|
||||
if err := validThreadID(ok); err != nil {
|
||||
t.Errorf("%q refused: %v", ok, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{strings.Repeat("a", maxThreadID+1), "a b", "../x", "x\n", "ş"} {
|
||||
if err := validThreadID(bad); err == nil {
|
||||
t.Errorf("%q accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
|
||||
@@ -19,6 +21,28 @@ import (
|
||||
// the router already reads.
|
||||
const threadHeader = "x-llmgw-thread"
|
||||
|
||||
// maxThreadID bounds a thread id, which is a key in the store chosen by the
|
||||
// client.
|
||||
const maxThreadID = 128
|
||||
|
||||
// validThreadID refuses an id too long to be a name, or with characters no
|
||||
// generated id has.
|
||||
//
|
||||
// A thread is shared by every key of its project: whoever knows the id reads
|
||||
// the conversation. So an id should be unguessable — a UUID — and not
|
||||
// "chat-1".
|
||||
func validThreadID(id string) error {
|
||||
if len(id) > maxThreadID {
|
||||
return fmt.Errorf("%s is %d characters; at most %d", threadHeader, len(id), maxThreadID)
|
||||
}
|
||||
for _, c := range id {
|
||||
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || strings.ContainsRune("-_.:", c)) {
|
||||
return fmt.Errorf("%s may hold letters, digits and - _ . : only", threadHeader)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// trimmedHeader reports how many stored turns were left out to make the
|
||||
// conversation fit the model that served it.
|
||||
//
|
||||
@@ -215,6 +239,9 @@ func (r *Router) loadThread(ctx context.Context, req *parsedRequest) (int, error
|
||||
if req.ThreadID == "" {
|
||||
return 0, nil
|
||||
}
|
||||
if err := validThreadID(req.ThreadID); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
project, err := r.threadProject(req.ProjectID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
||||
Reference in new issue
Block a user