The gateway embeds the documents llmgw-kb sends as text
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
01ad8c9207
commit
cacaa35c3f
11 files changed
+350
-27
No files matched your search
+44
-4
@@ -53,6 +53,9 @@ type KBCollectionView struct {
|
||||
Dims int `json:"dims"`
|
||||
// Writable is whether this caller may add and remove documents.
|
||||
Writable bool `json:"writable"`
|
||||
// ServerEmbeds says a document may be sent as text alone: the gateway
|
||||
// embeds its chunks, as the caller, with the collection's model.
|
||||
ServerEmbeds bool `json:"server_embeds"`
|
||||
}
|
||||
|
||||
// KBDocumentView is a document as the manifest lists it.
|
||||
@@ -77,8 +80,10 @@ type KBPutDocument struct {
|
||||
Digest string `json:"digest"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Chunks []struct {
|
||||
Text string `json:"text"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
Text string `json:"text"`
|
||||
// Embedding is the chunk's vector; with none on any chunk the
|
||||
// gateway embeds them.
|
||||
Embedding []float32 `json:"embedding,omitempty"`
|
||||
} `json:"chunks"`
|
||||
}
|
||||
|
||||
@@ -124,7 +129,7 @@ func (r *Router) kbCaller(w http.ResponseWriter, req *http.Request) (auth.Princi
|
||||
|
||||
func (r *Router) kbView(ctx context.Context, c storepkg.KBCollection, writable bool) KBCollectionView {
|
||||
return KBCollectionView{ID: c.ID, Name: c.Name, Owner: string(c.OwnerScope) + ":" + c.OwnerID,
|
||||
OwnerName: r.store.OwnerName(ctx, c.OwnerScope, c.OwnerID), EmbedModel: c.EmbedModel, Dims: c.Dims, Writable: writable}
|
||||
OwnerName: r.store.OwnerName(ctx, c.OwnerScope, c.OwnerID), EmbedModel: c.EmbedModel, Dims: c.Dims, Writable: writable, ServerEmbeds: true}
|
||||
}
|
||||
|
||||
// kbCollection finds a collection by id or name among those the caller may
|
||||
@@ -281,6 +286,19 @@ func (r *Router) handleKBPut(w http.ResponseWriter, req *http.Request) {
|
||||
fmt.Sprintf("%d chunks: a document has at most %d", len(in.Chunks), kbMaxChunks))
|
||||
return
|
||||
}
|
||||
// Text alone: the gateway embeds it, as the caller, so their limits and
|
||||
// usage count it as if they had asked.
|
||||
embedded := 0
|
||||
for _, ch := range in.Chunks {
|
||||
if len(ch.Embedding) > 0 {
|
||||
embedded++
|
||||
}
|
||||
}
|
||||
if embedded > 0 && embedded < len(in.Chunks) {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error",
|
||||
"some chunks have an embedding and some not: send every chunk's, or none for the gateway to embed")
|
||||
return
|
||||
}
|
||||
chunks := make([]storepkg.KBChunk, len(in.Chunks))
|
||||
for i, ch := range in.Chunks {
|
||||
if !utf8.ValidString(ch.Text) || strings.TrimSpace(ch.Text) == "" {
|
||||
@@ -292,7 +310,7 @@ func (r *Router) handleKBPut(w http.ResponseWriter, req *http.Request) {
|
||||
fmt.Sprintf("chunk %d is over %d characters", i, kbMaxChunkRunes))
|
||||
return
|
||||
}
|
||||
if len(ch.Embedding) != c.Dims {
|
||||
if embedded > 0 && len(ch.Embedding) != c.Dims {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error",
|
||||
fmt.Sprintf("chunk %d has %d dimensions; collection %s is embedded by %s in %d", i,
|
||||
len(ch.Embedding), c.Name, c.EmbedModel, c.Dims))
|
||||
@@ -300,6 +318,28 @@ func (r *Router) handleKBPut(w http.ResponseWriter, req *http.Request) {
|
||||
}
|
||||
chunks[i] = storepkg.KBChunk{Body: strings.ToValidUTF8(ch.Text, ""), Embedding: ch.Embedding}
|
||||
}
|
||||
if embedded == 0 {
|
||||
texts := make([]string, len(chunks))
|
||||
for i := range chunks {
|
||||
texts[i] = chunks[i].Body
|
||||
}
|
||||
vecs, err := r.kbEmbed(req, c, texts)
|
||||
if err != nil {
|
||||
var ee *kbEmbedError
|
||||
if errors.As(err, &ee) {
|
||||
if ee.retryAfter != "" {
|
||||
w.Header().Set("Retry-After", ee.retryAfter)
|
||||
}
|
||||
writeError(w, ee.status, "invalid_request_error", err.Error())
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusBadGateway, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
for i := range chunks {
|
||||
chunks[i].Embedding = vecs[i]
|
||||
}
|
||||
}
|
||||
d, changed, err := r.store.PutDocument(req.Context(), c, storepkg.KBDocument{Name: name, Digest: in.Digest,
|
||||
Bytes: in.Bytes, AddedBy: kbActor(p)}, chunks)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// A document sent as text alone is embedded here, through the gateway's own
|
||||
// /v1/embeddings as the caller: their key, limits and usage, as if they had
|
||||
// asked. The client then sends a kilobyte a chunk instead of the vector's
|
||||
// twenty in each direction.
|
||||
|
||||
const (
|
||||
// kbEmbedBatch is how many chunks go in one embeddings request, and
|
||||
// kbEmbedAtOnce how many such requests are in flight for one document:
|
||||
// enough to fill an embedding replica's slots.
|
||||
kbEmbedBatch = 32
|
||||
kbEmbedAtOnce = 4
|
||||
// kbEmbedWait is how long one batch waits out answers that say to try
|
||||
// again, before the document is refused with the same advice.
|
||||
kbEmbedWait = 5 * time.Minute
|
||||
)
|
||||
|
||||
// kbEmbedError is an embeddings answer that was not a success, passed on as
|
||||
// the API gave it.
|
||||
type kbEmbedError struct {
|
||||
status int
|
||||
retryAfter string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *kbEmbedError) Error() string { return e.msg }
|
||||
|
||||
// kbEmbed embeds texts with the collection's model, as the request's caller.
|
||||
func (r *Router) kbEmbed(req *http.Request, c storepkg.KBCollection, texts []string) ([][]float32, error) {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
defer cancel()
|
||||
out := make([][]float32, len(texts))
|
||||
var (
|
||||
mu sync.Mutex
|
||||
first error
|
||||
wg sync.WaitGroup
|
||||
)
|
||||
slots := make(chan struct{}, kbEmbedAtOnce)
|
||||
for start := 0; start < len(texts); start += kbEmbedBatch {
|
||||
end := min(start+kbEmbedBatch, len(texts))
|
||||
wg.Add(1)
|
||||
slots <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-slots }()
|
||||
vecs, err := r.kbEmbedBatch(ctx, req, c, texts[start:end])
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if err != nil {
|
||||
if first == nil {
|
||||
first = err
|
||||
cancel()
|
||||
}
|
||||
return
|
||||
}
|
||||
copy(out[start:end], vecs)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if first != nil {
|
||||
return nil, first
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// kbEmbedBatch embeds one batch, waiting out answers that say when to try
|
||||
// again.
|
||||
func (r *Router) kbEmbedBatch(ctx context.Context, req *http.Request, c storepkg.KBCollection, texts []string) ([][]float32, error) {
|
||||
call := mediaCall{req: req.WithContext(ctx)}
|
||||
deadline := time.Now().Add(kbEmbedWait)
|
||||
for {
|
||||
status, h, b, err := r.self(call, http.MethodPost, "/v1/embeddings",
|
||||
map[string]any{"model": c.EmbedModel, "input": texts, "encoding_format": "float"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if status < 300 {
|
||||
return kbVectors(b, len(texts), c)
|
||||
}
|
||||
retry := h.Get("Retry-After")
|
||||
if retry == "" || (status != http.StatusTooManyRequests && status < 500) || time.Now().After(deadline) {
|
||||
return nil, &kbEmbedError{status: status, retryAfter: retry,
|
||||
msg: fmt.Sprintf("embedding with %s: %s", c.EmbedModel, errorMessage(b))}
|
||||
}
|
||||
secs, _ := strconv.Atoi(retry)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(time.Duration(max(secs, 1)) * time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// kbVectors reads an embeddings answer: one vector a text, of the
|
||||
// collection's dimensions.
|
||||
func kbVectors(b []byte, n int, c storepkg.KBCollection) ([][]float32, error) {
|
||||
var res struct {
|
||||
Data []struct {
|
||||
Index int `json:"index"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &res); err != nil {
|
||||
return nil, fmt.Errorf("embedding with %s: the answer is not JSON: %w", c.EmbedModel, err)
|
||||
}
|
||||
if len(res.Data) != n {
|
||||
return nil, fmt.Errorf("embedding with %s: %d of %d texts came back", c.EmbedModel, len(res.Data), n)
|
||||
}
|
||||
out := make([][]float32, n)
|
||||
for _, d := range res.Data {
|
||||
if d.Index < 0 || d.Index >= n || out[d.Index] != nil {
|
||||
return nil, fmt.Errorf("embedding with %s: an embedding came back with index %d", c.EmbedModel, d.Index)
|
||||
}
|
||||
if len(d.Embedding) != c.Dims {
|
||||
return nil, &kbEmbedError{status: http.StatusBadRequest,
|
||||
msg: fmt.Sprintf("%s gave %d dimensions and collection %s takes %d", c.EmbedModel, len(d.Embedding), c.Name, c.Dims)}
|
||||
}
|
||||
out[d.Index] = d.Embedding
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// A document's text is embedded through the gateway's own API as the
|
||||
// caller, in batches, each vector in its chunk's place.
|
||||
func TestTheGatewayEmbedsADocumentSentAsText(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
_ = rt.Handler() // mounts the API the embedding goes through
|
||||
texts := make([]string, kbEmbedBatch*2+5)
|
||||
for i := range texts {
|
||||
texts[i] = strings.Repeat("söz ", i+1)
|
||||
}
|
||||
req := httptest.NewRequestWithContext(t.Context(), http.MethodPut, "/v1/kb/collections/x/documents", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+secret)
|
||||
c := store.KBCollection{Name: "x", EmbedModel: "qwen2.5-14b-instruct", Dims: 3}
|
||||
vecs, err := rt.kbEmbed(req, c, texts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, v := range vecs {
|
||||
// The fake engine's vector is the text's bytes, its words, and 1.
|
||||
if len(v) != 3 || int(v[1]) != i+1 || int(v[0]) != len(texts[i]) {
|
||||
t.Fatalf("chunk %d got %v", i, v)
|
||||
}
|
||||
}
|
||||
|
||||
// A model of other dimensions is refused, saying so.
|
||||
c.Dims = 4
|
||||
if _, err := rt.kbEmbed(req, c, texts[:2]); err == nil || !strings.Contains(err.Error(), "dimensions") {
|
||||
t.Errorf("4 dimensions: %v", err)
|
||||
}
|
||||
// Without the caller's key, nothing is embedded.
|
||||
bare := httptest.NewRequestWithContext(t.Context(), http.MethodPut, "/v1/kb/collections/x/documents", nil)
|
||||
c.Dims = 3
|
||||
if _, err := rt.kbEmbed(bare, c, texts[:1]); err == nil {
|
||||
t.Error("embedded without a key")
|
||||
}
|
||||
}
|
||||
@@ -206,3 +206,36 @@ func TestAPersonalKeyReadsOnlyInTheProjectItNames(t *testing.T) {
|
||||
t.Errorf("a project the person is not in: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADocumentSentAsTextIsEmbeddedByTheGateway(t *testing.T) {
|
||||
rt := kbRouter(t)
|
||||
h := rt.Handler()
|
||||
ctx := t.Context()
|
||||
s := rt.store
|
||||
sfx := strings.ToLower(store.NewID(""))
|
||||
org, _ := s.CreateOrg(ctx, "e-"+sfx)
|
||||
team, _ := s.CreateTeam(ctx, org, "t")
|
||||
prj, _ := s.CreateProject(ctx, team, "p")
|
||||
// The fake engine embeds in three dimensions.
|
||||
if _, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: prj,
|
||||
Name: "metin", EmbedModel: "qwen2.5-14b-instruct", Dims: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := kbKey(t, s, prj, "")
|
||||
base := "/v1/kb/collections/metin/"
|
||||
if code, _, body := call(t, h, key, http.MethodGet, base+"manifest", nil); !strings.Contains(body, `"server_embeds":true`) {
|
||||
t.Errorf("manifest: %d %s", code, body)
|
||||
}
|
||||
textOnly := map[string]any{"name": "a.md", "digest": "d1", "bytes": 10, "chunks": []map[string]any{
|
||||
{"text": "birinci parça"}, {"text": "ikinci parça burada"}}}
|
||||
code, out, body := call(t, h, key, http.MethodPut, base+"documents", textOnly)
|
||||
if code != http.StatusOK || out["chunks"] != float64(2) {
|
||||
t.Fatalf("text only: %d %s", code, body)
|
||||
}
|
||||
// Some with a vector and some without is refused.
|
||||
mixed := map[string]any{"name": "b.md", "digest": "d2", "bytes": 10, "chunks": []map[string]any{
|
||||
{"text": "bir", "embedding": []float32{1, 0, 0}}, {"text": "iki"}}}
|
||||
if code, _, body := call(t, h, key, http.MethodPut, base+"documents", mixed); code != http.StatusBadRequest {
|
||||
t.Errorf("mixed: %d %s", code, body)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user