The gateway embeds the documents llmgw-kb sends as text
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
01ad8c9207
commit
cacaa35c3f
11 files changed
+350
-27
No files matched your search
@@ -63,6 +63,8 @@ type Collection struct {
|
||||
EmbedModel string `json:"embed_model"`
|
||||
Dims int `json:"dims"`
|
||||
Writable bool `json:"writable"`
|
||||
// ServerEmbeds is a gateway that embeds a document sent as text alone.
|
||||
ServerEmbeds bool `json:"server_embeds"`
|
||||
}
|
||||
|
||||
// RemoteDoc is a document the collection holds.
|
||||
@@ -82,8 +84,9 @@ type Manifest struct {
|
||||
|
||||
// ChunkOut is one chunk as sent.
|
||||
type ChunkOut struct {
|
||||
Text string `json:"text"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
Text string `json:"text"`
|
||||
// Embedding is empty when the gateway embeds the chunk.
|
||||
Embedding []float32 `json:"embedding,omitempty"`
|
||||
}
|
||||
|
||||
// DocOut is one document as sent.
|
||||
|
||||
@@ -46,8 +46,11 @@ type Options struct {
|
||||
Retry bool
|
||||
// Workers is how many files are read and sent at once; 0 is 4.
|
||||
Workers int
|
||||
Extract extract.Options
|
||||
Chunk chunk.Options
|
||||
// EmbedHere embeds the chunks on this machine even when the gateway
|
||||
// would: a vector up and down for each, rather than the text alone.
|
||||
EmbedHere bool
|
||||
Extract extract.Options
|
||||
Chunk chunk.Options
|
||||
}
|
||||
|
||||
// Event is something a sync did to one document, for showing progress.
|
||||
@@ -359,8 +362,12 @@ func (s *syncer) send(ctx context.Context, f file, name, digest, text string) {
|
||||
texts[i] = c.Text()
|
||||
size += int64(len(texts[i]))
|
||||
}
|
||||
// About ten bytes of JSON a dimension.
|
||||
size += int64(len(cs) * s.coll.Dims * 10)
|
||||
// Sent as text alone when the gateway embeds it; else about ten bytes
|
||||
// of JSON a dimension with each chunk.
|
||||
serverEmbeds := s.coll.ServerEmbeds && !s.o.EmbedHere
|
||||
if !serverEmbeds {
|
||||
size += int64(len(cs) * s.coll.Dims * 10)
|
||||
}
|
||||
if size > maxDocJSON {
|
||||
s.fail(ctx, f, name, digest, fmt.Errorf("%d MiB to send is over the gateway's limit for one document; split the file", size>>20))
|
||||
return
|
||||
@@ -369,12 +376,19 @@ func (s *syncer) send(ctx context.Context, f file, name, digest, text string) {
|
||||
s.emit(Event{Kind: "would-send", Name: name, Chunks: len(cs), Bytes: size})
|
||||
return
|
||||
}
|
||||
out := DocOut{Name: name, Digest: digest, Bytes: f.size, Chunks: make([]ChunkOut, len(cs))}
|
||||
if serverEmbeds {
|
||||
for i := range cs {
|
||||
out.Chunks[i] = ChunkOut{Text: texts[i]}
|
||||
}
|
||||
s.put(ctx, f, name, digest, out, len(cs), size)
|
||||
return
|
||||
}
|
||||
vecs, err := s.c.Embed(ctx, s.coll.EmbedModel, texts)
|
||||
if err != nil {
|
||||
s.fail(ctx, f, name, digest, fmt.Errorf("embedding with %s: %w", s.coll.EmbedModel, err))
|
||||
return
|
||||
}
|
||||
out := DocOut{Name: name, Digest: digest, Bytes: f.size, Chunks: make([]ChunkOut, len(cs))}
|
||||
for i := range cs {
|
||||
if len(vecs[i]) != s.coll.Dims {
|
||||
s.fail(ctx, f, name, digest, fmt.Errorf("%s gave %d dimensions and the collection takes %d",
|
||||
@@ -383,12 +397,17 @@ func (s *syncer) send(ctx context.Context, f file, name, digest, text string) {
|
||||
}
|
||||
out.Chunks[i] = ChunkOut{Text: texts[i], Embedding: vecs[i]}
|
||||
}
|
||||
s.put(ctx, f, name, digest, out, len(cs), size)
|
||||
}
|
||||
|
||||
// put sends a document and records it.
|
||||
func (s *syncer) put(ctx context.Context, f file, name, digest string, out DocOut, chunks int, size int64) {
|
||||
if _, err := s.c.Put(ctx, s.coll.ID, out); err != nil {
|
||||
s.fail(ctx, f, name, digest, err)
|
||||
return
|
||||
}
|
||||
s.record(ctx, s.entry(f, name, digest, Sent, "", len(cs)))
|
||||
s.emit(Event{Kind: "sent", Name: name, Chunks: len(cs), Bytes: size})
|
||||
s.record(ctx, s.entry(f, name, digest, Sent, "", chunks))
|
||||
s.emit(Event{Kind: "sent", Name: name, Chunks: chunks, Bytes: size})
|
||||
}
|
||||
|
||||
func (s *syncer) fail(ctx context.Context, f file, name, digest string, err error) {
|
||||
|
||||
@@ -22,6 +22,10 @@ type fakeGateway struct {
|
||||
puts int
|
||||
embedded int
|
||||
busyFirst bool
|
||||
// serverEmbeds is a gateway that embeds text it is sent; vectors
|
||||
// counts chunks that came with one.
|
||||
serverEmbeds bool
|
||||
vectors int
|
||||
}
|
||||
|
||||
func (g *fakeGateway) serve(t *testing.T) *Client {
|
||||
@@ -37,6 +41,7 @@ func (g *fakeGateway) serve(t *testing.T) *Client {
|
||||
}
|
||||
m := Manifest{Collection: coll}
|
||||
m.Collection.Writable = g.writable
|
||||
m.Collection.ServerEmbeds = g.serverEmbeds
|
||||
for _, d := range g.docs {
|
||||
m.Documents = append(m.Documents, d)
|
||||
}
|
||||
@@ -48,6 +53,11 @@ func (g *fakeGateway) serve(t *testing.T) *Client {
|
||||
g.mu.Lock()
|
||||
defer g.mu.Unlock()
|
||||
g.puts++
|
||||
for _, c := range d.Chunks {
|
||||
if len(c.Embedding) > 0 {
|
||||
g.vectors++
|
||||
}
|
||||
}
|
||||
g.docs[d.Name] = RemoteDoc{Name: d.Name, Digest: d.Digest, Chunks: len(d.Chunks)}
|
||||
_, _ = w.Write([]byte(`{"changed":true}`))
|
||||
})
|
||||
@@ -270,3 +280,22 @@ func TestFiltersAndNames(t *testing.T) {
|
||||
t.Errorf("an empty folder gave %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAGatewayThatEmbedsIsSentTextAlone(t *testing.T) {
|
||||
g, c, l, dir := setup(t)
|
||||
g.serverEmbeds = true
|
||||
rep, k := run(t, c, l, Options{Collection: "ik", Roots: []string{dir}})
|
||||
if len(k["sent"]) != 2 || rep.Failed != 0 {
|
||||
t.Fatalf("%+v %v", rep, k)
|
||||
}
|
||||
if g.embedded != 0 || g.vectors != 0 {
|
||||
t.Errorf("embedded here: %d asked, %d vectors sent", g.embedded, g.vectors)
|
||||
}
|
||||
|
||||
// --embed-here sends the vectors all the same.
|
||||
write(t, dir, "politika/yeni.md", "Yeni kural.\n")
|
||||
run(t, c, l, Options{Collection: "ik", Roots: []string{dir}, EmbedHere: true})
|
||||
if g.embedded == 0 || g.vectors == 0 {
|
||||
t.Errorf("--embed-here: %d asked, %d vectors sent", g.embedded, g.vectors)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user