diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index c7fd4cd..a0a1b6e 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -8710,6 +8710,10 @@ "$ref": "#/components/schemas/catalog.Capabilities", "description": "Capabilities is what this model can do beyond plain completion." }, + "query_prefix": { + "type": "string", + "description": "QueryPrefix is put before a question an embedding model embeds for a knowledge-base search; documents are embedded without it. Models trained with an instruction on the query side want it, Qwen3-Embedding \"Instruct: Given a web search query, retrieve relevant passages that answer the query\\nQuery: \" for one." + }, "role": { "type": "string", "description": "Role \"guard\" is a model that judges the gateway's traffic for the guardrails rather than serving clients: hidden from /v1/models and the chat page, refused to any key that names it, critical by default, and always running. Guard says how its answers are read." diff --git a/docs/knowledge.md b/docs/knowledge.md index 76a1b01..c714d5a 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -45,6 +45,15 @@ is available to everyone who may use them. A collection stays with the model it was made with: vectors from two models cannot be compared. + Some embedding models are trained with an instruction before the + question and none before the documents. Give it as the entry's + `query_prefix`; llmgw puts it before questions only. For + Qwen3-Embedding: + + ```yaml + query_prefix: "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: " + ``` + With pgvector 0.7 or later a collection's index is kept at half precision (`halfvec`), which halves its memory and takes models of up to 4000 dimensions; with an older pgvector the limit is 2000. A collection diff --git a/docs/reference.md b/docs/reference.md index 4290e55..fb1384e 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -145,6 +145,7 @@ rather than repeated. | `context_length` | `int` | [Model](#model) | | `max_output` | `int` | [Model](#model) | | `capabilities` | `Capabilities` | [Model](#model) | +| `query_prefix` | `string` | [Model](#model) | | `role` | `string` | [Model](#model) | | `guard` | `*GuardModel` | [Model](#model) | | `origin` | `*Origin` | [Model](#model) | @@ -1389,6 +1390,16 @@ Capabilities is what this model can do beyond plain completion. Takes a [Capabilities](#capabilities) block. +### `query_prefix` + +`string` + +QueryPrefix is put before a question an embedding model embeds for a +knowledge-base search; documents are embedded without it. Models +trained with an instruction on the query side want it, Qwen3-Embedding +"Instruct: Given a web search query, retrieve relevant passages that +answer the query\nQuery: " for one. + ### `role` `string` diff --git a/internal/control/adminapi/kb.go b/internal/control/adminapi/kb.go index 4360eca..d7949ef 100644 --- a/internal/control/adminapi/kb.go +++ b/internal/control/adminapi/kb.go @@ -499,7 +499,11 @@ func (a *API) kbSearch(w http.ResponseWriter, r *http.Request) { // Said, not swallowed: a search that quietly lost its meaning half // looks like a collection that does not know the answer. if a.kb.Embed != nil { - v, err := a.kb.Embed(r.Context(), c.EmbedModel, in.Query) + question := in.Query + if m, ok := a.ctl.Catalog().Resolve(c.EmbedModel); ok { + question = m.QueryPrefix + question + } + v, err := a.kb.Embed(r.Context(), c.EmbedModel, question) if err != nil { writeError(w, http.StatusBadGateway, c.EmbedModel+" did not embed the question: "+err.Error()) return diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 3837756..06371b2 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -363,6 +363,13 @@ type Model struct { // Capabilities is what this model can do beyond plain completion. Capabilities Capabilities `json:"capabilities,omitempty" koanf:"capabilities"` + // QueryPrefix is put before a question an embedding model embeds for a + // knowledge-base search; documents are embedded without it. Models + // trained with an instruction on the query side want it, Qwen3-Embedding + // "Instruct: Given a web search query, retrieve relevant passages that + // answer the query\nQuery: " for one. + QueryPrefix string `json:"query_prefix,omitempty" koanf:"query_prefix"` + // Role "guard" is a model that judges the gateway's traffic for the // guardrails rather than serving clients: hidden from /v1/models and the // chat page, refused to any key that names it, critical by default, and diff --git a/internal/docgen/reference_gen.go b/internal/docgen/reference_gen.go index fd276c8..19ae5a0 100644 --- a/internal/docgen/reference_gen.go +++ b/internal/docgen/reference_gen.go @@ -181,6 +181,7 @@ var Reference = []Block{ {Key: "context_length", Type: "int", Doc: "ContextLength is how many tokens one request may use, prompt and\ncompletion together, and MaxOutput bounds the completion alone. Zero\nmeans unknown, which is what every model was until now.\n\nThe gateway knew neither, so an over-long prompt was not refused: it was\nadmitted, given a replica, and failed at the engine, and a client sizing\nits prompts from /v1/models had nothing to read. Worse, the number is not\nthe one an operator writes in the engine's arguments — llama.cpp divides\n--ctx-size across its parallel slots, so a pool of 8192 across four slots\nis 2048 a request, and nothing said so.", Nested: ""}, {Key: "max_output", Type: "int", Doc: "", Nested: ""}, {Key: "capabilities", Type: "Capabilities", Doc: "Capabilities is what this model can do beyond plain completion.", Nested: "catalog.Capabilities"}, + {Key: "query_prefix", Type: "string", Doc: "QueryPrefix is put before a question an embedding model embeds for a\nknowledge-base search; documents are embedded without it. Models\ntrained with an instruction on the query side want it, Qwen3-Embedding\n\"Instruct: Given a web search query, retrieve relevant passages that\nanswer the query\\nQuery: \" for one.", Nested: ""}, {Key: "role", Type: "string", Doc: "Role \"guard\" is a model that judges the gateway's traffic for the\nguardrails rather than serving clients: hidden from /v1/models and the\nchat page, refused to any key that names it, critical by default, and\nalways running. Guard says how its answers are read.", Nested: ""}, {Key: "guard", Type: "*GuardModel", Doc: "", Nested: "catalog.GuardModel"}, {Key: "origin", Type: "*Origin", Doc: "Origin is the registry repository the weights came from, at which\ncommit, and each file's checksum there. Written when a model is served\nfrom the registry; for one that was not, worked out from the mirror's\nrecord of where it fetched the file. It is what lets the control plane\nsay that the repository has published different weights since, and\nmove the model to them.", Nested: "catalog.Origin"}, diff --git a/internal/router/rag.go b/internal/router/rag.go index 76405f9..bb05620 100644 --- a/internal/router/rag.go +++ b/internal/router/rag.go @@ -276,6 +276,10 @@ func splitList(v string) []string { // ordinary path, so the caller's limits, budget and guardrails apply to it // and it is counted to them. func (r *Router) embedFor(ctx context.Context, p auth.Principal, model, text string) ([]float32, error) { + // A question, not a document: what the model wants before a query. + if _, m := r.resolve(model); m.QueryPrefix != "" { + text = m.QueryPrefix + text + } body, _ := json.Marshal(map[string]any{"model": model, "input": []string{text}}) hreq, err := http.NewRequestWithContext(auth.WithPrincipal(ctx, p), http.MethodPost, "/v1/embeddings", bytes.NewReader(body)) if err != nil { diff --git a/internal/router/rag_test.go b/internal/router/rag_test.go index 08614bb..1f346c5 100644 --- a/internal/router/rag_test.go +++ b/internal/router/rag_test.go @@ -9,7 +9,9 @@ import ( "strings" "testing" + "gitea.mynodes.xyz/ilker/llmgw/internal/auth" "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/kb/kbsync" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -253,3 +255,25 @@ func TestAVLLMReplicaGetsASaltOnlyWithPassages(t *testing.T) { t.Errorf("a salt without passages:\n%s", got) } } + +func TestAQuestionIsEmbeddedWithTheModelsQueryPrefix(t *testing.T) { + // Qwen3-Embedding and its like are trained with an instruction before + // the query, never before the documents: the documents come through + // /v1/embeddings as they are; the question gets the prefix here. + e := newFakeEngine(t, "emb") + cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"emb"}}) + m := catModel("emb", nil) + m.Capabilities.Embeddings = true + m.QueryPrefix = "Instruct: retrieve passages\nQuery: " + cfg.Models = []catalog.Model{m} + rt := newTestRouter(t, cfg) + // The test engine answers with the input's length in bytes and words. + v, err := rt.embedFor(t.Context(), auth.Principal{}, "emb", "kaç gün izin var") + if err != nil { + t.Fatal(err) + } + want := "Instruct: retrieve passages\nQuery: kaç gün izin var" + if int(v[0]) != len(want) || int(v[1]) != len(strings.Fields(want)) { + t.Errorf("embedded %v, not %q", v, want) + } +}