diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d1612d..c9821cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,11 @@ APIs change between minors without a deprecation period. ### Added +- **Extended thinking on `/v1/messages`**: `thinking: {type: "enabled"}` + tells a model that thinks by its chat template to (`enable_thinking`), + and its reasoning comes back as `thinking` blocks before the text, + streamed as `thinking_delta` events; `disabled` tells it not to. Not + asked for, the reasoning is left out as before. - **How long on-demand models hold a card** is a setting, in force on the leader within seconds: `on_demand.evict_after_idle` (2m unless set) and `on_demand.min_lifetime` (5m). On one card shared by a picture model and diff --git a/internal/router/messages.go b/internal/router/messages.go index 0069000..5389187 100644 --- a/internal/router/messages.go +++ b/internal/router/messages.go @@ -12,9 +12,10 @@ import ( // Anthropic's Messages API (/v1/messages), translated to a chat completion // on the way in and back on the way out, like the Responses API: so Claude // Code and Anthropic's SDKs reach the models here. Text, images, tools and -// their results, the system prompt and streaming are translated; extended -// thinking, prompt caching and server tools are not, and are ignored rather -// than refused. +// their results, the system prompt, streaming and extended thinking are +// translated; prompt caching and server tools are not, and are ignored rather +// than refused. Thinking asked for is a model's reasoning_content, given +// back as thinking blocks before the text; not asked for, it is left out. type msgRequest struct { Model string `json:"model"` @@ -34,8 +35,16 @@ type msgRequest struct { Metadata *struct { UserID string `json:"user_id"` } `json:"metadata"` + // Thinking is extended thinking: "enabled" (with a budget the engines + // here do not take) or "disabled". + Thinking *struct { + Type string `json:"type"` + } `json:"thinking"` } +// thinks is whether the request asked for thinking blocks. +func (in msgRequest) thinks() bool { return in.Thinking != nil && in.Thinking.Type == "enabled" } + type msgMessage struct { Role string `json:"role"` Content json.RawMessage `json:"content"` @@ -197,6 +206,16 @@ func messagesToChat(in msgRequest) (map[string]any, error) { if in.Metadata != nil && in.Metadata.UserID != "" { out["user"] = in.Metadata.UserID } + // A model that thinks or not as its chat template is told (Qwen3, and + // the reasoning models served the same way) is told what was asked. + if in.Thinking != nil { + switch in.Thinking.Type { + case "enabled": + out["chat_template_kwargs"] = map[string]any{"enable_thinking": true} + case "disabled": + out["chat_template_kwargs"] = map[string]any{"enable_thinking": false} + } + } var tools []map[string]any for _, t := range in.Tools { if t.Type != "" && t.Type != "custom" { @@ -250,7 +269,7 @@ func (r *Router) handleMessages(w http.ResponseWriter, req *http.Request) { next.ContentLength = int64(len(upstream)) next.Header.Set("Content-Type", "application/json") - t := &msgTranslator{w: w, model: in.Model, stream: in.Stream, id: newID("msg_")} + t := &msgTranslator{w: w, model: in.Model, stream: in.Stream, id: newID("msg_"), thinking: in.thinks()} r.handleInference("/v1/chat/completions")(t, next) t.finish() } @@ -309,6 +328,11 @@ type msgTranslator struct { model string stream bool id string + // thinking is whether the reasoning is given as thinking blocks. + thinking bool + // thinkIndex and thinkOpen are the thinking block being streamed. + thinkIndex int + thinkOpen bool status int failed bool @@ -415,6 +439,20 @@ func stopReason(finish string) string { return "end_turn" } +// thinkingSignature stands where Anthropic signs a thinking block: the +// thinking here comes from the gateway's own models, and nothing checks it, +// but the SDKs expect the field. +const thinkingSignature = "llmgw" + +func (t *msgTranslator) closeThinking() { + if t.thinkOpen { + t.event("content_block_delta", map[string]any{"index": t.thinkIndex, + "delta": map[string]any{"type": "signature_delta", "signature": thinkingSignature}}) + t.event("content_block_stop", map[string]any{"index": t.thinkIndex}) + t.thinkOpen = false + } +} + func (t *msgTranslator) closeText() { if t.textOpen { t.event("content_block_stop", map[string]any{"index": t.textIndex}) @@ -433,8 +471,11 @@ func (t *msgTranslator) chatLine(line string) { var chunk struct { Choices []struct { Delta struct { - Content string `json:"content"` - ToolCalls []struct { + Content string `json:"content"` + // The reasoning, by either name the engines give it. + ReasoningContent string `json:"reasoning_content"` + Reasoning string `json:"reasoning"` + ToolCalls []struct { Index int `json:"index"` ID string `json:"id"` Function struct { @@ -461,7 +502,18 @@ func (t *msgTranslator) chatLine(line string) { if ch.FinishReason != "" { t.stopReason = stopReason(ch.FinishReason) } + if r := ch.Delta.ReasoningContent + ch.Delta.Reasoning; r != "" && t.thinking { + if !t.thinkOpen { + t.thinkIndex, t.thinkOpen = t.nextBlock, true + t.nextBlock++ + t.event("content_block_start", map[string]any{"index": t.thinkIndex, + "content_block": map[string]any{"type": "thinking", "thinking": ""}}) + } + t.event("content_block_delta", map[string]any{"index": t.thinkIndex, + "delta": map[string]any{"type": "thinking_delta", "thinking": r}}) + } if d := ch.Delta.Content; d != "" { + t.closeThinking() if !t.textOpen && t.text.Len() == 0 { t.textIndex, t.textOpen = t.nextBlock, true t.nextBlock++ @@ -477,6 +529,7 @@ func (t *msgTranslator) chatLine(line string) { for _, tc := range ch.Delta.ToolCalls { c := t.byIndex[tc.Index] if c == nil { + t.closeThinking() t.closeText() for _, o := range t.calls { if o.open { @@ -532,6 +585,7 @@ func (t *msgTranslator) finish() { t.chatLine(rest) } t.begin() + t.closeThinking() t.closeText() for _, c := range t.calls { if c.open { @@ -553,8 +607,10 @@ func (t *msgTranslator) fromChatJSON(b []byte) { var c struct { Choices []struct { Message struct { - Content string `json:"content"` - ToolCalls []struct { + Content string `json:"content"` + ReasoningContent string `json:"reasoning_content"` + Reasoning string `json:"reasoning"` + ToolCalls []struct { ID string `json:"id"` Function struct { Name string `json:"name"` @@ -578,6 +634,9 @@ func (t *msgTranslator) fromChatJSON(b []byte) { if len(c.Choices) > 0 { ch := c.Choices[0] reason = stopReason(ch.FinishReason) + if r := ch.Message.ReasoningContent + ch.Message.Reasoning; r != "" && t.thinking { + content = append(content, map[string]any{"type": "thinking", "thinking": r, "signature": thinkingSignature}) + } if ch.Message.Content != "" { content = append(content, map[string]any{"type": "text", "text": ch.Message.Content}) } diff --git a/internal/router/messages_test.go b/internal/router/messages_test.go index 10192ba..9c6a870 100644 --- a/internal/router/messages_test.go +++ b/internal/router/messages_test.go @@ -139,3 +139,65 @@ func TestAnAnthropicClientsKeyIsAccepted(t *testing.T) { t.Errorf("%d %s", rec.Code, rec.Body.String()) } } + +func TestThinkingAskedForComesBackAsThinkingBlocks(t *testing.T) { + // A reasoning model's thoughts were dropped: Claude Code asking for + // extended thinking got the answer alone. + var in msgRequest + _ = json.Unmarshal([]byte(`{"model":"m","max_tokens":10,"thinking":{"type":"enabled","budget_tokens":2048}, + "messages":[{"role":"user","content":"hi"}]}`), &in) + chat, _ := messagesToChat(in) + if b, _ := json.Marshal(chat); !strings.Contains(string(b), `"chat_template_kwargs":{"enable_thinking":true}`) { + t.Errorf("the model was not told to think: %s", b) + } + + whole := `{"choices":[{"message":{"reasoning_content":"two plus two","content":"4"},"finish_reason":"stop"}], + "usage":{"prompt_tokens":3,"completion_tokens":5}}` + _, h := messagesRouter(t, whole) + body := map[string]any{"model": "m", "max_tokens": 10, "thinking": map[string]any{"type": "enabled", "budget_tokens": 1024}, + "messages": []any{map[string]any{"role": "user", "content": "2+2?"}}} + var out struct { + Content []map[string]any `json:"content"` + } + _ = json.Unmarshal(sendMessages(t, h, body).Body.Bytes(), &out) + if len(out.Content) != 2 || out.Content[0]["type"] != "thinking" || out.Content[0]["thinking"] != "two plus two" || + out.Content[0]["signature"] == nil || out.Content[1]["text"] != "4" { + t.Errorf("content %v", out.Content) + } + // Not asked for, it is left out, as before. + delete(body, "thinking") + _ = json.Unmarshal(sendMessages(t, h, body).Body.Bytes(), &out) + if len(out.Content) != 1 || out.Content[0]["type"] != "text" { + t.Errorf("unasked, content %v", out.Content) + } +} + +func TestStreamedThinkingIsItsOwnBlock(t *testing.T) { + stream := "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"two \"}}]}\n\n" + + "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"plus two\"}}]}\n\n" + + "data: {\"choices\":[{\"delta\":{\"content\":\"4\"},\"finish_reason\":\"stop\"}]}\n\n" + + "data: [DONE]\n\n" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/chat/completions" { + return + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = w.Write([]byte(stream)) + })) + t.Cleanup(srv.Close) + rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}})) + rec := sendMessages(t, rt.Handler(), map[string]any{"model": "m", "max_tokens": 10, "stream": true, + "thinking": map[string]any{"type": "enabled", "budget_tokens": 1024}, + "messages": []any{map[string]any{"role": "user", "content": "2+2?"}}}) + got := rec.Body.String() + for _, s := range []string{`"content_block":{"thinking":"","type":"thinking"},"index":0`, + `"delta":{"thinking":"two ","type":"thinking_delta"}`, `"type":"signature_delta"`, + `"content_block":{"text":"","type":"text"},"index":1`, `"text":"4","type":"text_delta"`} { + if !strings.Contains(got, s) { + t.Errorf("lacks %s in\n%s", s, got) + } + } + if strings.Index(got, "signature_delta") > strings.Index(got, `"text_delta"`) { + t.Error("the thinking block was not closed before the text began") + } +}