Extended thinking on /v1/messages: a model's reasoning as thinking blocks

Claude Code and the Anthropic SDKs asking for extended thinking got the
answer alone: a reasoning model's reasoning_content was dropped. Asked for,
it comes back as a thinking block before the text — streamed as
thinking_delta events and closed with a signature before the text begins —
and the model is told to think through its chat template
(enable_thinking); disabled tells it not to. Not asked for, it is left out
as before.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-14 11:05:02 +02:00
1 parent 0460f52dd2
commit 7c53380e68
3 files changed
+134 -8

No files matched your search

+5
View File
@@ -28,6 +28,11 @@ APIs change between minors without a deprecation period.
### Added
- **Extended thinking on `/v1/messages`**: `thinking: {type: "enabled"}`
tells a model that thinks by its chat template to (`enable_thinking`),
and its reasoning comes back as `thinking` blocks before the text,
streamed as `thinking_delta` events; `disabled` tells it not to. Not
asked for, the reasoning is left out as before.
- **How long on-demand models hold a card** is a setting, in force on the
leader within seconds: `on_demand.evict_after_idle` (2m unless set) and
`on_demand.min_lifetime` (5m). On one card shared by a picture model and
+67 -8
View File
@@ -12,9 +12,10 @@ import (
// Anthropic's Messages API (/v1/messages), translated to a chat completion
// on the way in and back on the way out, like the Responses API: so Claude
// Code and Anthropic's SDKs reach the models here. Text, images, tools and
// their results, the system prompt and streaming are translated; extended
// thinking, prompt caching and server tools are not, and are ignored rather
// than refused.
// their results, the system prompt, streaming and extended thinking are
// translated; prompt caching and server tools are not, and are ignored rather
// than refused. Thinking asked for is a model's reasoning_content, given
// back as thinking blocks before the text; not asked for, it is left out.
type msgRequest struct {
Model string `json:"model"`
@@ -34,8 +35,16 @@ type msgRequest struct {
Metadata *struct {
UserID string `json:"user_id"`
} `json:"metadata"`
// Thinking is extended thinking: "enabled" (with a budget the engines
// here do not take) or "disabled".
Thinking *struct {
Type string `json:"type"`
} `json:"thinking"`
}
// thinks is whether the request asked for thinking blocks.
func (in msgRequest) thinks() bool { return in.Thinking != nil && in.Thinking.Type == "enabled" }
type msgMessage struct {
Role string `json:"role"`
Content json.RawMessage `json:"content"`
@@ -197,6 +206,16 @@ func messagesToChat(in msgRequest) (map[string]any, error) {
if in.Metadata != nil && in.Metadata.UserID != "" {
out["user"] = in.Metadata.UserID
}
// A model that thinks or not as its chat template is told (Qwen3, and
// the reasoning models served the same way) is told what was asked.
if in.Thinking != nil {
switch in.Thinking.Type {
case "enabled":
out["chat_template_kwargs"] = map[string]any{"enable_thinking": true}
case "disabled":
out["chat_template_kwargs"] = map[string]any{"enable_thinking": false}
}
}
var tools []map[string]any
for _, t := range in.Tools {
if t.Type != "" && t.Type != "custom" {
@@ -250,7 +269,7 @@ func (r *Router) handleMessages(w http.ResponseWriter, req *http.Request) {
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
t := &msgTranslator{w: w, model: in.Model, stream: in.Stream, id: newID("msg_")}
t := &msgTranslator{w: w, model: in.Model, stream: in.Stream, id: newID("msg_"), thinking: in.thinks()}
r.handleInference("/v1/chat/completions")(t, next)
t.finish()
}
@@ -309,6 +328,11 @@ type msgTranslator struct {
model string
stream bool
id string
// thinking is whether the reasoning is given as thinking blocks.
thinking bool
// thinkIndex and thinkOpen are the thinking block being streamed.
thinkIndex int
thinkOpen bool
status int
failed bool
@@ -415,6 +439,20 @@ func stopReason(finish string) string {
return "end_turn"
}
// thinkingSignature stands where Anthropic signs a thinking block: the
// thinking here comes from the gateway's own models, and nothing checks it,
// but the SDKs expect the field.
const thinkingSignature = "llmgw"
func (t *msgTranslator) closeThinking() {
if t.thinkOpen {
t.event("content_block_delta", map[string]any{"index": t.thinkIndex,
"delta": map[string]any{"type": "signature_delta", "signature": thinkingSignature}})
t.event("content_block_stop", map[string]any{"index": t.thinkIndex})
t.thinkOpen = false
}
}
func (t *msgTranslator) closeText() {
if t.textOpen {
t.event("content_block_stop", map[string]any{"index": t.textIndex})
@@ -433,8 +471,11 @@ func (t *msgTranslator) chatLine(line string) {
var chunk struct {
Choices []struct {
Delta struct {
Content string `json:"content"`
ToolCalls []struct {
Content string `json:"content"`
// The reasoning, by either name the engines give it.
ReasoningContent string `json:"reasoning_content"`
Reasoning string `json:"reasoning"`
ToolCalls []struct {
Index int `json:"index"`
ID string `json:"id"`
Function struct {
@@ -461,7 +502,18 @@ func (t *msgTranslator) chatLine(line string) {
if ch.FinishReason != "" {
t.stopReason = stopReason(ch.FinishReason)
}
if r := ch.Delta.ReasoningContent + ch.Delta.Reasoning; r != "" && t.thinking {
if !t.thinkOpen {
t.thinkIndex, t.thinkOpen = t.nextBlock, true
t.nextBlock++
t.event("content_block_start", map[string]any{"index": t.thinkIndex,
"content_block": map[string]any{"type": "thinking", "thinking": ""}})
}
t.event("content_block_delta", map[string]any{"index": t.thinkIndex,
"delta": map[string]any{"type": "thinking_delta", "thinking": r}})
}
if d := ch.Delta.Content; d != "" {
t.closeThinking()
if !t.textOpen && t.text.Len() == 0 {
t.textIndex, t.textOpen = t.nextBlock, true
t.nextBlock++
@@ -477,6 +529,7 @@ func (t *msgTranslator) chatLine(line string) {
for _, tc := range ch.Delta.ToolCalls {
c := t.byIndex[tc.Index]
if c == nil {
t.closeThinking()
t.closeText()
for _, o := range t.calls {
if o.open {
@@ -532,6 +585,7 @@ func (t *msgTranslator) finish() {
t.chatLine(rest)
}
t.begin()
t.closeThinking()
t.closeText()
for _, c := range t.calls {
if c.open {
@@ -553,8 +607,10 @@ func (t *msgTranslator) fromChatJSON(b []byte) {
var c struct {
Choices []struct {
Message struct {
Content string `json:"content"`
ToolCalls []struct {
Content string `json:"content"`
ReasoningContent string `json:"reasoning_content"`
Reasoning string `json:"reasoning"`
ToolCalls []struct {
ID string `json:"id"`
Function struct {
Name string `json:"name"`
@@ -578,6 +634,9 @@ func (t *msgTranslator) fromChatJSON(b []byte) {
if len(c.Choices) > 0 {
ch := c.Choices[0]
reason = stopReason(ch.FinishReason)
if r := ch.Message.ReasoningContent + ch.Message.Reasoning; r != "" && t.thinking {
content = append(content, map[string]any{"type": "thinking", "thinking": r, "signature": thinkingSignature})
}
if ch.Message.Content != "" {
content = append(content, map[string]any{"type": "text", "text": ch.Message.Content})
}
+62
View File
@@ -139,3 +139,65 @@ func TestAnAnthropicClientsKeyIsAccepted(t *testing.T) {
t.Errorf("%d %s", rec.Code, rec.Body.String())
}
}
func TestThinkingAskedForComesBackAsThinkingBlocks(t *testing.T) {
// A reasoning model's thoughts were dropped: Claude Code asking for
// extended thinking got the answer alone.
var in msgRequest
_ = json.Unmarshal([]byte(`{"model":"m","max_tokens":10,"thinking":{"type":"enabled","budget_tokens":2048},
"messages":[{"role":"user","content":"hi"}]}`), &in)
chat, _ := messagesToChat(in)
if b, _ := json.Marshal(chat); !strings.Contains(string(b), `"chat_template_kwargs":{"enable_thinking":true}`) {
t.Errorf("the model was not told to think: %s", b)
}
whole := `{"choices":[{"message":{"reasoning_content":"two plus two","content":"4"},"finish_reason":"stop"}],
"usage":{"prompt_tokens":3,"completion_tokens":5}}`
_, h := messagesRouter(t, whole)
body := map[string]any{"model": "m", "max_tokens": 10, "thinking": map[string]any{"type": "enabled", "budget_tokens": 1024},
"messages": []any{map[string]any{"role": "user", "content": "2+2?"}}}
var out struct {
Content []map[string]any `json:"content"`
}
_ = json.Unmarshal(sendMessages(t, h, body).Body.Bytes(), &out)
if len(out.Content) != 2 || out.Content[0]["type"] != "thinking" || out.Content[0]["thinking"] != "two plus two" ||
out.Content[0]["signature"] == nil || out.Content[1]["text"] != "4" {
t.Errorf("content %v", out.Content)
}
// Not asked for, it is left out, as before.
delete(body, "thinking")
_ = json.Unmarshal(sendMessages(t, h, body).Body.Bytes(), &out)
if len(out.Content) != 1 || out.Content[0]["type"] != "text" {
t.Errorf("unasked, content %v", out.Content)
}
}
func TestStreamedThinkingIsItsOwnBlock(t *testing.T) {
stream := "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"two \"}}]}\n\n" +
"data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"plus two\"}}]}\n\n" +
"data: {\"choices\":[{\"delta\":{\"content\":\"4\"},\"finish_reason\":\"stop\"}]}\n\n" +
"data: [DONE]\n\n"
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/v1/chat/completions" {
return
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = w.Write([]byte(stream))
}))
t.Cleanup(srv.Close)
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}}))
rec := sendMessages(t, rt.Handler(), map[string]any{"model": "m", "max_tokens": 10, "stream": true,
"thinking": map[string]any{"type": "enabled", "budget_tokens": 1024},
"messages": []any{map[string]any{"role": "user", "content": "2+2?"}}})
got := rec.Body.String()
for _, s := range []string{`"content_block":{"thinking":"","type":"thinking"},"index":0`,
`"delta":{"thinking":"two ","type":"thinking_delta"}`, `"type":"signature_delta"`,
`"content_block":{"text":"","type":"text"},"index":1`, `"text":"4","type":"text_delta"`} {
if !strings.Contains(got, s) {
t.Errorf("lacks %s in\n%s", s, got)
}
}
if strings.Index(got, "signature_delta") > strings.Index(got, `"text_delta"`) {
t.Error("the thinking block was not closed before the text began")
}
}