Extended thinking on /v1/messages: a model's reasoning as thinking blocks
CI / test (push) Successful in 9m12s
CI / test (push) Successful in 9m12s
Claude Code and the Anthropic SDKs asking for extended thinking got the answer alone: a reasoning model's reasoning_content was dropped. Asked for, it comes back as a thinking block before the text — streamed as thinking_delta events and closed with a signature before the text begins — and the model is told to think through its chat template (enable_thinking); disabled tells it not to. Not asked for, it is left out as before. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0460f52dd2
commit
7c53380e68
3 files changed
+134
-8
No files matched your search
@@ -28,6 +28,11 @@ APIs change between minors without a deprecation period.
|
||||
|
||||
### Added
|
||||
|
||||
- **Extended thinking on `/v1/messages`**: `thinking: {type: "enabled"}`
|
||||
tells a model that thinks by its chat template to (`enable_thinking`),
|
||||
and its reasoning comes back as `thinking` blocks before the text,
|
||||
streamed as `thinking_delta` events; `disabled` tells it not to. Not
|
||||
asked for, the reasoning is left out as before.
|
||||
- **How long on-demand models hold a card** is a setting, in force on the
|
||||
leader within seconds: `on_demand.evict_after_idle` (2m unless set) and
|
||||
`on_demand.min_lifetime` (5m). On one card shared by a picture model and
|
||||
|
||||
@@ -12,9 +12,10 @@ import (
|
||||
// Anthropic's Messages API (/v1/messages), translated to a chat completion
|
||||
// on the way in and back on the way out, like the Responses API: so Claude
|
||||
// Code and Anthropic's SDKs reach the models here. Text, images, tools and
|
||||
// their results, the system prompt and streaming are translated; extended
|
||||
// thinking, prompt caching and server tools are not, and are ignored rather
|
||||
// than refused.
|
||||
// their results, the system prompt, streaming and extended thinking are
|
||||
// translated; prompt caching and server tools are not, and are ignored rather
|
||||
// than refused. Thinking asked for is a model's reasoning_content, given
|
||||
// back as thinking blocks before the text; not asked for, it is left out.
|
||||
|
||||
type msgRequest struct {
|
||||
Model string `json:"model"`
|
||||
@@ -34,8 +35,16 @@ type msgRequest struct {
|
||||
Metadata *struct {
|
||||
UserID string `json:"user_id"`
|
||||
} `json:"metadata"`
|
||||
// Thinking is extended thinking: "enabled" (with a budget the engines
|
||||
// here do not take) or "disabled".
|
||||
Thinking *struct {
|
||||
Type string `json:"type"`
|
||||
} `json:"thinking"`
|
||||
}
|
||||
|
||||
// thinks is whether the request asked for thinking blocks.
|
||||
func (in msgRequest) thinks() bool { return in.Thinking != nil && in.Thinking.Type == "enabled" }
|
||||
|
||||
type msgMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content json.RawMessage `json:"content"`
|
||||
@@ -197,6 +206,16 @@ func messagesToChat(in msgRequest) (map[string]any, error) {
|
||||
if in.Metadata != nil && in.Metadata.UserID != "" {
|
||||
out["user"] = in.Metadata.UserID
|
||||
}
|
||||
// A model that thinks or not as its chat template is told (Qwen3, and
|
||||
// the reasoning models served the same way) is told what was asked.
|
||||
if in.Thinking != nil {
|
||||
switch in.Thinking.Type {
|
||||
case "enabled":
|
||||
out["chat_template_kwargs"] = map[string]any{"enable_thinking": true}
|
||||
case "disabled":
|
||||
out["chat_template_kwargs"] = map[string]any{"enable_thinking": false}
|
||||
}
|
||||
}
|
||||
var tools []map[string]any
|
||||
for _, t := range in.Tools {
|
||||
if t.Type != "" && t.Type != "custom" {
|
||||
@@ -250,7 +269,7 @@ func (r *Router) handleMessages(w http.ResponseWriter, req *http.Request) {
|
||||
next.ContentLength = int64(len(upstream))
|
||||
next.Header.Set("Content-Type", "application/json")
|
||||
|
||||
t := &msgTranslator{w: w, model: in.Model, stream: in.Stream, id: newID("msg_")}
|
||||
t := &msgTranslator{w: w, model: in.Model, stream: in.Stream, id: newID("msg_"), thinking: in.thinks()}
|
||||
r.handleInference("/v1/chat/completions")(t, next)
|
||||
t.finish()
|
||||
}
|
||||
@@ -309,6 +328,11 @@ type msgTranslator struct {
|
||||
model string
|
||||
stream bool
|
||||
id string
|
||||
// thinking is whether the reasoning is given as thinking blocks.
|
||||
thinking bool
|
||||
// thinkIndex and thinkOpen are the thinking block being streamed.
|
||||
thinkIndex int
|
||||
thinkOpen bool
|
||||
|
||||
status int
|
||||
failed bool
|
||||
@@ -415,6 +439,20 @@ func stopReason(finish string) string {
|
||||
return "end_turn"
|
||||
}
|
||||
|
||||
// thinkingSignature stands where Anthropic signs a thinking block: the
|
||||
// thinking here comes from the gateway's own models, and nothing checks it,
|
||||
// but the SDKs expect the field.
|
||||
const thinkingSignature = "llmgw"
|
||||
|
||||
func (t *msgTranslator) closeThinking() {
|
||||
if t.thinkOpen {
|
||||
t.event("content_block_delta", map[string]any{"index": t.thinkIndex,
|
||||
"delta": map[string]any{"type": "signature_delta", "signature": thinkingSignature}})
|
||||
t.event("content_block_stop", map[string]any{"index": t.thinkIndex})
|
||||
t.thinkOpen = false
|
||||
}
|
||||
}
|
||||
|
||||
func (t *msgTranslator) closeText() {
|
||||
if t.textOpen {
|
||||
t.event("content_block_stop", map[string]any{"index": t.textIndex})
|
||||
@@ -433,8 +471,11 @@ func (t *msgTranslator) chatLine(line string) {
|
||||
var chunk struct {
|
||||
Choices []struct {
|
||||
Delta struct {
|
||||
Content string `json:"content"`
|
||||
ToolCalls []struct {
|
||||
Content string `json:"content"`
|
||||
// The reasoning, by either name the engines give it.
|
||||
ReasoningContent string `json:"reasoning_content"`
|
||||
Reasoning string `json:"reasoning"`
|
||||
ToolCalls []struct {
|
||||
Index int `json:"index"`
|
||||
ID string `json:"id"`
|
||||
Function struct {
|
||||
@@ -461,7 +502,18 @@ func (t *msgTranslator) chatLine(line string) {
|
||||
if ch.FinishReason != "" {
|
||||
t.stopReason = stopReason(ch.FinishReason)
|
||||
}
|
||||
if r := ch.Delta.ReasoningContent + ch.Delta.Reasoning; r != "" && t.thinking {
|
||||
if !t.thinkOpen {
|
||||
t.thinkIndex, t.thinkOpen = t.nextBlock, true
|
||||
t.nextBlock++
|
||||
t.event("content_block_start", map[string]any{"index": t.thinkIndex,
|
||||
"content_block": map[string]any{"type": "thinking", "thinking": ""}})
|
||||
}
|
||||
t.event("content_block_delta", map[string]any{"index": t.thinkIndex,
|
||||
"delta": map[string]any{"type": "thinking_delta", "thinking": r}})
|
||||
}
|
||||
if d := ch.Delta.Content; d != "" {
|
||||
t.closeThinking()
|
||||
if !t.textOpen && t.text.Len() == 0 {
|
||||
t.textIndex, t.textOpen = t.nextBlock, true
|
||||
t.nextBlock++
|
||||
@@ -477,6 +529,7 @@ func (t *msgTranslator) chatLine(line string) {
|
||||
for _, tc := range ch.Delta.ToolCalls {
|
||||
c := t.byIndex[tc.Index]
|
||||
if c == nil {
|
||||
t.closeThinking()
|
||||
t.closeText()
|
||||
for _, o := range t.calls {
|
||||
if o.open {
|
||||
@@ -532,6 +585,7 @@ func (t *msgTranslator) finish() {
|
||||
t.chatLine(rest)
|
||||
}
|
||||
t.begin()
|
||||
t.closeThinking()
|
||||
t.closeText()
|
||||
for _, c := range t.calls {
|
||||
if c.open {
|
||||
@@ -553,8 +607,10 @@ func (t *msgTranslator) fromChatJSON(b []byte) {
|
||||
var c struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
ToolCalls []struct {
|
||||
Content string `json:"content"`
|
||||
ReasoningContent string `json:"reasoning_content"`
|
||||
Reasoning string `json:"reasoning"`
|
||||
ToolCalls []struct {
|
||||
ID string `json:"id"`
|
||||
Function struct {
|
||||
Name string `json:"name"`
|
||||
@@ -578,6 +634,9 @@ func (t *msgTranslator) fromChatJSON(b []byte) {
|
||||
if len(c.Choices) > 0 {
|
||||
ch := c.Choices[0]
|
||||
reason = stopReason(ch.FinishReason)
|
||||
if r := ch.Message.ReasoningContent + ch.Message.Reasoning; r != "" && t.thinking {
|
||||
content = append(content, map[string]any{"type": "thinking", "thinking": r, "signature": thinkingSignature})
|
||||
}
|
||||
if ch.Message.Content != "" {
|
||||
content = append(content, map[string]any{"type": "text", "text": ch.Message.Content})
|
||||
}
|
||||
|
||||
@@ -139,3 +139,65 @@ func TestAnAnthropicClientsKeyIsAccepted(t *testing.T) {
|
||||
t.Errorf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestThinkingAskedForComesBackAsThinkingBlocks(t *testing.T) {
|
||||
// A reasoning model's thoughts were dropped: Claude Code asking for
|
||||
// extended thinking got the answer alone.
|
||||
var in msgRequest
|
||||
_ = json.Unmarshal([]byte(`{"model":"m","max_tokens":10,"thinking":{"type":"enabled","budget_tokens":2048},
|
||||
"messages":[{"role":"user","content":"hi"}]}`), &in)
|
||||
chat, _ := messagesToChat(in)
|
||||
if b, _ := json.Marshal(chat); !strings.Contains(string(b), `"chat_template_kwargs":{"enable_thinking":true}`) {
|
||||
t.Errorf("the model was not told to think: %s", b)
|
||||
}
|
||||
|
||||
whole := `{"choices":[{"message":{"reasoning_content":"two plus two","content":"4"},"finish_reason":"stop"}],
|
||||
"usage":{"prompt_tokens":3,"completion_tokens":5}}`
|
||||
_, h := messagesRouter(t, whole)
|
||||
body := map[string]any{"model": "m", "max_tokens": 10, "thinking": map[string]any{"type": "enabled", "budget_tokens": 1024},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "2+2?"}}}
|
||||
var out struct {
|
||||
Content []map[string]any `json:"content"`
|
||||
}
|
||||
_ = json.Unmarshal(sendMessages(t, h, body).Body.Bytes(), &out)
|
||||
if len(out.Content) != 2 || out.Content[0]["type"] != "thinking" || out.Content[0]["thinking"] != "two plus two" ||
|
||||
out.Content[0]["signature"] == nil || out.Content[1]["text"] != "4" {
|
||||
t.Errorf("content %v", out.Content)
|
||||
}
|
||||
// Not asked for, it is left out, as before.
|
||||
delete(body, "thinking")
|
||||
_ = json.Unmarshal(sendMessages(t, h, body).Body.Bytes(), &out)
|
||||
if len(out.Content) != 1 || out.Content[0]["type"] != "text" {
|
||||
t.Errorf("unasked, content %v", out.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamedThinkingIsItsOwnBlock(t *testing.T) {
|
||||
stream := "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"two \"}}]}\n\n" +
|
||||
"data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"plus two\"}}]}\n\n" +
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"4\"},\"finish_reason\":\"stop\"}]}\n\n" +
|
||||
"data: [DONE]\n\n"
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = w.Write([]byte(stream))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
rt := newTestRouter(t, baseConfig(config.Endpoint{ID: "a", URL: srv.URL, Models: []string{"m"}}))
|
||||
rec := sendMessages(t, rt.Handler(), map[string]any{"model": "m", "max_tokens": 10, "stream": true,
|
||||
"thinking": map[string]any{"type": "enabled", "budget_tokens": 1024},
|
||||
"messages": []any{map[string]any{"role": "user", "content": "2+2?"}}})
|
||||
got := rec.Body.String()
|
||||
for _, s := range []string{`"content_block":{"thinking":"","type":"thinking"},"index":0`,
|
||||
`"delta":{"thinking":"two ","type":"thinking_delta"}`, `"type":"signature_delta"`,
|
||||
`"content_block":{"text":"","type":"text"},"index":1`, `"text":"4","type":"text_delta"`} {
|
||||
if !strings.Contains(got, s) {
|
||||
t.Errorf("lacks %s in\n%s", s, got)
|
||||
}
|
||||
}
|
||||
if strings.Index(got, "signature_delta") > strings.Index(got, `"text_delta"`) {
|
||||
t.Error("the thinking block was not closed before the text began")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user