llmgw ctl tool-check asks with the engine's key

Every replica's engine demands its own key, and the check sent none, so it
failed with a 401 on every engine in the cluster. The key is asked of the
admin API for the replica at --endpoint, as bench does, or given with --key.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/cli/ctl.go

@@ -175,7 +175,7 @@ func newCtlCmd() *cobra.Command {
 		newJobsCmd(client, &asJSON),
 		newEngineCheckCmd(),
 		newEngineBuildCmd(client),
-		newToolCheckCmd(),
+		newToolCheckCmd(client),
 		newWeightsCmd(client, &asJSON),
 		newCapacityCmd(client, &asJSON),
 		newQuotaCmd(client, &asJSON),

internal/cli/toolcheck.go

@@ -34,10 +34,11 @@ import (
 // wrapper — left it in the content. A differently trained model fixes that. A
 // bigger one from the same family does not: qwen2.5-coder-7b invented
 // <function-call> and failed the same way.
-func newToolCheckCmd() *cobra.Command {
+func newToolCheckCmd(client func() *ctlClient) *cobra.Command {
 	var (
 		endpoint string
 		model    string
+		key      string
 		timeout  time.Duration
 	)
 
@@ -57,10 +58,21 @@ func newToolCheckCmd() *cobra.Command {
 			if endpoint == "" {
 				return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002")
 			}
+			// A replica's engine demands its own key, and refused the check
+			// with a 401 when none was sent. Asked of the admin API, as bench
+			// does; an endpoint that is no replica's is asked without one.
+			if key == "" {
+				reps, err := listReplicas(cmd, client)
+				if err != nil {
+					cmd.PrintErrf("The admin API could not be asked for the engine key of the replica at "+
+						"--endpoint (%v); an engine that demands one will refuse. Pass --key to give it.\n", err)
+				}
+				key = replicaKey(reps, endpoint)
+			}
 			ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
 			defer cancel()
 
-			res, err := probeToolCall(ctx, endpoint, model)
+			res, err := probeToolCall(ctx, endpoint, model, key)
 			if err != nil {
 				return err
 			}
@@ -75,6 +87,7 @@ func newToolCheckCmd() *cobra.Command {
 	}
 	cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002")
 	cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one")
+	cmd.Flags().StringVar(&key, "key", "", "the engine's key (default: the admin API's, for the replica at --endpoint)")
 	cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow")
 	return cmd
 }
@@ -102,7 +115,7 @@ type ToolProbe struct {
 	Raw     string
 }
 
-func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) {
+func probeToolCall(ctx context.Context, endpoint, model, key string) (ToolProbe, error) {
 	// A question with one obvious tool answer and no other sensible reply, so a
 	// model that can use tools has no excuse not to.
 	body := map[string]any{
@@ -138,6 +151,9 @@ func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, erro
 		return ToolProbe{}, err
 	}
 	req.Header.Set("Content-Type", "application/json")
+	if key != "" {
+		req.Header.Set("Authorization", "Bearer "+key)
+	}
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
@@ -150,6 +166,10 @@ func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, erro
 		return ToolProbe{}, err
 	}
 	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusUnauthorized {
+			return ToolProbe{}, fmt.Errorf("%s answered 401: the engine's key is the admin API's, "+
+				"for the replica `llmgw ctl replicas` lists at --endpoint; or pass it with --key", url)
+		}
 		return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300))
 	}
 	return readToolProbe(raw)

internal/cli/toolcheck_test.go

@@ -1,10 +1,65 @@
 package cli
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
 )
 
+// toolCheck runs the command against an engine that answers only its own key,
+// with an admin API that lists the replica there with that key.
+func toolCheck(t *testing.T, args ...string) (string, error) {
+	t.Helper()
+	engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer engine-key" {
+			http.Error(w, `{"error":{"message":"Invalid API Key"}}`, http.StatusUnauthorized)
+			return
+		}
+		fmt.Fprint(w, `{"choices":[{"message":{"tool_calls":[{"id":"c1","type":"function",
+		  "function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`)
+	}))
+	admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = json.NewEncoder(w).Encode([]adminapi.ReplicaView{
+			{ID: "r1", Model: "m", Profile: "gpu", Endpoint: engine.URL, EngineKey: "engine-key"},
+		})
+	}))
+	t.Cleanup(func() { engine.Close(); admin.Close() })
+	t.Setenv("LLMGW_ADMIN_TOKEN", "t")
+	cmd := newToolCheckCmd(func() *ctlClient { return newCtlClient(admin.URL) })
+	var out bytes.Buffer
+	cmd.SetOut(&out)
+	cmd.SetErr(&out)
+	cmd.SetArgs(append([]string{"--endpoint", engine.URL}, args...))
+	err := cmd.ExecuteContext(context.Background())
+	return out.String(), err
+}
+
+func TestToolCheckAsksWithTheEngineKeyTheAdminAPIGives(t *testing.T) {
+	// Every replica's engine demands its key, and the check was sent without
+	// one: it failed with a 401 on every engine in the cluster.
+	out, err := toolCheck(t)
+	if err != nil {
+		t.Fatalf("%v\n%s", err, out)
+	}
+	if !strings.Contains(out, "tool calls work") {
+		t.Errorf("unexpected report:\n%s", out)
+	}
+}
+
+func TestToolCheckSaysWhereTheEngineKeyComesFrom(t *testing.T) {
+	out, err := toolCheck(t, "--key", "wrong")
+	if err == nil || !strings.Contains(err.Error(), "401") || !strings.Contains(err.Error(), "--key") {
+		t.Fatalf("expected the refusal explained, got %v\n%s", err, out)
+	}
+}
+
 func TestAStructuredToolCallIsRecognised(t *testing.T) {
 	p, err := readToolProbe([]byte(`{"choices":[{"message":{"role":"assistant",
 	  "tool_calls":[{"id":"c1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Istanbul\"}"}}]}}]}`))