llmgw ctl tool-check asks with the engine's key
Every replica's engine demands its own key, and the check sent none, so it failed with a 401 on every engine in the cluster. The key is asked of the admin API for the replica at --endpoint, as bench does, or given with --key. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b0273ffb5f
commit
0bf7e46fbf
3 files changed
+79
-4
No files matched your search
@@ -34,10 +34,11 @@ import (
|
||||
// wrapper — left it in the content. A differently trained model fixes that. A
|
||||
// bigger one from the same family does not: qwen2.5-coder-7b invented
|
||||
// <function-call> and failed the same way.
|
||||
func newToolCheckCmd() *cobra.Command {
|
||||
func newToolCheckCmd(client func() *ctlClient) *cobra.Command {
|
||||
var (
|
||||
endpoint string
|
||||
model string
|
||||
key string
|
||||
timeout time.Duration
|
||||
)
|
||||
|
||||
@@ -57,10 +58,21 @@ func newToolCheckCmd() *cobra.Command {
|
||||
if endpoint == "" {
|
||||
return fmt.Errorf("--endpoint is required, e.g. http://10.0.0.145:20002")
|
||||
}
|
||||
// A replica's engine demands its own key, and refused the check
|
||||
// with a 401 when none was sent. Asked of the admin API, as bench
|
||||
// does; an endpoint that is no replica's is asked without one.
|
||||
if key == "" {
|
||||
reps, err := listReplicas(cmd, client)
|
||||
if err != nil {
|
||||
cmd.PrintErrf("The admin API could not be asked for the engine key of the replica at "+
|
||||
"--endpoint (%v); an engine that demands one will refuse. Pass --key to give it.\n", err)
|
||||
}
|
||||
key = replicaKey(reps, endpoint)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(cmd.Context(), timeout)
|
||||
defer cancel()
|
||||
|
||||
res, err := probeToolCall(ctx, endpoint, model)
|
||||
res, err := probeToolCall(ctx, endpoint, model, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -75,6 +87,7 @@ func newToolCheckCmd() *cobra.Command {
|
||||
}
|
||||
cmd.Flags().StringVar(&endpoint, "endpoint", "", "engine base URL, e.g. http://10.0.0.145:20002")
|
||||
cmd.Flags().StringVar(&model, "model", "", "model name to send, when the engine needs one")
|
||||
cmd.Flags().StringVar(&key, "key", "", "the engine's key (default: the admin API's, for the replica at --endpoint)")
|
||||
cmd.Flags().DurationVar(&timeout, "timeout", 5*time.Minute, "how long to wait; CPU inference is slow")
|
||||
return cmd
|
||||
}
|
||||
@@ -102,7 +115,7 @@ type ToolProbe struct {
|
||||
Raw string
|
||||
}
|
||||
|
||||
func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, error) {
|
||||
func probeToolCall(ctx context.Context, endpoint, model, key string) (ToolProbe, error) {
|
||||
// A question with one obvious tool answer and no other sensible reply, so a
|
||||
// model that can use tools has no excuse not to.
|
||||
body := map[string]any{
|
||||
@@ -138,6 +151,9 @@ func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, erro
|
||||
return ToolProbe{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if key != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
@@ -150,6 +166,10 @@ func probeToolCall(ctx context.Context, endpoint, model string) (ToolProbe, erro
|
||||
return ToolProbe{}, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
if resp.StatusCode == http.StatusUnauthorized {
|
||||
return ToolProbe{}, fmt.Errorf("%s answered 401: the engine's key is the admin API's, "+
|
||||
"for the replica `llmgw ctl replicas` lists at --endpoint; or pass it with --key", url)
|
||||
}
|
||||
return ToolProbe{}, fmt.Errorf("%s returned %d: %s", url, resp.StatusCode, trim(string(raw), 300))
|
||||
}
|
||||
return readToolProbe(raw)
|
||||
|
||||
Reference in new issue
Block a user