// Package client talks to llmgw's admin API over HTTP with the admin token. // // Its own types rather than the gateway's: the provider is a separate module // so the gateway does not carry Terraform's dependencies, and the gateway's // packages are internal to it. What keeps the two in step is the OpenAPI // document the gateway generates, docs/admin-api.openapi.json, which a test // here checks every endpoint and field this client uses against. package client import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "slices" "strings" "time" ) // Endpoints are the admin API's endpoints this client calls, as the OpenAPI // document names them. A request for any other is refused before it is // sent, so this list is the whole of what the contract test has to check. var Endpoints = []string{ "GET /api/v1/models", "POST /api/v1/models", "GET /api/v1/models/{name}", "DELETE /api/v1/models/{name}", "POST /api/v1/models/{name}/scale", "POST /api/v1/models/{name}/class", "POST /api/v1/models/{name}/owner", "POST /api/v1/models/{name}/adapters", "GET /api/v1/nodes", "GET /api/v1/quotas", "PUT /api/v1/quotas", "DELETE /api/v1/quotas/{scope}", "GET /api/v1/guard", "POST /api/v1/guard/attachments", "DELETE /api/v1/guard/attachments", } // Client is a connection to one control plane's admin API. type Client struct { base *url.URL token string actor string http *http.Client } // New builds a client for the admin listener at endpoint, such as // http://127.0.0.1:9090. Actor is who changes are recorded as made by. func New(endpoint, token, actor string) (*Client, error) { u, err := url.Parse(strings.TrimRight(endpoint, "/")) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { return nil, fmt.Errorf("endpoint %q is not an http or https address", endpoint) } if token == "" { return nil, errors.New("no admin token: set token, token_file or LLMGW_ADMIN_TOKEN") } return &Client{base: u, token: token, actor: actor, http: &http.Client{Timeout: 2 * time.Minute}}, nil } // Error is the API refusing a request, with what it said. type Error struct { Method, Path string Status int Message string } func (e *Error) Error() string { return fmt.Sprintf("%s %s: %d %s", e.Method, e.Path, e.Status, e.Message) } // IsNotFound reports whether err is the API saying there is no such thing. func IsNotFound(err error) bool { var e *Error return errors.As(err, &e) && e.Status == http.StatusNotFound } // do sends one request to an endpoint named as in Endpoints, with args put // in its path parameters in order. func (c *Client) do(ctx context.Context, endpoint string, args []string, in, out any) error { if !slices.Contains(Endpoints, endpoint) { return fmt.Errorf("the client does not call %s: add it to Endpoints", endpoint) } method, tmpl, _ := strings.Cut(endpoint, " ") p := tmpl for _, a := range args { i, j := strings.Index(p, "{"), strings.Index(p, "}") if i < 0 || j < i { return fmt.Errorf("%s takes fewer parameters than given", endpoint) } p = p[:i] + url.PathEscape(a) + p[j+1:] } if strings.Contains(p, "{") { return fmt.Errorf("%s takes more parameters than given", endpoint) } var body io.Reader if in != nil { b, err := json.Marshal(in) if err != nil { return err } body = bytes.NewReader(b) } plain, err := url.PathUnescape(p) if err != nil { return err } u := *c.base u.Path = strings.TrimRight(u.Path, "/") + plain // RawPath keeps an escaped slash in a name escaped: a model or a scope // called Acme/ml is one segment, not two. u.RawPath = strings.TrimRight(c.base.EscapedPath(), "/") + p req, err := http.NewRequestWithContext(ctx, method, u.String(), body) if err != nil { return err } req.Header.Set("Authorization", "Bearer "+c.token) req.Header.Set("Accept", "application/json") // Every change must say it is JSON, even one without a body: the admin // listener refuses any other, as a page's form post cannot set it. if method != http.MethodGet { req.Header.Set("Content-Type", "application/json") } if c.actor != "" { req.Header.Set("X-Llmgw-Actor", c.actor) } resp, err := c.http.Do(req) if err != nil { return err } defer resp.Body.Close() b, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) if err != nil { return err } if resp.StatusCode >= 300 { var e struct { Error string `json:"error"` } msg := strings.TrimSpace(string(b)) if json.Unmarshal(b, &e) == nil && e.Error != "" { msg = e.Error } return &Error{Method: method, Path: tmpl, Status: resp.StatusCode, Message: msg} } if out != nil { if err := json.Unmarshal(b, out); err != nil { return fmt.Errorf("%s %s: the answer is not what was expected: %w", method, tmpl, err) } } return nil } // --- models --- // Model is a model's definition, as the catalog holds it. type Model struct { Name string `json:"name,omitempty"` Aliases []string `json:"aliases,omitempty"` Description string `json:"description,omitempty"` Source string `json:"source,omitempty"` Profiles []Profile `json:"profiles,omitempty"` Scaling Scaling `json:"scaling,omitzero"` Class string `json:"class,omitempty"` Owner string `json:"owner,omitempty"` External *External `json:"external,omitempty"` Adapters []Adapter `json:"adapters,omitempty"` Fallbacks []string `json:"fallbacks,omitempty"` RoutingProfile string `json:"routing_profile,omitempty"` ContextLength int64 `json:"context_length,omitempty"` MaxOutput int64 `json:"max_output,omitempty"` Capabilities *Capabilities `json:"capabilities,omitempty"` } // Profile is how a model runs on one kind of node. type Profile struct { Name string `json:"name,omitempty"` Engine string `json:"engine,omitempty"` GPUClass string `json:"gpu_class,omitempty"` TP int64 `json:"tp,omitempty"` PP int64 `json:"pp,omitempty"` GPUsPerNode int64 `json:"gpus_per_node,omitempty"` Nodes int64 `json:"nodes,omitempty"` Args []string `json:"args,omitempty"` Env map[string]string `json:"env,omitempty"` MinMemoryGB int64 `json:"min_memory_gb,omitempty"` CPUCores int64 `json:"cpu_cores,omitempty"` MinDiskGB int64 `json:"min_disk_gb,omitempty"` NodeSelector map[string]string `json:"node_selector,omitempty"` Weight float64 `json:"weight,omitempty"` Priority int64 `json:"priority,omitempty"` StartupTimeout time.Duration `json:"startup_timeout,omitempty"` } // Scaling is how many replicas a model runs. type Scaling struct { Min int64 `json:"min,omitempty"` Max int64 `json:"max,omitempty"` IdleTimeout time.Duration `json:"idle_timeout,omitempty"` } // External is a model served by a provider outside the cluster. type External struct { URL string `json:"url"` Model string `json:"model"` APIKeyFile string `json:"api_key_file,omitempty"` } // Adapter is a LoRA adapter a model serves. type Adapter struct { Name string `json:"name"` Source string `json:"source"` Scale float64 `json:"scale,omitempty"` } // Capabilities are what a model can do beyond chat. type Capabilities struct { Tools bool `json:"tools,omitempty"` Vision bool `json:"vision,omitempty"` Embeddings bool `json:"embeddings,omitempty"` } // ModelView is a model as the listing shows it. type ModelView struct { Name string `json:"name"` Description string `json:"description,omitempty"` Aliases []string `json:"aliases,omitempty"` Source string `json:"source"` Min int64 `json:"min_replicas"` Max int64 `json:"max_replicas"` Ready int64 `json:"ready_replicas"` Total int64 `json:"total_replicas"` Desired int64 `json:"desired_replicas"` Parked bool `json:"parked,omitempty"` Reason string `json:"reason,omitempty"` Profiles []string `json:"profiles"` Class string `json:"class"` Owner string `json:"owner,omitempty"` External string `json:"external,omitempty"` } // ScaleRequest sets a model's replicas. type ScaleRequest struct { Min int64 `json:"min"` Max int64 `json:"max,omitempty"` IdleTimeout time.Duration `json:"idle_timeout,omitempty"` } // ClassRequest sets a model's capacity class. type ClassRequest struct { Class string `json:"class"` } // OwnerRequest sets a model's owner. type OwnerRequest struct { Owner string `json:"owner"` } // OwnerView is a model's owner, as scope:ID. type OwnerView struct { Model string `json:"model"` Owner string `json:"owner"` } // AdaptersRequest replaces a model's adapters. type AdaptersRequest struct { Adapters []Adapter `json:"adapters"` } // Models lists the catalog. func (c *Client) Models(ctx context.Context) ([]ModelView, error) { var out []ModelView return out, c.do(ctx, "GET /api/v1/models", nil, nil, &out) } // AddModel adds a model to the catalog. func (c *Client) AddModel(ctx context.Context, m Model) error { return c.do(ctx, "POST /api/v1/models", nil, m, nil) } // Model reads one model's definition, as it is in force. func (c *Client) Model(ctx context.Context, name string) (Model, error) { var out Model return out, c.do(ctx, "GET /api/v1/models/{name}", []string{name}, nil, &out) } // RemoveModel removes a model added at runtime. func (c *Client) RemoveModel(ctx context.Context, name string) error { return c.do(ctx, "DELETE /api/v1/models/{name}", []string{name}, nil, nil) } // Scale sets how many replicas a model runs. func (c *Client) Scale(ctx context.Context, name string, s ScaleRequest) error { return c.do(ctx, "POST /api/v1/models/{name}/scale", []string{name}, s, nil) } // SetClass sets a model's capacity class; empty goes back to its default. func (c *Client) SetClass(ctx context.Context, name, class string) error { return c.do(ctx, "POST /api/v1/models/{name}/class", []string{name}, ClassRequest{Class: class}, nil) } // SetOwner sets a model's owner, by path or ID; empty makes it shared. It // answers with the owner as scope:ID. func (c *Client) SetOwner(ctx context.Context, name, owner string) (string, error) { var out OwnerView err := c.do(ctx, "POST /api/v1/models/{name}/owner", []string{name}, OwnerRequest{Owner: owner}, &out) return out.Owner, err } // SetAdapters replaces a model's LoRA adapters. func (c *Client) SetAdapters(ctx context.Context, name string, adapters []Adapter) error { if adapters == nil { adapters = []Adapter{} } return c.do(ctx, "POST /api/v1/models/{name}/adapters", []string{name}, AdaptersRequest{Adapters: adapters}, nil) } // --- nodes --- // NodeView is a node as the listing shows it. type NodeView struct { ID string `json:"id"` Hostname string `json:"hostname,omitempty"` MgmtAddr string `json:"mgmt_addr,omitempty"` State string `json:"state"` Labels map[string]string `json:"labels,omitempty"` GPUTotal int64 `json:"gpu_total"` GPUFree int64 `json:"gpu_free"` MemoryGB int64 `json:"memory_gb,omitempty"` CPUCount int64 `json:"cpu_count,omitempty"` Engines []string `json:"engines,omitempty"` OS string `json:"os,omitempty"` Classes []string `json:"gpu_classes,omitempty"` Fabrics []string `json:"fabrics,omitempty"` AgentVersion string `json:"agent_version,omitempty"` Replicas int64 `json:"replicas"` } // Nodes lists the fleet. func (c *Client) Nodes(ctx context.Context) ([]NodeView, error) { var out []NodeView return out, c.do(ctx, "GET /api/v1/nodes", nil, nil, &out) } // --- quotas --- // Resources is an amount of the cluster; nil is not governed. type Resources struct { GPUs *int64 `json:"gpus,omitempty"` CPUCores *int64 `json:"cpu_cores,omitempty"` MemoryGB *int64 `json:"memory_gb,omitempty"` } // QuotaView is one scope's quota. type QuotaView struct { Scope string `json:"scope"` Name string `json:"name"` Guarantee Resources `json:"guarantee"` Limit Resources `json:"limit"` } // QuotaRequest sets a scope's quota, the scope by path or ID. type QuotaRequest struct { Scope string `json:"scope"` Guarantee Resources `json:"guarantee"` Limit Resources `json:"limit"` } // QuotaSet is the scope a quota was set on, as scope:ID. type QuotaSet struct { Scope string `json:"scope"` } // Quotas lists every capacity quota. func (c *Client) Quotas(ctx context.Context) ([]QuotaView, error) { var out []QuotaView return out, c.do(ctx, "GET /api/v1/quotas", nil, nil, &out) } // SetQuota sets a scope's quota whole, and answers with the scope as // scope:ID. func (c *Client) SetQuota(ctx context.Context, q QuotaRequest) (string, error) { var out QuotaSet err := c.do(ctx, "PUT /api/v1/quotas", nil, q, &out) return out.Scope, err } // ClearQuota removes a scope's quota. func (c *Client) ClearQuota(ctx context.Context, scope string) error { return c.do(ctx, "DELETE /api/v1/quotas/{scope}", []string{scope}, nil, nil) } // --- guardrails --- // Attachment is a policy attached to a scope. The API writes its fields by // their Go names. type Attachment struct { Scope string `json:"Scope"` ScopeID string `json:"ScopeID"` Policy string `json:"Policy"` } // GuardView is the policy library and its attachments. type GuardView struct { Policies []string `json:"policies"` Default []string `json:"default"` Attachments []Attachment `json:"attachments"` } // GuardAttachmentRequest names a policy and where it applies. type GuardAttachmentRequest struct { Target string `json:"target"` Policy string `json:"policy"` } // Guard reads the policies and where they are attached. func (c *Client) Guard(ctx context.Context) (GuardView, error) { var out GuardView return out, c.do(ctx, "GET /api/v1/guard", nil, nil, &out) } // Attach attaches a policy, answering with where it went by ID. func (c *Client) Attach(ctx context.Context, target, policy string) (Attachment, error) { var out Attachment err := c.do(ctx, "POST /api/v1/guard/attachments", nil, GuardAttachmentRequest{Target: target, Policy: policy}, &out) return out, err } // Detach takes a policy off. func (c *Client) Detach(ctx context.Context, target, policy string) error { return c.do(ctx, "DELETE /api/v1/guard/attachments", nil, GuardAttachmentRequest{Target: target, Policy: policy}, nil) }