Let a model say what it is for
A gateway serving nine models offered nine names and nothing to choose between them. /v1/models is the only view most clients ever have, and it listed ids. Models carry a description now: written when the model is added, shown on the dashboard and sent to clients in /v1/models. Written rather than derived. A registry offers a pipeline tag and a wall of README, and neither answers the question somebody picking from a list is asking — which of these should I send this request to. "General chat, strong Swedish; slower than the 8B" is not in any metadata field. It is optional, and absent means absent rather than an empty string in the response: a model nobody described should read as undescribed, not as described with nothing. On the dashboard it sits under its row rather than in a column. A sentence in a table cell either wraps every other column into a stripe or gets cut off. Co-Authored-By: Claude Opus 5 (1M context) <moreply@anthropic.com>
This commit is contained in:
1 parent
e0ea4f6be9
commit
d998ea7ef5
8 files changed
+86
-11
No files matched your search
@@ -84,6 +84,7 @@ rather than repeated.
|
|||||||
| `timeout` | `time.Duration` | [Mirror](#mirror) |
|
| `timeout` | `time.Duration` | [Mirror](#mirror) |
|
||||||
| `name` | `string` | [Model](#model) |
|
| `name` | `string` | [Model](#model) |
|
||||||
| `aliases` | `[]string` | [Model](#model) |
|
| `aliases` | `[]string` | [Model](#model) |
|
||||||
|
| `description` | `string` | [Model](#model) |
|
||||||
| `source` | `string` | [Model](#model) |
|
| `source` | `string` | [Model](#model) |
|
||||||
| `profiles` | `[]Profile` | [Model](#model) |
|
| `profiles` | `[]Profile` | [Model](#model) |
|
||||||
| `scaling` | `Scaling` | [Model](#model) |
|
| `scaling` | `Scaling` | [Model](#model) |
|
||||||
@@ -632,6 +633,21 @@ arriving long after this deadline has passed.
|
|||||||
|
|
||||||
`[]string`
|
`[]string`
|
||||||
|
|
||||||
|
### `description`
|
||||||
|
|
||||||
|
`string`
|
||||||
|
|
||||||
|
Description is what this model is for, in a line.
|
||||||
|
|
||||||
|
Written by whoever added it, because nobody else can say it. A registry
|
||||||
|
offers a pipeline tag and a wall of README, and neither answers the
|
||||||
|
question a person picking from a list is asking: which of these nine
|
||||||
|
names should I send this request to.
|
||||||
|
|
||||||
|
It reaches clients through /v1/models, which is the only view most of
|
||||||
|
them ever see. A gateway serving nine models by name alone makes that
|
||||||
|
choice by guesswork.
|
||||||
|
|
||||||
### `source`
|
### `source`
|
||||||
|
|
||||||
`string`
|
`string`
|
||||||
|
|||||||
@@ -108,15 +108,17 @@ type NodeView struct {
|
|||||||
|
|
||||||
// ModelView is a model as `ctl models` shows it.
|
// ModelView is a model as `ctl models` shows it.
|
||||||
type ModelView struct {
|
type ModelView struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Aliases []string `json:"aliases,omitempty"`
|
// Description is what the model is for, when somebody said.
|
||||||
Source string `json:"source"`
|
Description string `json:"description,omitempty"`
|
||||||
Min int `json:"min_replicas"`
|
Aliases []string `json:"aliases,omitempty"`
|
||||||
Max int `json:"max_replicas"`
|
Source string `json:"source"`
|
||||||
Ready int `json:"ready_replicas"`
|
Min int `json:"min_replicas"`
|
||||||
Total int `json:"total_replicas"`
|
Max int `json:"max_replicas"`
|
||||||
Fallbacks []string `json:"fallbacks,omitempty"`
|
Ready int `json:"ready_replicas"`
|
||||||
Profiles []string `json:"profiles"`
|
Total int `json:"total_replicas"`
|
||||||
|
Fallbacks []string `json:"fallbacks,omitempty"`
|
||||||
|
Profiles []string `json:"profiles"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReplicaView is a replica as `ctl replicas` shows it.
|
// ReplicaView is a replica as `ctl replicas` shows it.
|
||||||
@@ -260,7 +262,7 @@ func (a *API) Models() []ModelView {
|
|||||||
out := make([]ModelView, 0)
|
out := make([]ModelView, 0)
|
||||||
for _, m := range a.ctl.Catalog().Models() {
|
for _, m := range a.ctl.Catalog().Models() {
|
||||||
v := ModelView{
|
v := ModelView{
|
||||||
Name: m.Name, Aliases: m.Aliases, Source: m.Source,
|
Name: m.Name, Description: m.Description, Aliases: m.Aliases, Source: m.Source,
|
||||||
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
Min: m.Scaling.Min, Max: m.Scaling.Max,
|
||||||
Ready: ready[m.Name], Total: total[m.Name],
|
Ready: ready[m.Name], Total: total[m.Name],
|
||||||
Fallbacks: m.Fallbacks,
|
Fallbacks: m.Fallbacks,
|
||||||
|
|||||||
@@ -123,6 +123,17 @@ func cpuCapable(engine string) bool { return engine == "llamacpp" }
|
|||||||
type Model struct {
|
type Model struct {
|
||||||
Name string `json:"name,omitempty" koanf:"name"`
|
Name string `json:"name,omitempty" koanf:"name"`
|
||||||
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
|
Aliases []string `json:"aliases,omitempty" koanf:"aliases"`
|
||||||
|
// Description is what this model is for, in a line.
|
||||||
|
//
|
||||||
|
// Written by whoever added it, because nobody else can say it. A registry
|
||||||
|
// offers a pipeline tag and a wall of README, and neither answers the
|
||||||
|
// question a person picking from a list is asking: which of these nine
|
||||||
|
// names should I send this request to.
|
||||||
|
//
|
||||||
|
// It reaches clients through /v1/models, which is the only view most of
|
||||||
|
// them ever see. A gateway serving nine models by name alone makes that
|
||||||
|
// choice by guesswork.
|
||||||
|
Description string `json:"description,omitempty" koanf:"description"`
|
||||||
// Source is the weights path or a HuggingFace id.
|
// Source is the weights path or a HuggingFace id.
|
||||||
Source string `json:"source,omitempty" koanf:"source"`
|
Source string `json:"source,omitempty" koanf:"source"`
|
||||||
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
Profiles []Profile `json:"profiles,omitempty" koanf:"profiles"`
|
||||||
|
|||||||
@@ -715,6 +715,12 @@ func (r *Router) handleModels(w http.ResponseWriter, _ *http.Request) {
|
|||||||
if m.MaxOutput > 0 {
|
if m.MaxOutput > 0 {
|
||||||
entry["max_output_tokens"] = m.MaxOutput
|
entry["max_output_tokens"] = m.MaxOutput
|
||||||
}
|
}
|
||||||
|
// Not part of the OpenAI shape, and the reason to send it anyway is
|
||||||
|
// that this list is the only view most clients have: nine names and
|
||||||
|
// nothing to choose between them.
|
||||||
|
if m.Description != "" {
|
||||||
|
entry["description"] = m.Description
|
||||||
|
}
|
||||||
entry["capabilities"] = map[string]bool{
|
entry["capabilities"] = map[string]bool{
|
||||||
"tools": m.Capabilities.Tools, "vision": m.Capabilities.Vision,
|
"tools": m.Capabilities.Tools, "vision": m.Capabilities.Vision,
|
||||||
"embeddings": m.Capabilities.Embeddings,
|
"embeddings": m.Capabilities.Embeddings,
|
||||||
|
|||||||
@@ -169,6 +169,7 @@ func gpus(fleet Source) int {
|
|||||||
func buildModel(get func(string) string) (catalog.Model, error) {
|
func buildModel(get func(string) string) (catalog.Model, error) {
|
||||||
m := catalog.Model{
|
m := catalog.Model{
|
||||||
Name: strings.TrimSpace(get("name")),
|
Name: strings.TrimSpace(get("name")),
|
||||||
|
Description: strings.TrimSpace(get("description")),
|
||||||
Source: strings.TrimSpace(get("source")),
|
Source: strings.TrimSpace(get("source")),
|
||||||
RoutingProfile: "chat",
|
RoutingProfile: "chat",
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -295,3 +295,32 @@ func TestAnEstimateIsNotMadeWhenTheSizeIsUnknown(t *testing.T) {
|
|||||||
t.Errorf("an unknown size should leave no figure; got %s", body)
|
t.Errorf("an unknown size should leave no figure; got %s", body)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestADescriptionReachesTheCatalog(t *testing.T) {
|
||||||
|
cat := &fakeCatalog{}
|
||||||
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
||||||
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
||||||
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
||||||
|
"description": {"code completion and review; 7B, fits the large node"},
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(cat.added) != 1 {
|
||||||
|
t.Fatalf("expected one model, got %d", len(cat.added))
|
||||||
|
}
|
||||||
|
// The line a client sees in /v1/models, which for most of them is the only
|
||||||
|
// view of what this gateway serves.
|
||||||
|
if cat.added[0].Description != "code completion and review; 7B, fits the large node" {
|
||||||
|
t.Errorf("the description was lost: %q", cat.added[0].Description)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAModelWithNoDescriptionIsStillAccepted(t *testing.T) {
|
||||||
|
cat := &fakeCatalog{}
|
||||||
|
mirrorPost(t, servePage(t, nil, cat, 0), "/admin/mirror/serve", url.Values{
|
||||||
|
"csrf": {"tok"}, "name": {"coder"}, "source": {"http://x/m.gguf"},
|
||||||
|
"engine": {"llamacpp"}, "replicas": {"1"}, "context_length": {"4096"},
|
||||||
|
})
|
||||||
|
if len(cat.added) != 1 || cat.added[0].Description != "" {
|
||||||
|
t.Errorf("a description is optional: %+v", cat.added)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -100,6 +100,12 @@ reachable from this machine.</p>
|
|||||||
<input type="text" name="aliases" placeholder="coder-7b"></label>
|
<input type="text" name="aliases" placeholder="coder-7b"></label>
|
||||||
<label>Replicas <input type="text" name="replicas" value="1" required></label>
|
<label>Replicas <input type="text" name="replicas" value="1" required></label>
|
||||||
</div>
|
</div>
|
||||||
|
{{/* Reaches clients through /v1/models, which for most of them is the only
|
||||||
|
view of what this gateway serves. Nine names and nothing to choose
|
||||||
|
between them is a choice made by guesswork. */}}
|
||||||
|
<label>What it is for <span class="opt">shown to clients in /v1/models</span>
|
||||||
|
<input type="text" name="description"
|
||||||
|
placeholder="general chat, strong Swedish; 14B, slower than the 8B"></label>
|
||||||
<div class="field-row">
|
<div class="field-row">
|
||||||
<label>Context a request may use
|
<label>Context a request may use
|
||||||
<input type="text" name="context_length" value="4096" required></label>
|
<input type="text" name="context_length" value="4096" required></label>
|
||||||
|
|||||||
@@ -129,7 +129,11 @@ requested whenever a fallback served it.</p>
|
|||||||
<td class="num">{{.Min}}–{{.Max}}</td>
|
<td class="num">{{.Min}}–{{.Max}}</td>
|
||||||
<td>{{join .Profiles}}</td>
|
<td>{{join .Profiles}}</td>
|
||||||
<td>{{join .Fallbacks}}</td>
|
<td>{{join .Fallbacks}}</td>
|
||||||
</tr>{{end}}</table>
|
</tr>
|
||||||
|
{{/* Under the row rather than in a column: a sentence in a table cell either
|
||||||
|
wraps every other column into a stripe or gets cut off. */}}
|
||||||
|
{{if .Description}}<tr><td colspan="6" class="url-row">{{.Description}}</td></tr>{{end}}
|
||||||
|
{{end}}</table>
|
||||||
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
{{else}}<p class="empty">The catalog is empty.</p>{{end}}
|
||||||
|
|
||||||
{{if .Unplaced}}
|
{{if .Unplaced}}
|
||||||
|
|||||||
Reference in new issue
Block a user