Serve a model made of several files
The mirror would happily hold a four-shard model that nothing could serve. A catalog entry names one source, the agent fetched one URL, and so a version of several files had no way to be described — which is every safetensors model, and every GGUF too large to ship whole. A source ending in a slash names a version rather than a file. The agent fetches every file in it into one directory and hands the engine the directory: vLLM reads the shards and the config out of the folder, and llama.cpp is given the first part and finds the rest beside it. The trailing slash is the whole signal. A flag beside the source would be a second place for the two to disagree, and a URL already distinguishes a directory from a file everywhere else. The mirror answers a version with a manifest instead of a file listing. A plain file server gives HTML there, and something that has to know what a version consists of would be left parsing it. It is on the byte listener, not the management API: fetching weights is the one thing every node does, and making it depend on the administrative port would undo the split that keeps that port closeable. Built from the directory rather than a file written beside it. A manifest on disk is a second record of what is there, and the two disagree the first time somebody copies a file in by hand — which the mirror explicitly allows. The whole set or nothing. A directory with three of four shards loads as a broken model rather than failing to load, so a completion marker is written last and an interrupted fetch starts the missing files again rather than being found and served. Files already there at the right size are not fetched twice. Progress is reported under the version's address, not each file's: the shards waiting on it asked for the version, and progress under a file's own URL reaches nobody. A filename from the manifest is refused if it contains a separator. The mirror is a place an operator copies files into by hand, so what it reports is not automatically safe to join onto a path. An HTML listing where a manifest was expected says so. "invalid character '<'" sends somebody looking at the wrong thing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
c95cfbe426
commit
dfbb7557dd
10 files changed
+554
-23
No files matched your search
@@ -32,6 +32,8 @@ type serveForm struct {
|
||||
// be written later. Named for the hardware rather than the model, because
|
||||
// a profile is about where a model runs.
|
||||
Profile string
|
||||
// Files is how many files the version holds, above one for a set.
|
||||
Files int
|
||||
// MinMemoryGB is the host RAM to ask for, estimated from the weights.
|
||||
//
|
||||
// Filled in rather than left blank. Blank writes no constraint at all, and
|
||||
@@ -63,16 +65,26 @@ func planServe(st mirror.Status, files []mirror.Status, base string,
|
||||
Profile: "cpu",
|
||||
}
|
||||
|
||||
// A catalogue entry names one source. A version made of several files has
|
||||
// no single one, so an entry for it would be an entry for one shard — which
|
||||
// loads as a broken model rather than failing to load.
|
||||
// A version of several files is served as the version, not as one of them.
|
||||
// The source is the directory — a trailing slash — and the agent fetches
|
||||
// every file in it and hands the engine the folder.
|
||||
if len(files) > 1 {
|
||||
f.Refusal = fmt.Sprintf(
|
||||
"this version is %d files, and a catalogue entry names one source. "+
|
||||
"Serve a single-file build (a GGUF), or give the engine the "+
|
||||
"repository id so it fetches the shards itself",
|
||||
len(files))
|
||||
return f
|
||||
f.Source = strings.TrimSuffix(
|
||||
mirror.Entry{Model: st.Model, Version: st.Version}.URL(base), "/") + "/"
|
||||
f.Files = len(files)
|
||||
// From whichever file says, because a version is a mixture only when
|
||||
// somebody has put one there by hand. The weights decide the engine.
|
||||
for _, one := range files {
|
||||
if e := engineFor(one.File); e != "" {
|
||||
f.Engine = e
|
||||
break
|
||||
}
|
||||
}
|
||||
var total int64
|
||||
for _, one := range files {
|
||||
total += one.BytesDone
|
||||
}
|
||||
st.BytesTotal = total
|
||||
}
|
||||
|
||||
switch f.Engine {
|
||||
|
||||
@@ -76,22 +76,40 @@ func TestAGGUFCanBeServedStraightFromTheList(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShardedVersionIsRefusedWithTheReason(t *testing.T) {
|
||||
func TestAShardedVersionIsServedAsTheWholeVersion(t *testing.T) {
|
||||
models := []mirror.Status{
|
||||
held("llama", "v1", "model-00001-of-00002.safetensors"),
|
||||
held("llama", "v1", "model-00002-of-00002.safetensors"),
|
||||
}
|
||||
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8),
|
||||
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
|
||||
"/admin/mirror?serve=llama:v1").Body.String()
|
||||
|
||||
// A catalogue entry names one source, so an entry for a sharded version is
|
||||
// an entry for one shard — which loads as a broken model rather than
|
||||
// failing to load.
|
||||
if !strings.Contains(body, "a catalogue entry names one source") {
|
||||
t.Errorf("the refusal should say why; got %s", body)
|
||||
// A catalogue entry names one source, and for a version of several files
|
||||
// that source is the version. Refusing it meant a sharded model could be
|
||||
// mirrored and never served — which is most of what is not a GGUF.
|
||||
if !strings.Contains(body, `value="http://mirror:8081/llama/v1/"`) {
|
||||
t.Errorf("the source should be the version directory; got %s", body)
|
||||
}
|
||||
if strings.Contains(body, `action="/admin/mirror/serve"`) {
|
||||
t.Error("no form should be offered for something that cannot be served")
|
||||
if !strings.Contains(body, "The whole version, 2 files") {
|
||||
t.Error("the form should say it is fetching a set")
|
||||
}
|
||||
if !strings.Contains(body, `action="/admin/mirror/serve"`) {
|
||||
t.Error("it should be servable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheEngineForASetComesFromItsWeights(t *testing.T) {
|
||||
// A version is a mixture only when somebody has put a file there by hand.
|
||||
// config.json decides nothing; the safetensors do.
|
||||
models := []mirror.Status{
|
||||
held("llama", "v1", "config.json"),
|
||||
held("llama", "v1", "model-00001-of-00002.safetensors"),
|
||||
}
|
||||
body := mirrorGet(t, servePage(t, models, &fakeCatalog{}, 8, "vllm"),
|
||||
"/admin/mirror?serve=llama:v1").Body.String()
|
||||
|
||||
if !strings.Contains(body, `name="engine" value="vllm"`) {
|
||||
t.Errorf("the weights should pick the engine; got %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -93,7 +93,9 @@ reachable from this machine.</p>
|
||||
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
||||
<input type="hidden" name="source" value="{{.Source}}">
|
||||
<input type="hidden" name="engine" value="{{.Engine}}">
|
||||
<p class="sub">Serving <code>{{.Source}}</code> with <strong>{{.Engine}}</strong>.</p>
|
||||
<p class="sub">Serving <code>{{.Source}}</code> with <strong>{{.Engine}}</strong>.
|
||||
{{if .Files}}The whole version, {{.Files}} files: the node fetches them into one
|
||||
directory and the engine is given the directory.{{end}}</p>
|
||||
<div class="field-row">
|
||||
<label>Name <input type="text" name="name" value="{{.Model}}" required></label>
|
||||
<label>Aliases <span class="opt">comma separated</span>
|
||||
|
||||
Reference in new issue
Block a user