Serve vision models with their projector and original weights on vLLM; group the mirror by version

model serve takes --projector, fetched beside the weights and loaded by
llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp,
which fetches the weights with their config and tokenizer and runs them on
vLLM. The console offers both, F switching between GGUF and original
weights. The mirror view is one row per version with its files on Enter.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 00:42:59 +02:00
1 parent 7aefb236d6
commit a16097cfe8
12 files changed
+509 -81

No files matched your search

+82 -25
View File
@@ -62,26 +62,75 @@ func (m *consoleModel) mirrorCall(title string, do func(context.Context, *mirror
}
}
// usedBy names the catalog's models whose source is this file.
func (m consoleModel) usedBy(base string, s mirror.Status) []string {
u := mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(base)
dir := strings.TrimSuffix(mirror.Entry{Model: s.Model, Version: s.Version}.URL(base), "/") + "/"
// mirrorGroup is one version of one model on the mirror: a file, or the set
// of files a model of several is fetched as.
type mirrorGroup struct {
model, version string
files []mirror.Status
}
func (g mirrorGroup) bytes() (done, total int64) {
for _, f := range g.files {
done += f.BytesDone
total += max(f.BytesTotal, f.BytesDone)
}
return done, total
}
// source is what a model's definition names for it: the file, or the
// directory of a set.
func (g mirrorGroup) source(base string) string {
if len(g.files) == 1 {
return mirror.Entry{Model: g.model, Version: g.version, File: g.files[0].File}.URL(base)
}
return strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/"
}
func (m consoleModel) mirrorGroups() []mirrorGroup {
var out []mirrorGroup
at := map[string]int{}
for _, st := range m.mirror.res.Models {
k := st.Model + "/" + st.Version
i, ok := at[k]
if !ok {
i = len(out)
at[k] = i
out = append(out, mirrorGroup{model: st.Model, version: st.Version})
}
out[i].files = append(out[i].files, st)
}
return out
}
// usedBy names the catalog's models that run from a version's files.
func (m consoleModel) usedBy(base string, g mirrorGroup) []string {
dir := strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/"
var names []string
for _, md := range m.data.models {
if md.Source == u || strings.HasPrefix(md.Source, dir) {
if strings.HasPrefix(md.Source, dir) {
names = append(names, md.Name)
}
}
return names
}
func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog {
what := s.Model + "/" + s.Version
used := m.usedBy(m.mirror.res.Base, s)
items := []menuItem{{"Where nodes fetch it", func(m *consoleModel) tea.Cmd {
func (m consoleModel) mirrorMenu(g mirrorGroup) *menuDialog {
what := g.model + "/" + g.version
used := m.usedBy(m.mirror.res.Base, g)
items := []menuItem{{"Its files, and where nodes fetch them", func(m *consoleModel) tea.Cmd {
var b strings.Builder
for _, f := range g.files {
sum := ""
if f.SHA256 != "" {
sum = sDim.Render(" sha256 " + f.SHA256[:min(12, len(f.SHA256))])
}
fmt.Fprintf(&b, "%-50s %9s%s\n", truncate(f.File, 50), bytesShort(max(f.BytesTotal, f.BytesDone)), sum)
if f.Source != "" {
b.WriteString(sDim.Render(" from "+truncate(f.Source, 90)) + "\n")
}
}
m.dialog, m.dialogTitle = "error", what
m.dialogText = mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(m.mirror.res.Base) +
"\n\n" + sDim.Render("the source to give a model definition · any key")
m.dialogText = b.String() + "\nA model's source: " + g.source(m.mirror.res.Base) + "\n\n" + sDim.Render("any key")
return nil
}}}
items = append(items, menuItem{"Remove from the mirror…", func(m *consoleModel) tea.Cmd {
@@ -93,7 +142,7 @@ func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog {
m.confirmD = &confirmDialog{title: "Remove " + what, text: text,
yes: func(m *consoleModel) tea.Cmd {
return m.mirrorCall("Remove "+what, func(ctx context.Context, c *mirror.Client) (string, error) {
removed, err := c.Remove(ctx, s.Model, s.Version)
removed, err := c.Remove(ctx, g.model, g.version)
return fmt.Sprintf("Removed %d file(s).", len(removed)), err
})
}}
@@ -139,24 +188,32 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
}
head := "\n " + sDim.Render("Nodes fetch from "+m.mirror.res.Base) + "\n\n"
var rows [][]string
for _, s := range m.mirror.res.Models {
state := string(s.State)
switch s.State {
case mirror.Fetching:
if p, ok := s.Percent(); ok {
state = sWarn.Render(fmt.Sprintf("fetching %d%%", p))
} else {
state = sWarn.Render("fetching " + bytesShort(s.BytesDone))
for _, g := range m.mirrorGroups() {
done, total := g.bytes()
state := "present"
for _, f := range g.files {
switch f.State {
case mirror.Fetching:
state = sWarn.Render("fetching")
if total > 0 {
state = sWarn.Render(fmt.Sprintf("fetching %d%%", done*100/total))
}
case mirror.Failed:
state = sBad.Render("failed")
}
if f.State != mirror.Present {
break
}
case mirror.Failed:
state = sBad.Render("failed")
}
size := bytesShort(s.BytesDone)
used := strings.Join(m.usedBy(m.mirror.res.Base, s), ", ")
what := g.files[0].File
if len(g.files) > 1 {
what = fmt.Sprintf("%d files", len(g.files))
}
used := strings.Join(m.usedBy(m.mirror.res.Base, g), ", ")
if used == "" {
used = sDim.Render("unused")
}
rows = append(rows, []string{s.Model, s.Version, s.File, state, size, used})
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
}
if len(rows) == 0 {
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys