Serve vision models with their projector and original weights on vLLM; group the mirror by version
model serve takes --projector, fetched beside the weights and loaded by llama.cpp with --mmproj, and --format safetensors with --gpu-class and --tp, which fetches the weights with their config and tokenizer and runs them on vLLM. The console offers both, F switching between GGUF and original weights. The mirror view is one row per version with its files on Enter. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7aefb236d6
commit
a16097cfe8
12 files changed
+509
-81
No files matched your search
+82
-25
@@ -62,26 +62,75 @@ func (m *consoleModel) mirrorCall(title string, do func(context.Context, *mirror
|
||||
}
|
||||
}
|
||||
|
||||
// usedBy names the catalog's models whose source is this file.
|
||||
func (m consoleModel) usedBy(base string, s mirror.Status) []string {
|
||||
u := mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(base)
|
||||
dir := strings.TrimSuffix(mirror.Entry{Model: s.Model, Version: s.Version}.URL(base), "/") + "/"
|
||||
// mirrorGroup is one version of one model on the mirror: a file, or the set
|
||||
// of files a model of several is fetched as.
|
||||
type mirrorGroup struct {
|
||||
model, version string
|
||||
files []mirror.Status
|
||||
}
|
||||
|
||||
func (g mirrorGroup) bytes() (done, total int64) {
|
||||
for _, f := range g.files {
|
||||
done += f.BytesDone
|
||||
total += max(f.BytesTotal, f.BytesDone)
|
||||
}
|
||||
return done, total
|
||||
}
|
||||
|
||||
// source is what a model's definition names for it: the file, or the
|
||||
// directory of a set.
|
||||
func (g mirrorGroup) source(base string) string {
|
||||
if len(g.files) == 1 {
|
||||
return mirror.Entry{Model: g.model, Version: g.version, File: g.files[0].File}.URL(base)
|
||||
}
|
||||
return strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/"
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorGroups() []mirrorGroup {
|
||||
var out []mirrorGroup
|
||||
at := map[string]int{}
|
||||
for _, st := range m.mirror.res.Models {
|
||||
k := st.Model + "/" + st.Version
|
||||
i, ok := at[k]
|
||||
if !ok {
|
||||
i = len(out)
|
||||
at[k] = i
|
||||
out = append(out, mirrorGroup{model: st.Model, version: st.Version})
|
||||
}
|
||||
out[i].files = append(out[i].files, st)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// usedBy names the catalog's models that run from a version's files.
|
||||
func (m consoleModel) usedBy(base string, g mirrorGroup) []string {
|
||||
dir := strings.TrimSuffix(mirror.Entry{Model: g.model, Version: g.version}.URL(base), "/") + "/"
|
||||
var names []string
|
||||
for _, md := range m.data.models {
|
||||
if md.Source == u || strings.HasPrefix(md.Source, dir) {
|
||||
if strings.HasPrefix(md.Source, dir) {
|
||||
names = append(names, md.Name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog {
|
||||
what := s.Model + "/" + s.Version
|
||||
used := m.usedBy(m.mirror.res.Base, s)
|
||||
items := []menuItem{{"Where nodes fetch it", func(m *consoleModel) tea.Cmd {
|
||||
func (m consoleModel) mirrorMenu(g mirrorGroup) *menuDialog {
|
||||
what := g.model + "/" + g.version
|
||||
used := m.usedBy(m.mirror.res.Base, g)
|
||||
items := []menuItem{{"Its files, and where nodes fetch them", func(m *consoleModel) tea.Cmd {
|
||||
var b strings.Builder
|
||||
for _, f := range g.files {
|
||||
sum := ""
|
||||
if f.SHA256 != "" {
|
||||
sum = sDim.Render(" sha256 " + f.SHA256[:min(12, len(f.SHA256))])
|
||||
}
|
||||
fmt.Fprintf(&b, "%-50s %9s%s\n", truncate(f.File, 50), bytesShort(max(f.BytesTotal, f.BytesDone)), sum)
|
||||
if f.Source != "" {
|
||||
b.WriteString(sDim.Render(" from "+truncate(f.Source, 90)) + "\n")
|
||||
}
|
||||
}
|
||||
m.dialog, m.dialogTitle = "error", what
|
||||
m.dialogText = mirror.Entry{Model: s.Model, Version: s.Version, File: s.File}.URL(m.mirror.res.Base) +
|
||||
"\n\n" + sDim.Render("the source to give a model definition · any key")
|
||||
m.dialogText = b.String() + "\nA model's source: " + g.source(m.mirror.res.Base) + "\n\n" + sDim.Render("any key")
|
||||
return nil
|
||||
}}}
|
||||
items = append(items, menuItem{"Remove from the mirror…", func(m *consoleModel) tea.Cmd {
|
||||
@@ -93,7 +142,7 @@ func (m consoleModel) mirrorMenu(s mirror.Status) *menuDialog {
|
||||
m.confirmD = &confirmDialog{title: "Remove " + what, text: text,
|
||||
yes: func(m *consoleModel) tea.Cmd {
|
||||
return m.mirrorCall("Remove "+what, func(ctx context.Context, c *mirror.Client) (string, error) {
|
||||
removed, err := c.Remove(ctx, s.Model, s.Version)
|
||||
removed, err := c.Remove(ctx, g.model, g.version)
|
||||
return fmt.Sprintf("Removed %d file(s).", len(removed)), err
|
||||
})
|
||||
}}
|
||||
@@ -139,24 +188,32 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
}
|
||||
head := "\n " + sDim.Render("Nodes fetch from "+m.mirror.res.Base) + "\n\n"
|
||||
var rows [][]string
|
||||
for _, s := range m.mirror.res.Models {
|
||||
state := string(s.State)
|
||||
switch s.State {
|
||||
case mirror.Fetching:
|
||||
if p, ok := s.Percent(); ok {
|
||||
state = sWarn.Render(fmt.Sprintf("fetching %d%%", p))
|
||||
} else {
|
||||
state = sWarn.Render("fetching " + bytesShort(s.BytesDone))
|
||||
for _, g := range m.mirrorGroups() {
|
||||
done, total := g.bytes()
|
||||
state := "present"
|
||||
for _, f := range g.files {
|
||||
switch f.State {
|
||||
case mirror.Fetching:
|
||||
state = sWarn.Render("fetching")
|
||||
if total > 0 {
|
||||
state = sWarn.Render(fmt.Sprintf("fetching %d%%", done*100/total))
|
||||
}
|
||||
case mirror.Failed:
|
||||
state = sBad.Render("failed")
|
||||
}
|
||||
if f.State != mirror.Present {
|
||||
break
|
||||
}
|
||||
case mirror.Failed:
|
||||
state = sBad.Render("failed")
|
||||
}
|
||||
size := bytesShort(s.BytesDone)
|
||||
used := strings.Join(m.usedBy(m.mirror.res.Base, s), ", ")
|
||||
what := g.files[0].File
|
||||
if len(g.files) > 1 {
|
||||
what = fmt.Sprintf("%d files", len(g.files))
|
||||
}
|
||||
used := strings.Join(m.usedBy(m.mirror.res.Base, g), ", ")
|
||||
if used == "" {
|
||||
used = sDim.Render("unused")
|
||||
}
|
||||
rows = append(rows, []string{s.Model, s.Version, s.File, state, size, used})
|
||||
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
|
||||
|
||||
Reference in new issue
Block a user