Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
636 lines
34 KiB
HTML
636 lines
34 KiB
HTML
{{define "admin_mirror"}}<!doctype html>
|
|
<html lang="en"><head>
|
|
<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
|
|
{{/* Only while something is downloading. A reload at any other time would
|
|
take with it whatever the operator was typing into the form below. */}}
|
|
{{if .RefreshSec}}<meta http-equiv="refresh" content="{{.RefreshSec}}">{{end}}
|
|
<script defer src="/_static/pages.js"></script>
|
|
<title>Mirror · llmgw</title><link rel="stylesheet" href="{{stylesheet}}">
|
|
</head><body><div class="wrap">
|
|
<header>
|
|
<h1>Model mirror</h1>
|
|
<p class="sub">{{.Addr}}{{if .Unreachable}} did not answer{{end}}</p>
|
|
{{template "nav" .Nav}}
|
|
</header>
|
|
|
|
{{/* Links rather than a script's panels: each tab has an address, the back
|
|
button works, and only the one shown asks anything of the registry. */}}
|
|
<nav class="page-tabs" aria-label="Mirror">
|
|
{{range .Tabs}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="page"{{end}}>{{.Label}}{{with .Count}} <span class="count">{{.}}</span>{{end}}</a>
|
|
{{end}}</nav>
|
|
|
|
{{if .Error}}<p class="error">{{.Error}}</p>{{end}}
|
|
{{if .Message}}<p class="ok">{{.Message}}</p>{{end}}
|
|
|
|
{{if eq .Tab "held"}}
|
|
{{if .Unreachable}}
|
|
{{/* Said outright rather than shown as an empty table. A mirror holding
|
|
nothing and a mirror that is not there look the same in a list of zero
|
|
rows, and they call for opposite responses. */}}
|
|
<p class="note">Nothing was read from the mirror, so this page is not a list of
|
|
what it holds — it is not currently reachable at <code>{{.Addr}}</code>. Check
|
|
that <code>llmgw-mirror</code> is running there and that its management port is
|
|
reachable from this machine.</p>
|
|
{{else}}
|
|
|
|
<div id="held">
|
|
{{if .Rows}}
|
|
<table class="rows compact held">
|
|
<tr><th>Model</th><th>Files</th><th class="num">Size</th><th>Runs</th><th></th></tr>
|
|
{{range .Groups}}{{$grouped := gt .Models 1}}
|
|
{{if $grouped}}
|
|
<tr class="model-group"><td colspan="5"><strong>{{.Family}}</strong>
|
|
<span class="opt">{{.Models}} models · {{bytes .Bytes}}</span>
|
|
{{if .Fetching}}<span class="tag new">downloading</span>{{end}}
|
|
{{if .Failed}}<span class="tag bad">failed</span>{{end}}
|
|
{{if .InUse}}<span class="tag">in use</span>{{end}}</td></tr>
|
|
{{end}}
|
|
{{range .Rows}}{{$row0 := .}}<tr{{if $grouped}} class="model-version"{{end}}{{if .Detail}} class="open"{{end}}>
|
|
<td>{{if eq (printf "%s" .State) "present"}}<a class="open-held" aria-expanded="{{if .Detail}}true{{else}}false{{end}}"
|
|
href="/admin/mirror?tab=held{{if not .Detail}}&open={{.Open}}{{end}}"><strong>{{.Model}}</strong></a>
|
|
{{else}}<strong>{{.Model}}</strong>{{end}} <span class="opt">{{.Version}}</span>
|
|
{{if eq .Kind "adapter"}}<span class="tag new" title="a LoRA adapter for {{.Arch}} models: attach it on a model's card">LoRA adapter</span>{{end}}</td>
|
|
<td class="file" title="{{.File}}{{if .SHA256}} · sha256 {{.SHA256}}{{end}}">
|
|
{{if gt .Files 1}}{{.Files}} files{{else}}<code>{{.File}}</code>{{end}}</td>
|
|
<td class="num">
|
|
{{if eq (printf "%s" .State) "fetching"}}
|
|
{{if .HasPercent}}{{.Percent}}% of {{bytes .BytesTotal}}
|
|
{{else}}{{/* No total came from upstream, so no percentage is invented —
|
|
a bar that guesses its own end and then stops moving is worse
|
|
than a number that only goes up. */}}
|
|
{{bytes .BytesDone}} so far{{end}}
|
|
{{else if eq (printf "%s" .State) "failed"}}—
|
|
{{else}}{{bytes .BytesDone}}{{end}}
|
|
</td>
|
|
<td>
|
|
{{if eq (printf "%s" .State) "fetching"}}<span class="state state-starting">downloading</span>
|
|
{{else if eq (printf "%s" .State) "failed"}}<span class="state state-failed">failed</span>
|
|
{{else if .UsedBy}}{{$row := .}}{{range .UsedBy}}<div class="use">
|
|
{{/* Named only when the catalog calls it something other than the
|
|
mirror does; otherwise it is the model column again. */}}
|
|
{{if ne .Name $row.Model}}<span class="as" title="{{.Name}}">as {{.Name}}</span>{{end}}
|
|
{{if .Declared}}<span class="mode-fixed" title="set in server.yaml, which is read again on every start">
|
|
{{- if eq .Mode "always"}}always on{{else if eq .Mode "ondemand"}}on demand{{else}}stopped{{end}} · yaml</span>
|
|
{{else}}
|
|
{{/* Submitted on change: the choice is the action, and a second click
|
|
to confirm it would only be a way to forget to. */}}
|
|
<form method="post" action="/admin/mirror/mode" class="inline">
|
|
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
|
<input type="hidden" name="model" value="{{.Name}}">
|
|
<input type="hidden" name="replicas" value="{{.Replicas}}">
|
|
<select name="mode" class="mode" data-autosubmit aria-label="How {{.Name}} runs">
|
|
<option value="always"{{if eq .Mode "always"}} selected{{end}}>always on</option>
|
|
<option value="ondemand"{{if eq .Mode "ondemand"}} selected{{end}}>on demand</option>
|
|
<option value="stopped"{{if eq .Mode "stopped"}} selected{{end}}>stopped</option>
|
|
</select>
|
|
<noscript><button type="submit" class="link">set</button></noscript>
|
|
</form>
|
|
{{end}}
|
|
{{if .Fleet}}<span class="runs-now{{if or .Parked .Unplaceable}} bad{{else if lt .Ready .Wanted}} short{{end}}" title="replicas ready / wanted{{if .Starting}}, + starting{{end}}">
|
|
{{- if .Parked}}parked
|
|
{{- else if and (eq .Wanted 0) (eq .Ready 0)}}none running
|
|
{{- else}}{{.Ready}}/{{.Wanted}}{{if .Starting}} +{{.Starting}}{{end}}{{end}}
|
|
{{- if .Unplaceable}} · no room{{end}}
|
|
{{- with .Nodes}} · {{range $i, $n := .}}{{if $i}}, {{end}}<span class="node" title="{{$n.Full}}">{{$n.Short}}</span>{{end}}{{end}}</span>{{end}}
|
|
</div>{{end}}
|
|
{{else}}<span class="opt">not served</span>{{end}}
|
|
</td>
|
|
<td class="actions">
|
|
{{if eq (printf "%s" .State) "failed"}}
|
|
{{/* Retry, not remove. A fetch that failed left nothing on disk, so the
|
|
remove would come back "the mirror holds nothing for this" — a button
|
|
whose only outcome is a confusing error. Starting the same download
|
|
again is what somebody looking at a failed row wants, and the mirror
|
|
allows it: only a download still running is refused. */}}
|
|
<form method="post" action="/admin/mirror/fetch" class="inline">
|
|
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
|
<input type="hidden" name="source" value="{{.Source}}">
|
|
<input type="hidden" name="model" value="{{.Model}}">
|
|
<input type="hidden" name="version" value="{{.Version}}">
|
|
<input type="hidden" name="file" value="{{.File}}">
|
|
<button type="submit" class="link">retry</button>
|
|
</form>
|
|
{{else if eq (printf "%s" .State) "present"}}
|
|
<button type="button" class="link copy-url" data-url="{{.URL}}" title="{{.URL}}">copy address</button>
|
|
{{if and $.HasCatalog (not .UsedBy)}}{{if .Serve}}<a href="/admin/mirror">close</a>
|
|
{{else}}<a href="/admin/mirror?serve={{.Model}}:{{.Version}}">serve</a>{{end}}{{end}}
|
|
<form method="post" action="/admin/mirror/{{.Model}}/{{.Version}}/remove"
|
|
class="inline" data-confirm="Remove {{.Model}}/{{.Version}} and every file in it?{{if .UsedBy}} It is in use by the catalog.{{end}}">
|
|
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
|
<button type="submit" class="link danger">remove</button>
|
|
</form>
|
|
{{end}}
|
|
</td>
|
|
</tr>
|
|
{{if .Err}}<tr><td colspan="5" class="error-row">{{.Err}}</td></tr>{{end}}
|
|
{{with .Detail}}<tr class="held-detail"><td colspan="5">{{template "mirror_held_detail" dict "D" . "Row" $row0 "CSRF" $.CSRF}}</td></tr>{{end}}
|
|
|
|
{{with .Serve}}
|
|
<tr class="editing"><td colspan="5">
|
|
{{if .Refusal}}
|
|
{{/* Offered and then failing at placement is worse than not offered at all. */}}
|
|
<p class="error">Cannot be served as it stands: {{.Refusal}}.</p>
|
|
{{else}}
|
|
{{if .Warning}}<p class="note error-row">{{.Warning}}.</p>{{end}}
|
|
<form method="post" action="/admin/mirror/serve" class="serve-form" data-engine="{{.Engine}}" data-free-gpus="{{.FreeGPUs}}">
|
|
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
|
<input type="hidden" name="source" value="{{.Source}}">
|
|
<input type="hidden" name="engine" value="{{.Engine}}">
|
|
<p class="sub">Serving <code>{{.Source}}</code> with <strong>{{.Engine}}</strong>.
|
|
{{if .Files}}The whole version, {{.Files}} files: the node fetches them into one
|
|
directory and the engine is given the directory.{{end}}</p>
|
|
<div class="field-row">
|
|
<label>Name <input type="text" name="name" value="{{.Model}}" required></label>
|
|
<label>Aliases <span class="opt">comma separated</span>
|
|
<input type="text" name="aliases" placeholder="coder-7b"></label>
|
|
<label>Replicas <input type="text" name="replicas" value="1" required></label>
|
|
<label>Runs
|
|
<select name="mode">
|
|
<option value="always">always on</option>
|
|
<option value="ondemand">on demand: loads when asked, stops after 15 min unused</option>
|
|
</select></label>
|
|
</div>
|
|
{{/* Reaches clients through /v1/models, which for most of them is the only
|
|
view of what this gateway serves. Nine names and nothing to choose
|
|
between them is a choice made by guesswork. */}}
|
|
<label>What it is for <span class="opt">shown to clients in /v1/models</span>
|
|
<input type="text" name="description"
|
|
placeholder="general chat, strong Swedish; 14B, slower than the 8B"></label>
|
|
<div class="field-row">
|
|
<label>Context a request may use
|
|
<input type="text" name="context_length" value="4096" required></label>
|
|
<label>Output limit <span class="opt">optional</span>
|
|
<input type="text" name="max_output" value="1024"></label>
|
|
<label>Engine context pool <span class="opt">divided across slots</span>
|
|
<input type="text" name="ctx_size" value="{{.CtxPool}}"></label>
|
|
</div>
|
|
{{/* Where it runs is a choice with consequences, so it is a choice: the
|
|
GPU classes the fleet has, and the CPU for llama.cpp. A profile name
|
|
field stood here once, and reading as though naming a profile "gpu"
|
|
would put it on one, it put GGUF models on the CPU of GPU clusters. */}}
|
|
<div class="field-row">
|
|
<label>Runs on
|
|
<select name="placement">
|
|
{{range .Placements}}<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>{{end}}
|
|
</select></label>
|
|
<label>GPUs per replica <span class="opt">on one node</span>
|
|
<input type="text" name="gpus" value="{{.GPUs}}" size="3"></label>
|
|
{{if eq .Engine "llamacpp"}}<label>Parallel requests <span class="opt">per replica</span>
|
|
<input type="text" name="parallel" value="{{.Parallel}}" size="3"></label>{{end}}
|
|
</div>
|
|
{{if .FreeGPUs}}<label class="cpu-ok"><input type="checkbox" name="cpu_ok" value="yes">
|
|
Run on the CPU anyway: {{.FreeGPUs}} GPUs are free and would sit unused</label>{{end}}
|
|
<div class="field-row">
|
|
<label>Needs RAM (GB) <span class="opt">for the context pool beside it</span>
|
|
<input type="text" name="min_memory_gb" value="{{.MinMemoryGB}}" required></label>
|
|
<label>CPU cores <span class="opt">reserved; the engine's threads</span>
|
|
<input type="text" name="cpu_cores" value="{{.CPUCores}}"></label>
|
|
<label>Only nodes labelled <span class="opt">key / value, optional</span>
|
|
<input type="text" name="selector_key" placeholder="role" size="8">
|
|
<input type="text" name="selector_value" placeholder="cpu-large" size="10"></label>
|
|
</div>
|
|
<p class="sub serve-summary" hidden></p>
|
|
<button type="submit">Add to the catalog</button>
|
|
</form>
|
|
<script>
|
|
// What it will run as, in one line, kept up to date as the form changes:
|
|
// engine, where, how many cards, memory, context and how it is shared.
|
|
(function () {
|
|
const f = document.currentScript.previousElementSibling;
|
|
if (!f || !f.classList.contains("serve-form")) return;
|
|
const out = f.querySelector(".serve-summary"), ok = f.querySelector(".cpu-ok");
|
|
const v = (n) => (f.elements[n] ? f.elements[n].value.trim() : "");
|
|
const engine = { llamacpp: "llama.cpp", vllm: "vLLM", sglang: "SGLang" }[f.dataset.engine] || f.dataset.engine;
|
|
function draw() {
|
|
const place = v("placement"), gpus = parseInt(v("gpus"), 10) || 1;
|
|
const onCPU = place === "cpu" || place === "";
|
|
let where = "on the CPU";
|
|
if (!onCPU) {
|
|
const cls = place.replace(/^gpu:/, "");
|
|
where = gpus > 1 ? "on " + gpus + " " + cls + " GPUs of one node" : "on 1 GPU " + cls;
|
|
}
|
|
const parts = [engine + " " + where];
|
|
if (v("min_memory_gb")) parts.push(v("min_memory_gb") + " GB of host memory");
|
|
if (onCPU && v("cpu_cores")) parts.push(v("cpu_cores") + " cores");
|
|
const pool = parseInt(v("ctx_size"), 10), par = parseInt(v("parallel"), 10);
|
|
if (pool && par) parts.push("context pool " + pool + " over " + par + " parallel requests (" + Math.floor(pool / par) + " each)");
|
|
else if (pool) parts.push("context pool " + pool);
|
|
out.textContent = "Will run as: " + parts.join(", ") + ".";
|
|
const warn = onCPU && +f.dataset.freeGpus > 0;
|
|
out.classList.toggle("error", warn);
|
|
if (warn) out.textContent += " " + f.dataset.freeGpus + " GPUs are free; on the CPU this runs many times slower.";
|
|
if (ok) ok.hidden = !onCPU;
|
|
out.hidden = false;
|
|
}
|
|
f.addEventListener("input", draw);
|
|
f.addEventListener("change", draw);
|
|
draw();
|
|
})();
|
|
</script>
|
|
<p class="note">The context pool is what the engine is given; llama.cpp divides
|
|
it across its parallel slots, so the number one request may use is the smaller
|
|
one above.</p>
|
|
<p class="note">Memory: {{.MemoryBasis}}. Change the context pool and this changes
|
|
with it, by the KV cache's share — an estimate, and editable. It is filled in rather than
|
|
left blank because a CPU profile has no card to be short of: with no figure,
|
|
every ready node is a candidate and the first one picked is whichever the
|
|
scheduler reached, not whichever can hold the model. A node label does the same
|
|
job by name if you would rather pin it.</p>
|
|
{{end}}
|
|
</td></tr>
|
|
{{end}}
|
|
{{end}}
|
|
{{end}}
|
|
</table>
|
|
<p class="note">“copy address” gives what a catalog entry's <code>source</code>
|
|
should say — the file, or for a version of several files the directory, with its
|
|
trailing slash. It is rendered rather than assembled by hand because the mirror
|
|
and the catalog have to spell a file the same way, which is also how this page
|
|
knows which models use it.</p>
|
|
<script>
|
|
document.addEventListener('click', function (e) {
|
|
var b = e.target.closest('.copy-url');
|
|
if (!b) return;
|
|
var done = function (t) { var o = b.textContent; b.textContent = t; setTimeout(function () { b.textContent = o; }, 1500); };
|
|
if (navigator.clipboard) {
|
|
navigator.clipboard.writeText(b.dataset.url).then(function () { done('copied'); },
|
|
function () { window.prompt('Address', b.dataset.url); });
|
|
} else { window.prompt('Address', b.dataset.url); }
|
|
});
|
|
</script>
|
|
{{else}}<p class="empty">The mirror is reachable and holds nothing yet.
|
|
{{if .HasRegistry}}<a href="/admin/mirror?tab=explore">Explore Hugging Face</a> for a model, or
|
|
{{end}}<a href="/admin/mirror?tab=fetch">fetch one by URL</a>.</p>{{end}}
|
|
</div>
|
|
{{end}}
|
|
{{end}}
|
|
|
|
{{if and (eq .Tab "explore") .HasRegistry}}
|
|
{{if not .Repo}}{{with .Explore}}
|
|
<nav class="pills" aria-label="Which models">{{range .Views}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="page"{{end}}>{{.Label}}</a>{{end}}</nav>
|
|
{{if eq .View "adapters"}}
|
|
<p class="note">{{if .Of}}LoRA adapters made for <code>{{.Of}}</code> ·
|
|
<a href="/admin/mirror?tab=explore&view=adapters">all adapters</a>{{else}}LoRA adapters for any model.
|
|
From a held model's card, <em>find adapters</em> lists those made for it.{{end}}
|
|
An adapter runs on the model it was made for, and only as a GGUF adapter file;
|
|
open one to see which of its files are.</p>
|
|
{{else}}
|
|
<div class="explore-filters">
|
|
<nav class="pills small" aria-label="Task">{{range .Tasks}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="true"{{end}}>{{.Label}}</a>{{end}}</nav>
|
|
<nav class="pills small" aria-label="Size">{{range .Sizes}}<a href="{{.Href}}"{{if .Active}} class="on" aria-current="true"{{end}}>{{.Label}}</a>{{end}}</nav>
|
|
</div>
|
|
{{end}}
|
|
{{end}}
|
|
<form method="get" action="/admin/mirror" class="explore-search">
|
|
<input type="hidden" name="tab" value="explore">
|
|
{{with .Explore}}<input type="hidden" name="view" value="{{.View}}">
|
|
<input type="hidden" name="task" value="{{.Task}}">
|
|
<input type="hidden" name="size" value="{{.Size}}">
|
|
{{if .Of}}<input type="hidden" name="of" value="{{.Of}}">{{end}}
|
|
<input type="hidden" name="x" value="1">{{end}}
|
|
<input type="search" name="q" value="{{.Query}}" placeholder="Search Hugging Face — qwen coder, gemma, bge…" aria-label="Search Hugging Face">
|
|
<label class="check"><input type="checkbox" name="here" value="1" data-autosubmit{{if .Explore}}{{if .Explore.Here}} checked{{end}}{{end}}> only what these nodes can run</label>
|
|
<button type="submit">Search</button>
|
|
</form>
|
|
{{end}}
|
|
|
|
{{if .RegistryErr}}
|
|
{{/* Its own line, not the page's error: the registry is somebody else's server
|
|
and it being down says nothing about the mirror, which is what this page is
|
|
for and has to keep working when Hugging Face does not. */}}
|
|
<p class="note error-row">The registry: {{.RegistryErr}}</p>
|
|
{{end}}
|
|
|
|
{{if not .Repo}}{{with .Explore}}
|
|
{{if .Rows}}
|
|
<table class="rows explore">
|
|
<tr><th>Repository</th><th class="num">Size</th><th>Architecture</th><th class="num">Context</th>
|
|
<th class="num">Downloads</th><th class="num">Likes</th><th class="num">Updated</th></tr>
|
|
{{range .Rows}}<tr class="explore-row">
|
|
<td><a href="{{.Href}}" class="open-repo" data-repo="{{.ID}}" aria-expanded="false"><code>{{.ID}}</code></a>
|
|
{{if .New}}<span class="tag new">new</span>{{end}}{{if .Gated.Is}}<span class="tag">gated</span>{{end}}
|
|
{{if or .BaseModel .License}}<span class="from" title="{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}">{{with .BaseModel}}from {{.}}{{end}}{{if and .BaseModel .License}} · {{end}}{{.License}}</span>{{end}}</td>
|
|
<td class="num">{{.ParamsLabel}}</td>
|
|
<td>{{if .Architecture}}<span{{if not .Verdict.Servable}} class="cannot"{{end}} title="{{.Verdict.Reason}}">{{.Architecture}}{{if .Verdict.Checked}}{{if .Verdict.Servable}} ✓{{else}} ✗{{end}}{{end}}</span>{{else}}—{{end}}</td>
|
|
<td class="num">{{.ContextLabel}}</td>
|
|
<td class="num">{{compact .Downloads}}</td>
|
|
<td class="num">{{compact .Likes}}</td>
|
|
<td class="num">{{.Age}}</td>
|
|
</tr>{{end}}
|
|
</table>
|
|
<script>
|
|
// A repository opens beneath its row rather than replacing the list: the
|
|
// list is what somebody is choosing from, and losing it to read one card
|
|
// means finding their place again for the next. The link still goes to the
|
|
// repository's own page when this does not run, or with a modifier key.
|
|
document.querySelector("table.explore").addEventListener("click", async function (e) {
|
|
const a = e.target.closest("a.open-repo");
|
|
if (!a || e.metaKey || e.ctrlKey || e.shiftKey || e.button !== 0) return;
|
|
e.preventDefault();
|
|
const row = a.closest("tr");
|
|
let detail = row.nextElementSibling;
|
|
if (detail && detail.classList.contains("repo-detail")) {
|
|
detail.hidden = !detail.hidden;
|
|
a.setAttribute("aria-expanded", String(!detail.hidden));
|
|
row.classList.toggle("open", !detail.hidden);
|
|
return;
|
|
}
|
|
detail = document.createElement("tr");
|
|
detail.className = "repo-detail";
|
|
const td = document.createElement("td");
|
|
td.colSpan = row.children.length;
|
|
td.textContent = "Loading…";
|
|
detail.appendChild(td);
|
|
row.after(detail);
|
|
row.classList.add("open");
|
|
a.setAttribute("aria-expanded", "true");
|
|
try {
|
|
const resp = await fetch("/admin/mirror/repo?repo=" + encodeURIComponent(a.dataset.repo));
|
|
const text = await resp.text();
|
|
if (!resp.ok) { td.textContent = text.slice(0, 300); td.classList.add("error-row"); return; }
|
|
// Our own server's rendering of the repository, the card already
|
|
// stripped of anything that runs: the same HTML the full page shows.
|
|
td.innerHTML = text;
|
|
} catch (err) {
|
|
td.textContent = "Could not load it: " + err;
|
|
}
|
|
});
|
|
|
|
// A fetch started from an opened repository answers where it was asked:
|
|
// the outcome under its button, and the mirror's table above brought up to
|
|
// date and followed until the download is done. A form post would reload
|
|
// the page, closing the repository and losing the list's place.
|
|
document.querySelector("table.explore").addEventListener("submit", async function (e) {
|
|
const form = e.target.closest('form[action="/admin/mirror/fetch"]');
|
|
if (!form) return;
|
|
e.preventDefault();
|
|
const btn = form.querySelector("button[type=submit]");
|
|
const was = btn.textContent;
|
|
btn.disabled = true;
|
|
btn.textContent = "Starting…";
|
|
let note = form.parentElement.querySelector(".fetch-note");
|
|
if (!note) {
|
|
note = document.createElement("div");
|
|
note.className = "fetch-note note";
|
|
form.after(note);
|
|
}
|
|
try {
|
|
const resp = await fetch(form.action, {
|
|
method: "POST", body: new URLSearchParams(new FormData(form)),
|
|
headers: { Accept: "application/json" },
|
|
});
|
|
const out = await resp.json().catch(() => ({ error: "(" + resp.status + ")" }));
|
|
if (!resp.ok) {
|
|
note.textContent = out.error || "refused";
|
|
note.classList.add("bad");
|
|
btn.disabled = false;
|
|
btn.textContent = was;
|
|
return;
|
|
}
|
|
note.classList.remove("bad");
|
|
note.textContent = "✓ " + out.message + " — ";
|
|
const link = document.createElement("a");
|
|
link.href = "/admin/mirror?tab=held";
|
|
link.textContent = "follow it under Held";
|
|
note.appendChild(link);
|
|
btn.textContent = "Started";
|
|
} catch (err) {
|
|
note.textContent = "Could not reach the gateway: " + err;
|
|
note.classList.add("bad");
|
|
btn.disabled = false;
|
|
btn.textContent = was;
|
|
}
|
|
});
|
|
</script>
|
|
{{if .Hidden}}<p class="note">{{.Hidden}} more {{if eq .Hidden 1}}is{{else}}are{{end}} hidden: their architecture is one the nodes' llama.cpp cannot load.
|
|
Untick “only what these nodes can run” to see them.</p>{{end}}
|
|
<p class="note">A gated repository needs its terms accepted with a Hugging Face
|
|
account, and this browses anonymously — the files will list, and fetching one
|
|
will fail at the mirror. Download it by hand and put it in the mirror
|
|
directory, or give the mirror a token.</p>
|
|
{{else if not $.RegistryErr}}
|
|
<p class="empty">{{if $.Query}}Nothing matched “{{$.Query}}”{{else}}Nothing to list{{end}}{{if .Hidden}} that these nodes can run — {{.Hidden}} {{if eq .Hidden 1}}result was{{else}}results were{{end}} hidden for an architecture llama.cpp cannot load{{end}}.</p>
|
|
{{end}}
|
|
{{end}}{{end}}
|
|
|
|
{{if .Repo}}{{template "mirror_repo" .}}{{end}}
|
|
{{end}}
|
|
|
|
{{if eq .Tab "fetch"}}
|
|
<p class="note">By hand, for anything the search does not reach.</p>
|
|
<form method="post" action="/admin/mirror/fetch">
|
|
<input type="hidden" name="csrf" value="{{.CSRF}}">
|
|
<input type="hidden" name="tab" value="fetch">
|
|
<label>Source URL
|
|
<input type="text" name="source" required
|
|
placeholder="https://huggingface.co/…/resolve/main/model.gguf"></label>
|
|
<div class="field-row">
|
|
<label>Model <input type="text" name="model" required placeholder="qwen2.5-coder-7b"></label>
|
|
<label>Version <input type="text" name="version" required placeholder="2026-09"></label>
|
|
<label>File <span class="opt">optional</span>
|
|
<input type="text" name="file" placeholder="taken from the URL"></label>
|
|
</div>
|
|
<label class="inline-check"><input type="checkbox" name="force" value="1">
|
|
Fetch anyway <span class="opt">even if the nodes' llama.cpp cannot load it</span></label>
|
|
<button type="submit">Start download</button>
|
|
</form>
|
|
<p class="note">The mirror answers as soon as the download is under way, not when
|
|
it finishes — a model is tens of gigabytes and nothing here holds a connection
|
|
open for that. The Held tab shows it arriving. A version is never
|
|
overwritten: fetching into one that exists is refused, so a new build is a new
|
|
version and the old weights stay where they are.</p>
|
|
{{end}}
|
|
|
|
<footer><span>llmgw {{.Version}}</span><span>{{.Now}}</span></footer>
|
|
</div></body></html>{{end}}
|
|
|
|
{{/* One repository: its card and its files. The whole page when opened by
|
|
address, and the row beneath a listing entry when opened from the list. */}}
|
|
{{define "mirror_repo"}}
|
|
{{if not .Inline}}<p><a href="{{.Back}}">← back to the list</a></p>
|
|
<h3>{{.Repo.ID}} <span class="opt">at {{.Repo.ShortRevision}}</span></h3>{{end}}
|
|
<p class="repo-meta">
|
|
{{if .Inline}}<span>at <code>{{.Repo.ShortRevision}}</code></span>{{end}}
|
|
{{with .Repo.BaseModel}}<span>from {{range $i, $b := .}}{{if $i}}, {{end}}<code>{{$b}}</code>{{end}}</span>{{end}}
|
|
{{with .Repo.License}}<span>license {{.}}</span>{{end}}
|
|
{{with .Repo.Languages}}<span>{{join .}}</span>{{end}}
|
|
<span><a href="{{.RepoURL}}" rel="noopener noreferrer">on Hugging Face ↗</a></span>
|
|
</p>
|
|
{{if .Card}}
|
|
<details class="model-card" open>
|
|
<summary>Model card</summary>
|
|
<div class="card-body">{{.Card}}</div>
|
|
</details>
|
|
{{end}}
|
|
{{if .RepoSets}}
|
|
<table class="rows repo-files">
|
|
<tr><th>What</th><th class="num">Size</th><th>Published checksum</th><th></th></tr>
|
|
{{range .RepoSets}}<tr>
|
|
<td><code>{{.Name}}</code>
|
|
{{if .Shards}}<span class="opt">{{.Shards}} shards, fetched together</span>{{end}}
|
|
{{with .Companions}}<span class="opt" title="{{join .}}">with {{len .}} files the engine reads beside them</span>{{end}}
|
|
{{if eq .Kind "adapter"}}<span class="tag new" title="a LoRA adapter: it runs on a {{.Arch}} model it was made for, not on its own">LoRA adapter · {{.Arch}}</span>
|
|
{{else if eq .Kind "model"}}<span class="opt">model{{with .Arch}} · {{.}}{{end}}</span>{{end}}</td>
|
|
<td class="num">{{bytes .Bytes}}</td>
|
|
<td>{{with .SHA256}}<code title="{{.}}">{{slice . 0 12}}</code>{{else}}—{{end}}</td>
|
|
<td>
|
|
{{if not .Verdict.Servable}}
|
|
<span class="state state-failed" title="{{.Verdict.Reason}}">cannot be served</span>
|
|
<span class="clip-inline" title="{{.Verdict.Reason}}">{{.Verdict.Reason}}</span>
|
|
{{else}}
|
|
{{if .Verdict.Architecture}}<span class="opt" title="{{.Verdict.Reason}}">{{.Verdict.Architecture}}{{if .Verdict.Checked}} ✓{{end}}</span>{{end}}
|
|
<form method="post" action="/admin/mirror/fetch" class="inline">
|
|
<input type="hidden" name="csrf" value="{{$.CSRF}}">
|
|
<input type="hidden" name="tab" value="explore">
|
|
{{range .Sources}}<input type="hidden" name="source" value="{{.URL}}">
|
|
<input type="hidden" name="file" value="{{.Path}}">
|
|
{{end}}
|
|
<input type="text" name="model" value="{{.Model}}" size="24">
|
|
<input type="text" name="version" value="{{.Version}}" size="10">
|
|
<button type="submit">Fetch</button>
|
|
</form>
|
|
{{end}}
|
|
</td>
|
|
</tr>{{end}}
|
|
</table>
|
|
<p class="note">Only the weights are listed; a repository is mostly a README and
|
|
a tokenizer. A safetensors set is fetched with the files vLLM reads beside it —
|
|
config.json, the tokenizer, the shard index — and never with code. Shards of one file are one row and one button — a fifteen-shard
|
|
model is one decision, not fifteen. Two separate files are two rows because they
|
|
are usually alternatives: one quantisation or the other, not both.</p>
|
|
<p class="note">The source URLs are pinned to
|
|
<code>{{.Repo.ShortRevision}}</code> rather than to a branch, so what arrives is
|
|
what is listed here even if the branch moves in between — and a single file's
|
|
checksum above is what the registry publishes, to compare with what the mirror
|
|
computes once it has it.</p>
|
|
{{else}}
|
|
<p class="empty">No weight files in {{.Repo.ID}}.</p>
|
|
{{end}}
|
|
{{end}}
|
|
|
|
{{/* The card beneath a held version: what runs it, where, and how many. */}}
|
|
{{define "mirror_held_detail"}}{{$row := .Row}}{{$csrf := .CSRF}}{{$scopes := .D.Scopes}}
|
|
<div class="held-card">
|
|
<p class="repo-meta">
|
|
<span>{{if gt $row.Files 1}}{{$row.Files}} files{{else}}<code>{{$row.File}}</code>{{end}}</span>
|
|
<span>{{bytes $row.BytesDone}}</span>
|
|
{{with $row.SHA256}}<span>sha256 <code title="{{.}}">{{slice . 0 12}}</code></span>{{end}}
|
|
<span><code>{{$row.URL}}</code></span>
|
|
</p>
|
|
{{if not .D.Models}}
|
|
<p class="note">Not in the catalog: nothing runs this version.
|
|
{{if $row.Serve}}{{else}}Use <strong>serve</strong> on its row to put it into service.{{end}}</p>
|
|
{{end}}
|
|
{{range .D.Models}}
|
|
<section class="held-model">
|
|
<h3>{{.Name}}
|
|
{{with .View}}<span class="opt">{{.Ready}} of {{.Desired}} ready{{if ne .Total .Ready}} · {{.Total}} in all{{end}}</span>
|
|
{{if .Parked}}<span class="tag bad">parked</span>{{else if .Unplaceable}}<span class="tag bad">does not fit</span>{{end}}{{end}}
|
|
</h3>
|
|
{{with .View.Reason}}<p class="note error-row">{{.}}</p>{{end}}
|
|
|
|
{{if .Replicas}}
|
|
<table class="rows compact replicas">
|
|
<tr><th>Replica</th><th>Node</th><th>State</th><th class="num">Age</th><th class="num">CPU</th><th class="num">Memory</th></tr>
|
|
{{range .Replicas}}{{$rep := .}}{{range $i, $s := .Nodes}}<tr>
|
|
<td>{{if eq $i 0}}<code>{{$rep.ID}}</code>{{end}}</td>
|
|
<td><a href="/admin/nodes/{{$s.Node}}">{{$s.Node}}</a>{{if $s.MemoryGB}} <span class="opt">{{$s.MemoryGB}} GB</span>{{end}}</td>
|
|
<td>{{if eq $i 0}}<span class="state state-{{if eq $rep.State "ready"}}ready{{else if eq $rep.State "failed"}}failed{{else}}starting{{end}}">{{$rep.State}}</span>
|
|
{{with $rep.Restart}}<span class="opt">restart {{.}}</span>{{end}}{{end}}</td>
|
|
<td class="num">{{if eq $i 0}}{{age $rep.Age}}{{end}}</td>
|
|
<td class="num">{{if $s.CPU}}{{printf "%.1f" $s.CPU}} cores{{else}}—{{end}}</td>
|
|
<td class="num">{{if $s.MemBytes}}{{bytes $s.MemBytes}}{{else}}—{{end}}</td>
|
|
</tr>{{end}}{{end}}
|
|
</table>
|
|
{{else}}<p class="note">No replica runs now{{if eq .Mode "ondemand"}}: it starts when somebody asks for it{{end}}.</p>{{end}}
|
|
|
|
{{with .Headroom}}
|
|
<p class="headroom">
|
|
{{if .Limited}}Room for at least {{.More}} more replicas.
|
|
{{else if .More}}Room for {{.More}} more now{{if .Nodes}}, on {{range $i, $n := .Nodes}}{{if $i}}, {{end}}{{join $n}}{{end}}{{end}}.
|
|
{{else}}No free room for another replica now.{{end}}
|
|
{{if .ByPreempting}}<br><span class="note">{{if .More}}And {{end}}{{.ByPreempting}} {{if .More}}more {{end}}by stopping replicas its class outranks: {{join .Preempts}}.</span>{{end}}
|
|
</p>
|
|
{{if and (not .Limited) .Blocked}}<details class="blocked"><summary>Why not {{if .More}}more{{else}}another{{end}}</summary>
|
|
<ul>{{range .Blocked}}<li>{{.}}</li>{{end}}</ul></details>{{end}}
|
|
{{end}}
|
|
|
|
{{if not .Declared}}{{$mname := .Name}}
|
|
<div class="adapters">
|
|
<h4>LoRA adapters <a class="note" href="{{.FindAdapters}}">find adapters on Hugging Face →</a></h4>
|
|
{{if .Adapters}}<table class="compact rows">
|
|
<tr><th>Asked for as</th><th>File</th><th class="num">Scale</th><th></th></tr>
|
|
{{range .Adapters}}{{$a := .}}<tr>
|
|
<td><code>{{.Name}}</code></td>
|
|
<td class="clip" title="{{.Source}}">{{.Source}}</td>
|
|
<td class="num">{{.EffectiveScale}}</td>
|
|
<td class="actions"><form method="post" action="/admin/mirror/adapters" class="inline">
|
|
<input type="hidden" name="csrf" value="{{$csrf}}">
|
|
<input type="hidden" name="model" value="{{$mname}}">
|
|
<input type="hidden" name="open" value="{{$row.Open}}">
|
|
<input type="hidden" name="op" value="remove">
|
|
<input type="hidden" name="name" value="{{$a.Name}}">
|
|
<button type="submit" class="link danger">remove</button></form></td>
|
|
</tr>{{end}}</table>
|
|
{{else}}<p class="note">None. A request for an adapter's name is served by this model's replicas with it applied.</p>{{end}}
|
|
{{if .AdapterChoices}}{{$m := .}}
|
|
<form method="post" action="/admin/mirror/adapters" class="adapter-form">
|
|
<input type="hidden" name="csrf" value="{{$csrf}}">
|
|
<input type="hidden" name="model" value="{{.Name}}">
|
|
<input type="hidden" name="open" value="{{$row.Open}}">
|
|
<input type="hidden" name="op" value="add">
|
|
<div class="field-row">
|
|
<label>Adapter file <select name="source" required data-suggest-into="name">
|
|
{{range .AdapterChoices}}<option value="{{.URL}}" data-suggest="{{.Suggest}}">{{.Label}}</option>{{end}}
|
|
</select></label>
|
|
<label>Asked for as <input type="text" name="name" value="{{(index .AdapterChoices 0).Suggest}}" required></label>
|
|
<label>Scale <input type="text" name="scale" value="1" size="4"></label>
|
|
<button type="submit">Attach</button>
|
|
</div>
|
|
</form>{{else}}<p class="note">No held adapter file for this model's architecture. Fetch one from Hugging Face first.</p>{{end}}
|
|
</div>
|
|
{{end}}
|
|
|
|
{{if .Declared}}
|
|
<p class="note">Set in server.yaml, which is read again on every start: change its replicas there.</p>
|
|
{{else}}
|
|
<form method="post" action="/admin/mirror/replicas" class="replicas-form">
|
|
<input type="hidden" name="csrf" value="{{$csrf}}">
|
|
<input type="hidden" name="model" value="{{.Name}}">
|
|
<input type="hidden" name="open" value="{{$row.Open}}">
|
|
<div class="field-row">
|
|
<label>Always running
|
|
<input type="number" name="min" value="{{.Min}}" min="0"{{if .MostNow}} max="{{.MostNow}}"{{end}} required></label>
|
|
<label>Up to, under load
|
|
<input type="number" name="max" value="{{.Max}}" min="0" max="64" required></label>
|
|
<label>Class
|
|
<select name="class">
|
|
<option value=""{{if eq .Class ""}} selected{{end}}>default ({{if eq .Mode "ondemand"}}opportunistic{{else}}standard{{end}})</option>
|
|
<option value="critical"{{if eq .Class "critical"}} selected{{end}}>critical</option>
|
|
<option value="standard"{{if eq .Class "standard"}} selected{{end}}>standard</option>
|
|
<option value="opportunistic"{{if eq .Class "opportunistic"}} selected{{end}}>opportunistic</option>
|
|
</select></label>
|
|
{{if $scopes}}{{$owner := .Owner}}<label>Owner
|
|
<select name="owner">
|
|
<option value=""{{if eq $owner ""}} selected{{end}}>shared</option>
|
|
{{range $scopes}}<option value="{{.Key}}"{{if eq .Key $owner}} selected{{end}}>{{.Name}}</option>{{end}}
|
|
</select></label>{{end}}
|
|
</div>
|
|
<div class="field-row">
|
|
<label>Memory GB <span class="opt">per replica</span>
|
|
<input type="number" name="memory" value="{{.MemoryGB}}" min="0"></label>
|
|
<label>Cores <span class="opt">per replica, its threads</span>
|
|
<input type="number" name="cores" value="{{.CPUCores}}" min="0"></label>
|
|
<button type="submit">Save</button>
|
|
</div>
|
|
<p class="note">{{if .MostNow}}At most {{.MostNow}} can always run now{{if .Headroom}}{{if .Headroom.ByPreempting}}, counting what its class may stop to make room{{end}}{{end}}. {{end}}0 always running with a
|
|
higher limit loads it when asked for; 0 and 0 stops it. A replica is added under load when requests wait, and
|
|
taken away after ten quiet minutes. A higher class may stop replicas of lower ones to make room; memory and
|
|
cores apply to replicas started after the change.</p>
|
|
</form>
|
|
{{end}}
|
|
</section>
|
|
{{end}}
|
|
</div>
|
|
{{end}}
|