TensorRT-LLM as an engine, and models from Ollama into the mirror
Engine trtllm runs trtllm-serve on one node, through a launcher of llmgw's that demands the replica's key and turns on the Prometheus endpoint its queue and cache gauges are read from; the router reads it at /prometheus/metrics. Brought by the operator (engines.trtllm.python), not installed; checked by ctl engine-check trtllm. Written against the documented CLI, server and metrics and tested with stand-ins: not yet run on a datacenter GPU. mirror import-ollama copies a model's GGUF, and a vision model's projector, from Ollama's directory into the mirror, checked against the digest Ollama named each blob by; mirror import does the same for any local file. The console's Mirror view imports from Ollama with I. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
30f7f3474a
commit
7940e8593c
28 files changed
+2376
-18
No files matched your search
@@ -106,7 +106,7 @@ func newBenchCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
f := cmd.Flags()
|
||||
f.StringVar(&target, "target", "http://127.0.0.1:8080", "base URL: a replica, or the gateway")
|
||||
f.StringVar(&metrics, "metrics", "", "the replica's /metrics, to observe queue depth and KV usage")
|
||||
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp")
|
||||
f.StringVar(&engine, "engine", "", "which metric names --metrics uses: vllm | sglang | llamacpp | trtllm (whose are at /prometheus/metrics)")
|
||||
f.StringVar(&key, "key", "", "API key, when the gateway requires one")
|
||||
f.IntSliceVar(&steps, "concurrency", nil, "levels to try (default 1,2,4,…,128)")
|
||||
f.DurationVar(&cfg.Duration, "duration", cfg.Duration, "how long to hold each level")
|
||||
|
||||
@@ -57,6 +57,8 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
cfg.VLLM.Bin = bin
|
||||
case "sglang":
|
||||
cfg.SGLang.Python = bin
|
||||
case "trtllm":
|
||||
cfg.TRTLLM.Python = bin
|
||||
case "sim":
|
||||
cfg.Sim.Bin = bin
|
||||
}
|
||||
@@ -97,7 +99,7 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
continue
|
||||
default:
|
||||
var err error
|
||||
if text, err = runHelp(cmd.Context(), c); err != nil {
|
||||
if text, err = runHelp(cmd.Context(), eng, c); err != nil {
|
||||
skipped++
|
||||
fmt.Fprintf(errOut, " skipped: %v\n", err)
|
||||
continue
|
||||
@@ -167,9 +169,10 @@ func newEngineCheckCmd() *cobra.Command {
|
||||
|
||||
// representativeShard is the widest command llmgw would ever build, because a
|
||||
// flag that only appears in the multi-node case is exactly the one nobody
|
||||
// notices has been renamed.
|
||||
// notices has been renamed. For an engine that runs a replica on one node,
|
||||
// that is one node.
|
||||
func representativeShard(engine string, nodes, tp, perNode int) engines.ShardSpec {
|
||||
if nodes < 1 {
|
||||
if nodes < 1 || engine == "trtllm" {
|
||||
nodes = 1
|
||||
}
|
||||
return engines.ShardSpec{
|
||||
@@ -203,9 +206,10 @@ var helpArgs = map[string][]string{
|
||||
}
|
||||
|
||||
// runHelp asks one program to describe itself.
|
||||
func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
func runHelp(ctx context.Context, eng engines.Engine, c engines.Command) (string, error) {
|
||||
// Everything up to the first flag, plus the help request. Passing the real
|
||||
// arguments would start the engine.
|
||||
path := c.Path
|
||||
var args []string
|
||||
for _, a := range c.Args {
|
||||
if strings.HasPrefix(a, "-") {
|
||||
@@ -218,11 +222,16 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
ask = []string{"--help"}
|
||||
}
|
||||
args = append(args, ask...)
|
||||
// An engine started through a launcher of llmgw's is asked through its
|
||||
// own entry point: the interpreter's --help describes the interpreter.
|
||||
if h, ok := eng.(engines.HelpCommand); ok {
|
||||
path, args = h.HelpCommand(c)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
ec := exec.CommandContext(ctx, c.Path, args...)
|
||||
ec := exec.CommandContext(ctx, path, args...)
|
||||
// vLLM refuses to do anything at all on a machine with no accelerator,
|
||||
// including print its help, unless it is told what to assume. Asking an
|
||||
// engine about its flags should not require the hardware it runs on.
|
||||
@@ -231,7 +240,7 @@ func runHelp(ctx context.Context, c engines.Command) (string, error) {
|
||||
raw, err := ec.CombinedOutput()
|
||||
if err != nil && len(raw) == 0 {
|
||||
return "", fmt.Errorf("could not run %s: %w (install it here, or pass --help-output)",
|
||||
c.Path, err)
|
||||
path, err)
|
||||
}
|
||||
return string(raw), nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -69,3 +74,75 @@ func TestATruncatedHelpIsSuspected(t *testing.T) {
|
||||
t.Error("a truncated help was taken for a complete one")
|
||||
}
|
||||
}
|
||||
|
||||
// fakeInterpreter stands in for TensorRT-LLM's Python: asked for
|
||||
// trtllm-serve's help through its module, it prints the help given; asked
|
||||
// anything else, it fails, as running the launcher here would.
|
||||
func fakeInterpreter(t *testing.T, help string) string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "help.txt"), []byte(help), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := filepath.Join(dir, "python")
|
||||
body := "#!/bin/sh\n" +
|
||||
`if [ "$*" = "-m tensorrt_llm.commands.serve serve --help" ]; then cat "` + filepath.Join(dir, "help.txt") + `"; exit 0; fi` + "\n" +
|
||||
"echo \"unexpected: $*\" >&2; exit 2\n"
|
||||
if err := os.WriteFile(p, []byte(body), 0o755); err != nil { //nolint:gosec // an executable stand-in
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// trtllmHelp is the part of trtllm-serve serve --help (1.3) that holds the
|
||||
// flags llmgw builds.
|
||||
const trtllmHelp = `Usage: trtllm-serve serve [OPTIONS] MODEL
|
||||
|
||||
Options:
|
||||
--host TEXT Hostname of the server.
|
||||
--port INTEGER Port of the server.
|
||||
--tensor_parallel_size, --tp_size INTEGER
|
||||
Tensor parallelism size.
|
||||
--pipeline_parallel_size, --pp_size INTEGER
|
||||
Pipeline parallelism size.
|
||||
--config, --extra_llm_api_options TEXT
|
||||
Path to a YAML configuration file.
|
||||
--served_model_name TEXT The model name used in the API.
|
||||
--help Show this message and exit.
|
||||
`
|
||||
|
||||
func runEngineCheck(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
c := newEngineCheckCmd()
|
||||
var out bytes.Buffer
|
||||
c.SetOut(&out)
|
||||
c.SetErr(&out)
|
||||
c.SetArgs(args)
|
||||
err := c.ExecuteContext(context.Background())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
func TestEngineCheckAsksTrtllmServeThroughItsModule(t *testing.T) {
|
||||
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, trtllmHelp))
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(out, "every flag llmgw builds is accepted") {
|
||||
t.Errorf("unexpected report:\n%s", out)
|
||||
}
|
||||
// The interpreter's own help would describe Python, not trtllm-serve; and
|
||||
// one node is what trtllm runs, so no multi-node flag is in the command.
|
||||
if strings.Contains(out, "unexpected:") || strings.Contains(out, "skipped") {
|
||||
t.Errorf("the help was not asked of trtllm-serve:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineCheckCatchesATensorRTLLMTooOldForTheServedName(t *testing.T) {
|
||||
// TensorRT-LLM 1.2 has no --served_model_name, and a replica of it would
|
||||
// never be seen serving the name the agent waits for.
|
||||
old := strings.ReplaceAll(trtllmHelp, " --served_model_name TEXT The model name used in the API.\n", "")
|
||||
out, err := runEngineCheck(t, "trtllm", "--bin", fakeInterpreter(t, old))
|
||||
if err == nil || !strings.Contains(out, "trtllm-serve: --served_model_name") {
|
||||
t.Errorf("an old trtllm-serve passed: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,8 @@ func newMirrorCmd() *cobra.Command {
|
||||
|
||||
c.AddCommand(
|
||||
newMirrorAddCmd(&dir),
|
||||
newMirrorImportCmd(&dir),
|
||||
newMirrorImportOllamaCmd(&dir),
|
||||
newMirrorListCmd(&dir),
|
||||
newMirrorUnusedCmd(&dir),
|
||||
newMirrorRemoveCmd(&dir),
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
// placeholderBase is how the mirror's address is written when the operator
|
||||
// has not said it, as `mirror add` writes it.
|
||||
const placeholderBase = "http://<this-host>:8081"
|
||||
|
||||
func newMirrorImportCmd(dir *string) *cobra.Command {
|
||||
var model, version, as, sum, base string
|
||||
|
||||
c := &cobra.Command{
|
||||
Use: "import <file>... --model NAME",
|
||||
Short: "Copy model files on this machine into the mirror",
|
||||
Long: "For weights already here: a GGUF you converted or downloaded by hand,\n" +
|
||||
"or the files of a safetensors model. The same discipline as add — copied\n" +
|
||||
"under a temporary name and renamed when whole, the checksum kept beside\n" +
|
||||
"it for nodes to check against, an existing version never replaced.\n\n" +
|
||||
"A file named .gguf is checked to be one. Several files go into one\n" +
|
||||
"version, as the files of one model; its source is then the version,\n" +
|
||||
"ending in a slash. Run it on the mirror; for a model in Ollama, use\n" +
|
||||
"import-ollama.",
|
||||
Example: " llmgw mirror import ./qwen2.5-7b-instruct-q4_k_m.gguf --model qwen2.5-7b-instruct",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if model == "" {
|
||||
return fmt.Errorf("--model is required: it is the mirror's directory for the model")
|
||||
}
|
||||
if as != "" && len(args) > 1 {
|
||||
return fmt.Errorf("--as names one file; with %d files each keeps its own name", len(args))
|
||||
}
|
||||
if sum != "" && len(args) > 1 {
|
||||
return fmt.Errorf("--sha256 is one file's checksum; with %d files, import them one at a time", len(args))
|
||||
}
|
||||
if version == "" {
|
||||
version = time.Now().Format("2006-01-02")
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
var added []mirror.Added
|
||||
for _, src := range args {
|
||||
abs, err := filepath.Abs(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
file := as
|
||||
if file == "" {
|
||||
file = filepath.Base(src)
|
||||
}
|
||||
e := mirror.Entry{Model: model, Version: version, File: file}
|
||||
fmt.Fprintf(out, "copying %s\n", src)
|
||||
a, err := mirror.Import(cmd.Context(), *dir, abs, e, sum, "file://"+abs, progressTo(out))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s\n", model, version, file, fileSize(a.Entry.Bytes), a.SHA256)
|
||||
added = append(added, a)
|
||||
}
|
||||
fmt.Fprintf(out, "\nPoint a model at it with:\n source: %s\n", importedSource(base, added))
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&model, "model", "", "the model's name, one directory level")
|
||||
c.Flags().StringVar(&version, "version", "", "the version, one directory level (default today's date)")
|
||||
c.Flags().StringVar(&as, "as", "", "the filename to keep it as, for one file; its own name when omitted")
|
||||
c.Flags().StringVar(&sum, "sha256", "", "the checksum its publisher gave, to refuse a file that does not match")
|
||||
c.Flags().StringVar(&base, "base", placeholderBase, "the address nodes reach this mirror at, for the source printed")
|
||||
return c
|
||||
}
|
||||
|
||||
func newMirrorImportOllamaCmd(dir *string) *cobra.Command {
|
||||
var o ollamaImport
|
||||
|
||||
c := &cobra.Command{
|
||||
Use: "import-ollama [NAME[:TAG]]",
|
||||
Short: "Put a model you already have in Ollama into the mirror, for llama.cpp",
|
||||
Long: "Ollama keeps each model as a GGUF under its models directory, named by\n" +
|
||||
"its sha256. This copies that file into the mirror — checked against the\n" +
|
||||
"digest it is named by, and with a vision model's projector beside it —\n" +
|
||||
"so llama.cpp can serve it on any node, from your own network.\n\n" +
|
||||
"With no name it lists the models Ollama has. The mirror version is the\n" +
|
||||
"tag and the start of the digest, since a tag such as latest moves in\n" +
|
||||
"Ollama and a mirror version must not; --model and --version choose\n" +
|
||||
"otherwise.\n\n" +
|
||||
"Ollama's prompt template is not carried: llama.cpp renders the chat\n" +
|
||||
"template inside the GGUF. Run it on the mirror, where Ollama's directory\n" +
|
||||
"is readable — ~/.ollama/models, /usr/share/ollama/.ollama/models for\n" +
|
||||
"Ollama's Linux service, or OLLAMA_MODELS.",
|
||||
Example: " llmgw mirror import-ollama\n" +
|
||||
" sudo llmgw mirror import-ollama llama3.2:3b --ollama-dir /usr/share/ollama/.ollama/models",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
o.mirrorDir = *dir
|
||||
if len(args) == 1 {
|
||||
o.name = args[0]
|
||||
}
|
||||
return o.run(cmd.Context(), cmd.OutOrStdout())
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&o.ollamaDir, "ollama-dir", "", "Ollama's models directory (default: found where Ollama keeps it)")
|
||||
c.Flags().StringVar(&o.model, "model", "", "the mirror's directory for it (default the Ollama model's name)")
|
||||
c.Flags().StringVar(&o.version, "version", "", "the mirror version (default its tag and the start of its digest)")
|
||||
c.Flags().StringVar(&o.base, "base", placeholderBase, "the address nodes reach this mirror at, for the source printed")
|
||||
return c
|
||||
}
|
||||
|
||||
// ollamaImport is one import-ollama, from the command line or the console.
|
||||
type ollamaImport struct {
|
||||
mirrorDir, ollamaDir, name, model, version, base string
|
||||
// quiet leaves out the copy's progress, for the console, which shows
|
||||
// the output once the import is over.
|
||||
quiet bool
|
||||
}
|
||||
|
||||
func (o ollamaImport) run(ctx context.Context, out io.Writer) error {
|
||||
dir, err := mirror.OllamaDir(o.ollamaDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if o.name == "" {
|
||||
names, err := mirror.ListOllama(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(names) == 0 {
|
||||
fmt.Fprintf(out, "Ollama in %s has no models.\n", dir)
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(out, "Ollama's models in %s:\n", dir)
|
||||
for _, n := range names {
|
||||
fmt.Fprintf(out, " %s\n", n)
|
||||
}
|
||||
fmt.Fprintln(out, "\nImport one with: llmgw mirror import-ollama NAME:TAG")
|
||||
return nil
|
||||
}
|
||||
|
||||
n, err := mirror.ParseOllamaName(o.name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m, err := mirror.ReadOllama(dir, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(out, "importing %s from %s\n", n, dir)
|
||||
last := ""
|
||||
added, err := mirror.ImportOllama(ctx, o.mirrorDir, dir, m, o.model, o.version, func(file string, done, total int64) {
|
||||
if file != last {
|
||||
fmt.Fprintf(out, "copying %s\n", file)
|
||||
last = file
|
||||
}
|
||||
if !o.quiet {
|
||||
progressTo(out)(done, total)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, a := range added {
|
||||
fmt.Fprintf(out, "%s/%s/%s %s\n sha256 %s, as Ollama named it\n",
|
||||
a.Entry.Model, a.Entry.Version, a.Entry.File, fileSize(a.Entry.Bytes), a.SHA256)
|
||||
}
|
||||
for _, l := range m.Left {
|
||||
fmt.Fprintf(out, "Not carried: %s.\n", l)
|
||||
}
|
||||
|
||||
name := n.Model
|
||||
if n.Tag != "latest" {
|
||||
name += "-" + n.Tag
|
||||
}
|
||||
fmt.Fprintf(out, "\nA model definition for it, for llmgw ctl model add -f:\n\n"+
|
||||
"name: %s\nsource: %s\nprofiles:\n - name: cpu\n engine: llamacpp\n",
|
||||
name, importedSource(o.base, added))
|
||||
if m.ContextLength > 0 {
|
||||
// Ollama's context, which llama.cpp's own default would not match;
|
||||
// llama.cpp divides it between its parallel slots.
|
||||
fmt.Fprintf(out, " args: [\"--ctx-size\", %q] # Ollama ran it with num_ctx %d\n",
|
||||
strconv.Itoa(m.ContextLength), m.ContextLength)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// importedSource is what a model's source says for what was just imported:
|
||||
// the file, or the version's directory when it is several.
|
||||
func importedSource(base string, added []mirror.Added) string {
|
||||
if base == "" {
|
||||
base = placeholderBase
|
||||
}
|
||||
if len(added) == 1 {
|
||||
return added[0].Entry.URL(base)
|
||||
}
|
||||
e := added[0].Entry
|
||||
return strings.TrimSuffix(mirror.Entry{Model: e.Model, Version: e.Version}.URL(base), "/") + "/"
|
||||
}
|
||||
|
||||
// progressTo reports a copy's progress the way add reports a download's.
|
||||
func progressTo(out io.Writer) func(done, total int64) {
|
||||
return func(done, total int64) {
|
||||
if total > 0 {
|
||||
fmt.Fprintf(out, " %s of %s (%d%%)\n", fileSize(done), fileSize(total), done*100/total)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(out, " %s so far\n", fileSize(done))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
)
|
||||
|
||||
// runMirror runs `llmgw mirror ...` as an operator would and returns what it
|
||||
// printed.
|
||||
func runMirror(t *testing.T, args ...string) (string, error) {
|
||||
t.Helper()
|
||||
c := Root()
|
||||
var out bytes.Buffer
|
||||
c.SetOut(&out)
|
||||
c.SetErr(&out)
|
||||
c.SetArgs(append([]string{"mirror"}, args...))
|
||||
err := c.ExecuteContext(t.Context())
|
||||
return out.String(), err
|
||||
}
|
||||
|
||||
// ollamaWith lays out an Ollama models directory holding one model, as
|
||||
// Ollama writes it, and returns the directory and the weights' sha256.
|
||||
func ollamaWith(t *testing.T, name, tag string, weights []byte, params string) (string, string) {
|
||||
t.Helper()
|
||||
dir := filepath.Join(t.TempDir(), "models")
|
||||
blob := func(content []byte) string {
|
||||
s := sha256.Sum256(content)
|
||||
h := hex.EncodeToString(s[:])
|
||||
if err := os.MkdirAll(filepath.Join(dir, "blobs"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "blobs", "sha256-"+h), content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
w := blob(weights)
|
||||
p := blob([]byte(params))
|
||||
manifest := `{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json",` +
|
||||
`"config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:` + blob([]byte("{}")) + `","size":2},` +
|
||||
`"layers":[{"mediaType":"application/vnd.ollama.image.model","digest":"sha256:` + w + `","size":` + strconv.Itoa(len(weights)) + `},` +
|
||||
`{"mediaType":"application/vnd.ollama.image.params","digest":"sha256:` + p + `","size":` + strconv.Itoa(len(params)) + `}]}`
|
||||
mdir := filepath.Join(dir, "manifests", "registry.ollama.ai", "library", name)
|
||||
if err := os.MkdirAll(mdir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(mdir, tag), []byte(manifest), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return dir, w
|
||||
}
|
||||
|
||||
func TestImportOllamaListsThenImportsAModelForLlamaCpp(t *testing.T) {
|
||||
weights := []byte("GGUF\x03\x00\x00\x00qwen weights")
|
||||
ollama, sum := ollamaWith(t, "qwen2.5", "7b", weights, `{"num_ctx":16384}`)
|
||||
root := t.TempDir()
|
||||
|
||||
out, err := runMirror(t, "--dir", root, "import-ollama", "--ollama-dir", ollama)
|
||||
if err != nil || !strings.Contains(out, " qwen2.5:7b\n") {
|
||||
t.Fatalf("listing: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
out, err = runMirror(t, "--dir", root, "import-ollama", "qwen2.5:7b", "--ollama-dir", ollama,
|
||||
"--base", "http://models.internal:8081")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
file := filepath.Join(root, "qwen2.5", "7b-"+sum[:12], "qwen2.5-7b.gguf")
|
||||
if b, err := os.ReadFile(file); err != nil || !bytes.Equal(b, weights) {
|
||||
t.Fatalf("the GGUF is not in the mirror at %s: %v", file, err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"sha256 " + sum,
|
||||
"source: http://models.internal:8081/qwen2.5/7b-" + sum[:12] + "/qwen2.5-7b.gguf",
|
||||
"engine: llamacpp",
|
||||
`args: ["--ctx-size", "16384"]`,
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("the output does not say %q:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleImportsFromOllamaIntoTheMirrorBesideIt(t *testing.T) {
|
||||
weights := []byte("GGUF\x03\x00\x00\x00llama weights")
|
||||
ollama, sum := ollamaWith(t, "llama3.2", "3b", weights, `{}`)
|
||||
root := t.TempDir()
|
||||
|
||||
m, _ := consoleWith(t, cMirror)
|
||||
var tm tea.Model = press(m, "i")
|
||||
f := tm.(consoleModel).formD
|
||||
if f == nil || f.title != "Import from Ollama" {
|
||||
t.Fatalf("I opened %+v", f)
|
||||
}
|
||||
f.fields[0].text.SetValue("llama3.2:3b")
|
||||
f.fields[1].text.SetValue(ollama)
|
||||
f.fields[2].text.SetValue(root)
|
||||
for range len(f.fields) {
|
||||
tm = press(tm, "tab")
|
||||
}
|
||||
tm = pressRun(tm, "enter")
|
||||
|
||||
if _, err := os.Stat(filepath.Join(root, "llama3.2", "3b-"+sum[:12], "llama3.2-3b.gguf")); err != nil {
|
||||
t.Fatalf("not imported: %v", err)
|
||||
}
|
||||
text := tm.(consoleModel).dialogText
|
||||
if !strings.Contains(text, "engine: llamacpp") || strings.Contains(text, "(100%)") {
|
||||
t.Errorf("the console said:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImportCopiesALocalGGUFIntoTheMirror(t *testing.T) {
|
||||
src := filepath.Join(t.TempDir(), "q4_k_m.gguf")
|
||||
content := []byte("GGUF\x03\x00\x00\x00converted")
|
||||
if err := os.WriteFile(src, content, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := t.TempDir()
|
||||
out, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10")
|
||||
if err != nil {
|
||||
t.Fatalf("%v\n%s", err, out)
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(root, "qwen", "2026-10", "q4_k_m.gguf")); err != nil || !bytes.Equal(b, content) {
|
||||
t.Fatalf("not imported: %v", err)
|
||||
}
|
||||
if !strings.Contains(out, "source: http://<this-host>:8081/qwen/2026-10/q4_k_m.gguf") {
|
||||
t.Errorf("no source given:\n%s", out)
|
||||
}
|
||||
// The layout is the point: a version is never replaced.
|
||||
if _, err := runMirror(t, "--dir", root, "import", src, "--model", "qwen", "--version", "2026-10"); err == nil {
|
||||
t.Error("an existing version was replaced")
|
||||
}
|
||||
if _, err := runMirror(t, "--dir", root, "import", src); err == nil {
|
||||
t.Error("imported without --model")
|
||||
}
|
||||
}
|
||||
@@ -536,6 +536,10 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
case cOrgs:
|
||||
m.createOrgForm()
|
||||
}
|
||||
case "i":
|
||||
if m.menu == cMirror {
|
||||
m.mirrorImportOllamaForm()
|
||||
}
|
||||
case "n":
|
||||
if m.menu == cMonitor {
|
||||
return m, m.runCLI("Test the webhooks", "ctl", "--server="+m.client.base, "notify", "test")
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -178,10 +179,41 @@ func (m *consoleModel) mirrorFetchForm() {
|
||||
}})
|
||||
}
|
||||
|
||||
// mirrorImportOllamaForm puts a model from an Ollama on this machine into a
|
||||
// mirror on this machine: the control plane's own, when the mirror runs
|
||||
// beside it. A mirror elsewhere cannot read this machine's files, and its
|
||||
// API is given none; there it is `llmgw mirror import-ollama`, on the mirror.
|
||||
func (m *consoleModel) mirrorImportOllamaForm() {
|
||||
m.openForm(&formDialog{title: "Import from Ollama",
|
||||
note: "From an Ollama on this machine into the mirror's directory on this machine.\n" +
|
||||
"A mirror elsewhere: run llmgw mirror import-ollama there.",
|
||||
fields: []formField{
|
||||
textF("model", "", "as Ollama names it, like llama3.2:3b"),
|
||||
textF("ollama dir", "", "optional: found where Ollama keeps it"),
|
||||
textF("mirror dir", "/srv/models", "the mirror's directory on this machine"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
if v[0] == "" {
|
||||
return nil, fmt.Errorf("a model, as Ollama names it")
|
||||
}
|
||||
if st, err := os.Stat(v[2]); err != nil || !st.IsDir() {
|
||||
return nil, fmt.Errorf("%s is not a directory on this machine", v[2])
|
||||
}
|
||||
title := "Import " + v[0]
|
||||
m.busy = title
|
||||
o := ollamaImport{name: v[0], ollamaDir: v[1], mirrorDir: v[2], base: m.mirror.res.Base, quiet: true}
|
||||
return func() tea.Msg {
|
||||
var out bytes.Buffer
|
||||
err := o.run(context.Background(), &out)
|
||||
return actionDoneMsg{title: title, text: strings.TrimSpace(out.String()), err: err}
|
||||
}, nil
|
||||
}})
|
||||
}
|
||||
|
||||
func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
keys := "→ or Enter open · ↑↓ choose · Q leave"
|
||||
if m.inPanel {
|
||||
keys = "↑↓ row · Enter actions · A download one · ← back · Q leave"
|
||||
keys = "↑↓ row · Enter actions · A download one · I import from Ollama · ← back · Q leave"
|
||||
}
|
||||
if m.mirror.err != nil {
|
||||
return "Mirror", "\n " + sBad.Render(m.mirror.err.Error()), keys
|
||||
@@ -216,7 +248,7 @@ func (m consoleModel) mirrorPanel(pw int) (string, string, string) {
|
||||
rows = append(rows, []string{g.model, g.version, what, state, bytesShort(total), used})
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one."), keys
|
||||
return "Mirror", head + " " + sDim.Render("Empty. A downloads one; I imports one from Ollama."), keys
|
||||
}
|
||||
return "Mirror", head + table([]string{"MODEL", "VERSION", "FILE", "STATE", "SIZE", "USED BY"}, rows, m.row, pw, m.inPanel), keys
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user