Ilker ManapandClaude Opus 5.5
c0d5194ea0
Strata (github.com/Niko1221/Strata) keeps a mixture-of-experts model's experts in RAM and the ones used most in VRAM; on cuda01's RTX 6000 it serves Qwen3.8-Flash-Next 125B at 70-87 tokens a second to one client. - engines/strata.go: the adapter. The agent starts llmgw strata-serve, which runs Strata's setup the way its desktop app does (STRATA_CONTROL_INSTALL: installs nothing, writes only under the replica's and the node's directories) on the fetched GGUF files, points the configuration it writes at the installed engine, and execs Strata's server. The key goes in STRATA_API_KEY, never on a command line. - A profile's args go to setup: --family, --size (setup's --model, which is the source everywhere else), --context, --parallel, ... Flags that move files or pick the engine or GPU are refused. - provision: the source pinned by digest in a Python environment, and the release's prebuilt engine unpacked beside it, checked against the digest GitHub publishes: CUDA 13 (7.5+, driver 580) or CUDA 12.9 (6.0-8.9, driver 575). Python builds can now carry release assets. - Metrics are vLLM's names under /metrics/prometheus. - The catalog wants one card and a family, and a directory, not one .gguf. Opt-in: not in DefaultEngines. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
44 lines
1.7 KiB
Go
44 lines
1.7 KiB
Go
package cli
|
|
|
|
import (
|
|
"os/signal"
|
|
"syscall"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
|
)
|
|
|
|
func newStrataServeCmd() *cobra.Command {
|
|
var cfg strata.Config
|
|
cmd := &cobra.Command{
|
|
Use: "strata-serve [flags] [-- setup.py flags]",
|
|
Short: "Run Strata as an engine: prepare the model, then serve it (started by the agent)",
|
|
Hidden: true,
|
|
Long: "Started by the node agent for a replica whose engine is strata. Prepares the\n" +
|
|
"model from the fetched GGUF files with Strata's setup, which skips what it has\n" +
|
|
"done before, then becomes Strata's server on the replica's address. The key is\n" +
|
|
"read from " + strata.KeyEnv + ".",
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM)
|
|
defer stop()
|
|
cfg.Args = args
|
|
cfg.Log = logger()
|
|
return strata.Run(ctx, cfg)
|
|
},
|
|
}
|
|
f := cmd.Flags()
|
|
f.StringVar(&cfg.Listen, "listen", "", "the replica's address")
|
|
f.StringVar(&cfg.Model, "model", "", "the model's name, for /v1/models")
|
|
f.StringVar(&cfg.Weights, "weights", "", "the fetched GGUF files' directory")
|
|
f.StringVar(&cfg.State, "state", "", "the replica's own directory")
|
|
f.StringVar(&cfg.Data, "data", "", "the node's directory for prepared models, shared by its replicas")
|
|
f.StringVar(&cfg.Python, "python", "python3", "the interpreter of Strata's environment")
|
|
f.StringVar(&cfg.Dir, "strata", "", "Strata's directory, where setup.py is")
|
|
f.StringVar(&cfg.Engine, "engine", "", "Strata's engine directory, where the strata binary is")
|
|
for _, n := range []string{"listen", "weights", "state", "data", "strata", "engine"} {
|
|
_ = cmd.MarkFlagRequired(n)
|
|
}
|
|
return cmd
|
|
}
|