Ilker ManapandClaude Opus 5.5
dc483d8c68
Each engine has a list of builds, best first, from what its project publishes and how each was compiled: llama.cpp CUDA 13.3, CUDA 12.8 (driver 570 for the GPUs it ships as PTX), then Vulkan, never the CPU build on an NVIDIA node; vLLM CUDA 13, then CUDA 12.9, and for GPUs older than every build of the version asked for, the newest built for them (V100: 0.19.1). The node takes the first its hardware meets and that installs and computes on the GPU, moving on when one does not, and reports the version it runs. GPU processes are read from nvidia-smi's full report, so Vulkan engines count as on their GPU. Refs #2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
40 lines
1.6 KiB
Go
40 lines
1.6 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
func newEngineBuildCmd(client func() *ctlClient) *cobra.Command {
|
|
return &cobra.Command{
|
|
Use: "engine-build <node> <engine> <build>",
|
|
Short: "Choose which build of an engine a GPU node installs",
|
|
Long: "A node installs the build of vLLM its hardware runs: CUDA 13 where the\n" +
|
|
"driver and GPUs allow, else the same version built for CUDA 12.9. This\n" +
|
|
"chooses instead, as the cluster page's vLLM build column does:\n\n" +
|
|
" auto what the hardware runs (the default)\n" +
|
|
" cu13 the CUDA 13 build, from PyPI\n" +
|
|
" cu129 the CUDA 12.9 build, from vLLM's GitHub release\n" +
|
|
" none no vLLM on this node\n\n" +
|
|
"The node is told at once and installs it in the background; a build it\n" +
|
|
"cannot compute with on its GPU fails to install, saying why, and the\n" +
|
|
"cluster page shows it. A V100 runs no build of vLLM 0.20 or later, and\n" +
|
|
"gets 0.19.1, the newest built for it, by itself.\n\n" +
|
|
" llmgw ctl engine-build gpu07 vllm cu129",
|
|
Args: cobra.ExactArgs(3),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
body, _ := json.Marshal(adminapi.NodeEngineRequest{Engine: args[1], Build: args[2]})
|
|
if err := client().post(cmd.Context(), "/api/v1/nodes/"+args[0]+"/engines", body, nil); err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s build %s; the node installs it in the background (llmgw ctl nodes shows it)\n",
|
|
args[0], args[1], args[2])
|
|
return nil
|
|
},
|
|
}
|
|
}
|