Ilker ManapandClaude Opus 5.5
15a209e652
llmgw_org, llmgw_team and llmgw_project rename and move in place; llmgw_key holds the key in a sensitive secret and is made again when revoked or expired outside Terraform; llmgw_membership puts a person in the tree. Tried with OpenTofu against the fake admin API, drift and import included. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
122 lines
2.8 KiB
Terraform
122 lines
2.8 KiB
Terraform
# A model, a team's quota and a guardrail policy, managed through the control
|
|
# plane's admin API. Run where the admin listener is reachable: on the control
|
|
# plane itself, or through `ssh -N -L 9090:127.0.0.1:9090 control-plane`.
|
|
|
|
terraform {
|
|
required_providers {
|
|
llmgw = {
|
|
source = "gitea.mynodes.xyz/ilker/llmgw"
|
|
}
|
|
}
|
|
}
|
|
|
|
provider "llmgw" {
|
|
# The defaults, written out. The token comes from LLMGW_ADMIN_TOKEN when it
|
|
# is set, and otherwise from token_file, which only root can read.
|
|
endpoint = "http://127.0.0.1:9090"
|
|
token_file = "/var/lib/llmgw/admin-token"
|
|
actor = "terraform"
|
|
}
|
|
|
|
# A small model on CPUs, loaded when asked for and stopped after half an hour
|
|
# unused.
|
|
resource "llmgw_model" "qwen_small" {
|
|
name = "qwen-small"
|
|
description = "Qwen 2.5 1.5B instruct, for quick questions"
|
|
source = "/srv/models/qwen2.5-1.5b-instruct-q4_k_m.gguf"
|
|
aliases = ["qwen"]
|
|
|
|
profiles = [{
|
|
name = "cpu"
|
|
engine = "llamacpp"
|
|
min_memory_gb = 4
|
|
cpu_cores = 4
|
|
}]
|
|
|
|
scaling = {
|
|
min = 0
|
|
max = 2
|
|
idle_timeout = "30m"
|
|
}
|
|
|
|
class = "opportunistic"
|
|
owner = "team:Acme/ml"
|
|
}
|
|
|
|
# A larger one on GPUs, always running.
|
|
resource "llmgw_model" "llama_70b" {
|
|
name = "llama-3.3-70b"
|
|
source = "meta-llama/Llama-3.3-70B-Instruct"
|
|
|
|
profiles = [{
|
|
name = "h100"
|
|
engine = "vllm"
|
|
gpu_class = "h100-80g"
|
|
tp = 4
|
|
}]
|
|
|
|
scaling = {
|
|
min = 1
|
|
max = 3
|
|
}
|
|
|
|
capabilities = {
|
|
tools = true
|
|
}
|
|
fallbacks = [llmgw_model.qwen_small.name]
|
|
}
|
|
|
|
# A team and a project in Acme, a key for its CI, and a person in the team.
|
|
resource "llmgw_team" "search" {
|
|
name = "search"
|
|
org = "Acme"
|
|
}
|
|
|
|
resource "llmgw_project" "ranking" {
|
|
name = "ranking"
|
|
team = llmgw_team.search.id
|
|
}
|
|
|
|
resource "llmgw_key" "ranking_ci" {
|
|
name = "ci"
|
|
project = llmgw_project.ranking.id
|
|
expires_in = "2160h"
|
|
}
|
|
|
|
resource "llmgw_membership" "ayse" {
|
|
user = "ayse@example.com"
|
|
target = "team:${llmgw_team.search.id}"
|
|
}
|
|
|
|
output "ranking_ci_key" {
|
|
value = llmgw_key.ranking_ci.secret
|
|
sensitive = true
|
|
}
|
|
|
|
# The ml team may hold at most eight GPUs, and is always given two.
|
|
resource "llmgw_quota" "ml" {
|
|
scope = "team:Acme/ml"
|
|
guarantee = { gpus = 2 }
|
|
limit = { gpus = 8 }
|
|
}
|
|
|
|
# The finance policy, from server.yaml's guard.policies, on the ml team.
|
|
resource "llmgw_guard_attachment" "ml_finance" {
|
|
target = "team:Acme/ml"
|
|
policy = "finance"
|
|
}
|
|
|
|
data "llmgw_nodes" "all" {}
|
|
|
|
data "llmgw_models" "all" {
|
|
depends_on = [llmgw_model.qwen_small, llmgw_model.llama_70b]
|
|
}
|
|
|
|
output "free_gpus" {
|
|
value = sum([for n in data.llmgw_nodes.all.nodes : n.gpu_free if n.state == "ready"])
|
|
}
|
|
|
|
output "models_not_ready" {
|
|
value = [for m in data.llmgw_models.all.models : m.name if m.ready_replicas < m.min_replicas]
|
|
}
|