# A model, a team's quota and a guardrail policy, managed through the control # plane's admin API. Run where the admin listener is reachable: on the control # plane itself, or through `ssh -N -L 9090:127.0.0.1:9090 control-plane`. terraform { required_providers { llmgw = { source = "gitea.mynodes.xyz/ilker/llmgw" } } } provider "llmgw" { # The defaults, written out. The token comes from LLMGW_ADMIN_TOKEN when it # is set, and otherwise from token_file, which only root can read. endpoint = "http://127.0.0.1:9090" token_file = "/var/lib/llmgw/admin-token" actor = "terraform" } # A small model on CPUs, loaded when asked for and stopped after half an hour # unused. resource "llmgw_model" "qwen_small" { name = "qwen-small" description = "Qwen 2.5 1.5B instruct, for quick questions" source = "/srv/models/qwen2.5-1.5b-instruct-q4_k_m.gguf" aliases = ["qwen"] profiles = [{ name = "cpu" engine = "llamacpp" min_memory_gb = 4 cpu_cores = 4 }] scaling = { min = 0 max = 2 idle_timeout = "30m" } class = "opportunistic" owner = "team:Acme/ml" } # A larger one on GPUs, always running. resource "llmgw_model" "llama_70b" { name = "llama-3.3-70b" source = "meta-llama/Llama-3.3-70B-Instruct" profiles = [{ name = "h100" engine = "vllm" gpu_class = "h100-80g" tp = 4 }] scaling = { min = 1 max = 3 } capabilities = { tools = true } fallbacks = [llmgw_model.qwen_small.name] } # A team and a project in Acme, a key for its CI, and a person in the team. resource "llmgw_team" "search" { name = "search" org = "Acme" } resource "llmgw_project" "ranking" { name = "ranking" team = llmgw_team.search.id } resource "llmgw_key" "ranking_ci" { name = "ci" project = llmgw_project.ranking.id expires_in = "2160h" } resource "llmgw_membership" "ayse" { user = "ayse@example.com" target = "team:${llmgw_team.search.id}" } output "ranking_ci_key" { value = llmgw_key.ranking_ci.secret sensitive = true } # The ml team may hold at most eight GPUs, and is always given two. resource "llmgw_quota" "ml" { scope = "team:Acme/ml" guarantee = { gpus = 2 } limit = { gpus = 8 } } # The finance policy, from server.yaml's guard.policies, on the ml team. resource "llmgw_guard_attachment" "ml_finance" { target = "team:Acme/ml" policy = "finance" } data "llmgw_nodes" "all" {} data "llmgw_models" "all" { depends_on = [llmgw_model.qwen_small, llmgw_model.llama_70b] } output "free_gpus" { value = sum([for n in data.llmgw_nodes.all.nodes : n.gpu_free if n.state == "ready"]) } output "models_not_ready" { value = [for m in data.llmgw_models.all.models : m.name if m.ready_replicas < m.min_replicas] }