terraform-provider-llmgw/ is a module of its own, built on terraform-plugin-framework, so the gateway does not carry its dependencies. llmgw_model is a model added at runtime: scaling, class, owner and adapters change in place, anything else replaces it; the definition is kept as written rather than as the catalog filled it in. llmgw_quota and llmgw_guard_attachment keep the scope as written beside the ID it resolved to, and import by that ID. Data sources llmgw_models and llmgw_nodes. The client lists every endpoint it calls and refuses any other; a test checks each, and every field it sends and reads, against docs/admin-api.openapi.json, so a change to the admin API that breaks the provider fails here. Acceptance tests run Terraform against an admin API in memory when TF_ACC is set. CI vets and tests the module. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
95 lines
2.2 KiB
Terraform
95 lines
2.2 KiB
Terraform
# A model, a team's quota and a guardrail policy, managed through the control
|
|
# plane's admin API. Run where the admin listener is reachable: on the control
|
|
# plane itself, or through `ssh -N -L 9090:127.0.0.1:9090 control-plane`.
|
|
|
|
terraform {
|
|
required_providers {
|
|
llmgw = {
|
|
source = "gitea.mynodes.xyz/ilker/llmgw"
|
|
}
|
|
}
|
|
}
|
|
|
|
provider "llmgw" {
|
|
# The defaults, written out. The token comes from LLMGW_ADMIN_TOKEN when it
|
|
# is set, and otherwise from token_file, which only root can read.
|
|
endpoint = "http://127.0.0.1:9090"
|
|
token_file = "/var/lib/llmgw/admin-token"
|
|
actor = "terraform"
|
|
}
|
|
|
|
# A small model on CPUs, loaded when asked for and stopped after half an hour
|
|
# unused.
|
|
resource "llmgw_model" "qwen_small" {
|
|
name = "qwen-small"
|
|
description = "Qwen 2.5 1.5B instruct, for quick questions"
|
|
source = "/srv/models/qwen2.5-1.5b-instruct-q4_k_m.gguf"
|
|
aliases = ["qwen"]
|
|
|
|
profiles = [{
|
|
name = "cpu"
|
|
engine = "llamacpp"
|
|
min_memory_gb = 4
|
|
cpu_cores = 4
|
|
}]
|
|
|
|
scaling = {
|
|
min = 0
|
|
max = 2
|
|
idle_timeout = "30m"
|
|
}
|
|
|
|
class = "opportunistic"
|
|
owner = "team:Acme/ml"
|
|
}
|
|
|
|
# A larger one on GPUs, always running.
|
|
resource "llmgw_model" "llama_70b" {
|
|
name = "llama-3.3-70b"
|
|
source = "meta-llama/Llama-3.3-70B-Instruct"
|
|
|
|
profiles = [{
|
|
name = "h100"
|
|
engine = "vllm"
|
|
gpu_class = "h100-80g"
|
|
tp = 4
|
|
}]
|
|
|
|
scaling = {
|
|
min = 1
|
|
max = 3
|
|
}
|
|
|
|
capabilities = {
|
|
tools = true
|
|
}
|
|
fallbacks = [llmgw_model.qwen_small.name]
|
|
}
|
|
|
|
# The ml team may hold at most eight GPUs, and is always given two.
|
|
resource "llmgw_quota" "ml" {
|
|
scope = "team:Acme/ml"
|
|
guarantee = { gpus = 2 }
|
|
limit = { gpus = 8 }
|
|
}
|
|
|
|
# The finance policy, from server.yaml's guard.policies, on the ml team.
|
|
resource "llmgw_guard_attachment" "ml_finance" {
|
|
target = "team:Acme/ml"
|
|
policy = "finance"
|
|
}
|
|
|
|
data "llmgw_nodes" "all" {}
|
|
|
|
data "llmgw_models" "all" {
|
|
depends_on = [llmgw_model.qwen_small, llmgw_model.llama_70b]
|
|
}
|
|
|
|
output "free_gpus" {
|
|
value = sum([for n in data.llmgw_nodes.all.nodes : n.gpu_free if n.state == "ready"])
|
|
}
|
|
|
|
output "models_not_ready" {
|
|
value = [for m in data.llmgw_models.all.models : m.name if m.ready_replicas < m.min_replicas]
|
|
}
|