Ilker ManapandClaude Opus 5.5
81685ecd75
With restore: true on a guardrail policy's input, what redact rewrote as placeholders comes back in the answer: the model reads "[EMAIL_1]", the client reads the address it sent. A streamed placeholder split across events is held until it is whole, and what is held is sent at the end. The swap runs before the output filter, so the output rules judge the answer as the client reads it and a placeholder they make of a value the model invented is never taken for the request's own. The map lives in the Verdict for the request only; nothing stores, logs or traces it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
633 lines
94 KiB
Go
633 lines
94 KiB
Go
// Code generated by go run ./internal/docgen/gen; DO NOT EDIT.
|
|
|
|
package docgen
|
|
|
|
// Reference is every configuration block, as Collect reads them.
|
|
var Reference = []Block{
|
|
{Package: "catalog", Name: "Adapter", Title: "Adapter", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "Name is what clients ask for, like a model name.", Nested: ""},
|
|
{Key: "source", Type: "string", Doc: "Source is the adapter: a GGUF file for llama.cpp; for vLLM a PEFT\nadapter directory (a mirror version ending in \"/\", a path on the node,\nor a Hugging Face repository id). Fetched by each node as the model's\nweights are.", Nested: ""},
|
|
{Key: "scale", Type: "float64", Doc: "Scale is how strongly it is applied; zero takes 1.", Nested: ""},
|
|
{Key: "owner", Type: "string", Doc: "Owner is the part of the tenancy tree the adapter belongs to —\norg:ID, team:ID, project:ID, user:ID or key:ID — and only callers in\nit see it: to anyone else it is a model that does not exist. An\nadapter trained on an organisation's documents carries some of what\nthey say in its weights. Empty is every caller the model allows.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Admission", Title: "Admission", Doc: "", Fields: []Field{
|
|
{Key: "enabled", Type: "bool", Doc: "Enabled turns the layer on. Off, every request goes straight through,\nwhich is the v0.2.0 behaviour.", Nested: ""},
|
|
{Key: "bands", Type: "[]Band", Doc: "", Nested: ""},
|
|
{Key: "default_band", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "band_header", Type: "string", Doc: "BandHeader lets a client declare its band. Until authentication lands in\nv1.0.0 this is trusted, so the port must not be exposed to callers who\nshould not be able to promote themselves.", Nested: ""},
|
|
{Key: "queue_ttl", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "queue_capacity_per_band", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "queue_capacity_total", Type: "int", Doc: "QueueCapacityTotal bounds everything queued across bands. Reached, a\nrequest is not refused while a lower band has any queued: the newest of\nthe lowest band is turned away instead, with a 503 and Retry-After.\nZero takes half of all the bands' capacities together.", Nested: ""},
|
|
{Key: "interval", Type: "time.Duration", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "AgentSettings", Title: "AgentSettings", Doc: "", Fields: []Field{
|
|
{Key: "default", Type: "map[string]any", Doc: "Default is agent.yaml's keys for every node.", Nested: ""},
|
|
{Key: "nodes", Type: "map[string]map[string]any", Doc: "Nodes are agent.yaml's keys for one node each, by node id, over\nDefault.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Auth", Title: "Auth", Doc: "", Fields: []Field{
|
|
{Key: "enabled", Type: "bool", Doc: "", Nested: ""},
|
|
{Key: "store", Type: "string", Doc: "Store is the SQLite file holding keys, tenancy and usage.", Nested: ""},
|
|
{Key: "cache_ttl", Type: "time.Duration", Doc: "CacheTTL bounds how long a revoked key keeps working, traded against a\ndatabase read per key per TTL.", Nested: ""},
|
|
{Key: "negative_ttl", Type: "time.Duration", Doc: "NegativeTTL is how long a rejected key is remembered as rejected, so a\nclient looping with a bad key cannot become a database read per request.", Nested: ""},
|
|
{Key: "default_rpm", Type: "int", Doc: "DefaultRPM and DefaultTPM apply to a key that no scope gives a limit to.\n\nUnlimited used to be the default, and a project created without limits is\nthe easy path — the \"default\" project on this cluster still is one. The\nfirst sign of that is the bill or the outage. Zero keeps the old\nbehaviour for anyone who wants it, but it should be a choice rather than\nan omission. A fallback, not a ceiling: an explicit limit at any scope\nwins, above or below.", Nested: ""},
|
|
{Key: "default_tpm", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "signup_open", Type: "bool", Doc: "SignupOpen serves the signup form. Accounts it creates are pending until\nan operator approves them, so this is not a way in — but turning it off\nmeans accounts are made with `llmgw users create` instead.", Nested: ""},
|
|
{Key: "oidc", Type: "OIDC", Doc: "OIDC is sign-in through an identity provider.", Nested: "config.OIDC"},
|
|
{Key: "ldap", Type: "LDAP", Doc: "LDAP is sign-in against a directory, Active Directory above all, and\nthe accounts kept in step with it.", Nested: "config.LDAP"},
|
|
{Key: "scim", Type: "SCIM", Doc: "SCIM is account provisioning by the identity provider.", Nested: "config.SCIM"},
|
|
{Key: "max_key_ttl", Type: "time.Duration", Doc: "MaxKeyTTL is how long any API key may live; a key asked for with no\nexpiry is given this. Organizations, teams and projects can set\nshorter ones (llmgw keys ttl). Zero is no limit.", Nested: ""},
|
|
{Key: "personal", Type: "Personal", Doc: "Personal is what a person may do on their own, outside any project: make\npersonal keys and use the chat page personally, within these limits.", Nested: "config.Personal"},
|
|
{Key: "pricing", Type: "map[string]Price", Doc: "Pricing is what each model costs, for budgets. A model absent from the\ntable costs whatever DefaultPrice says, which is nothing unless set.", Nested: ""},
|
|
{Key: "default_price", Type: "Price", Doc: "", Nested: "config.Price"},
|
|
{Key: "flush_interval", Type: "time.Duration", Doc: "FlushInterval and BatchSize decide when usage records are written.", Nested: ""},
|
|
{Key: "batch_size", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "buffer_size", Type: "int", Doc: "BufferSize bounds the queue. Beyond it records are dropped rather than\nblocking a request: losing accounting is bad, turning a stalled database\ninto an outage is worse.", Nested: ""},
|
|
{Key: "usage_retention", Type: "time.Duration", Doc: "UsageRetention is how long the record of each request is kept: who sent\nit, which replica answered, how long it queued — what `ctl request`\nreads. Zero keeps it for ever. Totals, charts and budgets are read from\nfive-minute sums kept beside the records, which are not pruned, so this\nbounds the table that grows by a row a request without changing any\nfigure a page shows. At least a day when set.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Batch", Title: "Batch", Doc: "", Fields: []Field{
|
|
{Key: "max_file_bytes", Type: "int64", Doc: "MaxFileBytes bounds one uploaded file; 100 MB unless set.", Nested: ""},
|
|
{Key: "max_lines", Type: "int", Doc: "MaxLines bounds the requests in one batch; 50 000 unless set.", Nested: ""},
|
|
{Key: "concurrency", Type: "int", Doc: "Concurrency is how many of one batch's requests are in flight at\nonce; 4 unless set. They queue in the lowest admission band, behind\nevery interactive request.", Nested: ""},
|
|
{Key: "retain_days", Type: "int", Doc: "RetainDays is how long a batch's answer and error files are kept;\n30 unless set. Uploaded files are kept until deleted.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Cache", Title: "Cache", Doc: "", Fields: []Field{
|
|
{Key: "max_bytes", Type: "int64", Doc: "MaxBytes bounds every project's answers together; 128 MB unless set.\nPast it, the answers used longest ago go first.", Nested: ""},
|
|
{Key: "max_entry_bytes", Type: "int64", Doc: "MaxEntryBytes bounds one answer; a longer one is not kept. 1 MB\nunless set.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Canary", Title: "Canary", Doc: "", Fields: []Field{
|
|
{Key: "model", Type: "string", Doc: "Model is the candidate: a catalog model, usually this one at newer\nweights, which clients cannot name themselves.", Nested: ""},
|
|
{Key: "percent", Type: "int", Doc: "Percent of requests go to it, 0 to 100.", Nested: ""},
|
|
{Key: "min_requests", Type: "int", Doc: "The candidate is taken out — rolled back — when, over at least\nMinRequests of its own, its error rate is MaxErrorIncrease above\nthis model's (0.05 is five points), or its mean time to the first\ntoken is MaxTTFTRatio times this model's. Zero is 50, 0.05 and 1.5.", Nested: ""},
|
|
{Key: "max_error_increase", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "max_ttft_ratio", Type: "float64", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Capabilities", Title: "Capabilities", Doc: "", Fields: []Field{
|
|
{Key: "tools", Type: "bool", Doc: "", Nested: ""},
|
|
{Key: "vision", Type: "bool", Doc: "", Nested: ""},
|
|
{Key: "embeddings", Type: "bool", Doc: "", Nested: ""},
|
|
{Key: "images", Type: "bool", Doc: "Images is a model that makes pictures, asked for at\n/v1/images/generations. Implied by an external model whose API is\nComfyUI's.", Nested: ""},
|
|
{Key: "meshes", Type: "bool", Doc: "Meshes is a model that makes 3D models from a picture, asked for as a\njob at /v1/3d/generations. Implied by a ComfyUI model whose output is\nmeshes.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "ComfyUI", Title: "ComfyUI", Doc: "", Fields: []Field{
|
|
{Key: "workflow", Type: "string", Doc: "Workflow is a built-in's name (flux-schnell, the default, or\nflux-dev) or a JSON file on the control plane, in ComfyUI's API\nformat.", Nested: ""},
|
|
{Key: "checkpoint", Type: "string", Doc: "Checkpoint is the file the workflow's {{checkpoint}} names: the\nsource's own name unless set, which a source that is a set of files\nmust.", Nested: ""},
|
|
{Key: "output", Type: "string", Doc: "Output is what the workflow makes: pictures (the default), asked for\nat /v1/images/generations; meshes, asked for as a job at\n/v1/3d/generations from a picture; or video, audio or files. Every\nkind is asked for as a job at /v1/workflows/runs too.", Nested: ""},
|
|
{Key: "inputs", Type: "[]Input", Doc: "Inputs are what the workflow takes from a request at\n/v1/workflows/runs, each the placeholder of its name.", Nested: ""},
|
|
{Key: "nodes", Type: "[]CustomNode", Doc: "Nodes are custom nodes the workflow needs, put in each replica's own\ncustom_nodes at the commit named.", Nested: ""},
|
|
{Key: "huggingface_repo", Type: "string", Doc: "HuggingFaceRepo is the repository a custom node loads the weights by\nname from (Stable Fast 3D's loader asks for stabilityai/stable-fast-3d):\nthe replica's weights are shown to the node there, in the Hugging Face\ncache of the engines' user, so nothing is fetched and no token is\nneeded when it runs.", Nested: ""},
|
|
{Key: "environment", Type: "string", Doc: "Environment names the models that may share one ComfyUI replica: every\nmodel of an environment agrees on the custom nodes they both name, so\none replica can carry all of them and ComfyUI moves between their\nweights itself. Empty is an environment of the model alone.", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "ComfyUIConfig", Title: "ComfyUIConfig", Doc: "", Fields: []Field{
|
|
{Key: "dir", Type: "string", Doc: "Dir is ComfyUI's directory, where main.py is. Readable by the user\nengines run as.", Nested: ""},
|
|
{Key: "python", Type: "string", Doc: "Python is the interpreter of its environment; Dir's .venv when unset.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Control", Title: "Control", Doc: "", Fields: []Field{
|
|
{Key: "interval", Type: "time.Duration", Doc: "Interval is how often desired and actual state are compared.", Nested: ""},
|
|
{Key: "base_backoff", Type: "time.Duration", Doc: "BaseBackoff and MaxBackoff bound how fast a failing model is retried.", Nested: ""},
|
|
{Key: "max_backoff", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "unreachable_after", Type: "time.Duration", Doc: "UnreachableAfter and LostAfter stage node expiry. They are deliberately\nfar apart: a network blip must not tear down replicas that still serve,\nand rebuilding a multi-node replica costs tens of minutes of loading.", Nested: ""},
|
|
{Key: "lost_after", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "port_range", Type: "[]int", Doc: "PortRange is the per-node range for engine HTTP and dist-init ports.", Nested: ""},
|
|
{Key: "mgmt_interface", Type: "string", Doc: "MgmtInterface is the ethernet interface NCCL bootstraps over.", Nested: ""},
|
|
{Key: "timezone", Type: "string", Doc: "Timezone is where models' scaling schedules are read, an IANA name\nlike Europe/Istanbul; empty is the control plane's own.", Nested: ""},
|
|
{Key: "eval_dir", Type: "string", Doc: "EvalDir is where quality suites are read from by name, for `llmgw ctl\neval MODEL --suite NAME`, a model's eval gate and the console. The\nsuites the binary carries (arithmetic, json-format) are found when it\nhas none of the name; a suite here of the same name replaces one.", Nested: ""},
|
|
{Key: "enabled", Type: "bool", Doc: "Enabled runs the control plane even when configuration declares no\nmodels.\n\nWithout it, a deployment whose catalog lives entirely in the database\ncannot start: control plane mode is inferred from the config file, so an\nempty models block means \"router only\", the process refuses to start for\nwant of anything to serve, and the models it would have loaded are never\nread. That is a cluster that cannot be managed at runtime because it is\nmanaged at runtime. AgentListen cannot carry this signal — it has a\ndefault, so every deployment would become a control plane.", Nested: ""},
|
|
{Key: "lease_ttl", Type: "time.Duration", Doc: "LeaseTTL is how long this control plane's claim to be the one placing\nwork stands without being renewed.\n\nOnly meaningful with a shared store, where a second control plane can\nsee the claim. Long enough to survive a slow query or a brief network\nstall, short enough that a standby takes over before anyone has finished\nreading the alert. Zero takes the default, 10s.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "CustomNode", Title: "CustomNode", Doc: "", Fields: []Field{
|
|
{Key: "repo", Type: "string", Doc: "Repo is the repository, https only.", Nested: ""},
|
|
{Key: "commit", Type: "string", Doc: "Commit is the full commit the node is used at: a branch moves, and a\nnode that changed under a model is a model that changed unasked.", Nested: ""},
|
|
{Key: "requirements", Type: "[]string", Doc: "Requirements are the PyPI packages the node needs, by name and version\nrange only. They are installed beside the replica, never into\nComfyUI's environment: a package named here at the range named, and\nevery other one at the version ComfyUI already runs. One named here\nis first on ComfyUI's path, so its range must suit ComfyUI too.", Nested: ""},
|
|
{Key: "build", Type: "[]string", Doc: "Build are directories of the repository that are Python packages to\nbe compiled rather than downloaded: C++ and CUDA extensions no index\nhas (Stable Fast 3D's texture_baker and uv_unwrapper). Each is built\nas a wheel with ComfyUI's Python for the card the replica runs on,\nkept on the node for the next replica on the same kind of card, and\ninstalled beside the replica like its requirements.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Endpoint", Title: "Endpoint", Doc: "", Fields: []Field{
|
|
{Key: "id", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "url", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "api_key", Type: "string", Doc: "APIKey is sent to the engine as a bearer token, for an engine started\nwith one.", Nested: ""},
|
|
{Key: "models", Type: "[]string", Doc: "", Nested: ""},
|
|
{Key: "engine", Type: "string", Doc: "Engine names the inference engine, which decides what its metrics are\ncalled. Empty means vLLM's names.", Nested: ""},
|
|
{Key: "workflow", Type: "string", Doc: "Workflow is the ComfyUI workflow for engine: comfyui, a built-in's\nname or its JSON; flux-schnell unless set.", Nested: ""},
|
|
{Key: "pool", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "weight", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "priority", Type: "int", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "EngineOverride", Title: "EngineOverride", Doc: "", Fields: []Field{
|
|
{Key: "gpu_class", Type: "string", Doc: "GPUClass picks nodes with a GPU of this class (\"v100-32g\").", Nested: ""},
|
|
{Key: "nodes", Type: "[]string", Doc: "Nodes picks nodes by id.", Nested: ""},
|
|
{Key: "labels", Type: "map[string]string", Doc: "Labels picks nodes carrying all of these.", Nested: ""},
|
|
{Key: "engines", Type: "map[string]string", Doc: "Engines are laid over the cluster's: a version, or \"\" for none.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "EvalGate", Title: "EvalGate", Doc: "", Fields: []Field{
|
|
{Key: "suite", Type: "string", Doc: "Suite is a suite's name in control.eval_dir, or one the binary\ncarries (arithmetic, json-format).", Nested: ""},
|
|
{Key: "min_score", Type: "float64", Doc: "MinScore is the score, 0 to 1, to pass at; zero is the suite's own\nthreshold.", Nested: ""},
|
|
{Key: "judge", Type: "string", Doc: "Judge grades the suite's rubric cases, instead of the judge it names.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "External", Title: "External", Doc: "", Fields: []Field{
|
|
{Key: "url", Type: "string", Doc: "URL is the provider's base address, without /v1: https://api.openai.com.", Nested: ""},
|
|
{Key: "model", Type: "string", Doc: "Model is what the provider calls it; requests are rewritten to it.", Nested: ""},
|
|
{Key: "api_key_file", Type: "string", Doc: "APIKeyFile holds the provider's key, on the control plane. A file\nrather than the key itself, so the key is in neither the catalog, the\nstore nor the audit log; empty for a provider that asks for none.", Nested: ""},
|
|
{Key: "api_key_files", Type: "[]ExternalKey", Doc: "APIKeyFiles are several keys for the provider, in place of\nAPIKeyFile: one endpoint each (external:MODEL#1, #2…), so a key that\nanswers 429 or 401 is passed over while its breaker is open and the\nrequest goes to another. A provider's limits are per key.", Nested: ""},
|
|
{Key: "peer", Type: "string", Doc: "Peer names the federated llmgw cluster this model is offered by, for\na model that federation added (`llmgw ctl federation add`) rather\nthan one written by hand. Such a model is named PEER/MODEL, comes and\ngoes with the peer's own list, and is sent only while the peer is up.", Nested: ""},
|
|
{Key: "api", Type: "string", Doc: "API is what the provider speaks: OpenAI's, unless this says comfyui —\na ComfyUI server, asked for pictures at /v1/images/generations and\nsent the workflow below with the request's prompt in it. Model is then\nthe checkpoint file the workflow loads, as ComfyUI lists it.", Nested: ""},
|
|
{Key: "workflow", Type: "string", Doc: "Workflow is the ComfyUI workflow, in its API format: a built-in's name\n(flux-schnell, the default, or flux-dev) or a JSON file on the control\nplane.", Nested: ""},
|
|
{Key: "output", Type: "string", Doc: "Output is what it makes: pictures, or meshes (see ComfyUI.Output).", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "ExternalKey", Title: "ExternalKey", Doc: "", Fields: []Field{
|
|
{Key: "file", Type: "string", Doc: "File holds the key, on the control plane, as APIKeyFile does.", Nested: ""},
|
|
{Key: "weight", Type: "float64", Doc: "Weight is its share of the requests against the others'; zero is 1.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Facility", Title: "Facility", Doc: "", Fields: []Field{
|
|
{Key: "budgets", Type: "[]FacilityBudget", Doc: "Budgets are the domains with a power cap.", Nested: ""},
|
|
{Key: "margin", Type: "float64", Doc: "Margin is the share of every cap kept free, 0.1 for ten percent. Zero\nkeeps none, for caps already given derated.", Nested: ""},
|
|
{Key: "gpu_watts", Type: "float64", Doc: "GPUWatts is a card's draw when it does not report its power limit:\nan agent without nvidia-smi's power fields, or a node that has not\nsent a heartbeat yet. Zero counts such a card as drawing nothing.", Nested: ""},
|
|
{Key: "cpu_core_watts", Type: "float64", Doc: "CPUCoreWatts is the draw of a core reserved by a replica with no GPU\n(cpu_cores). Zero counts such replicas as drawing nothing.", Nested: ""},
|
|
{Key: "node_watts", Type: "float64", Doc: "NodeWatts is what a node draws besides its cards and reserved cores —\nidle CPUs, memory, fans, disks — counted for every node in a capped\ndomain, busy or not.", Nested: ""},
|
|
{Key: "measured_fresh", Type: "time.Duration", Doc: "MeasuredFresh is how old a node's power reading may be and still be\ncounted against its domains. Zero takes two minutes.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "FacilityBudget", Title: "FacilityBudget", Doc: "", Fields: []Field{
|
|
{Key: "domain", Type: "string", Doc: "Domain is level:name, as the nodes' labels give it: rack:r12-07,\npdu:r12-07-a, row:12.", Nested: ""},
|
|
{Key: "cap_watts", Type: "float64", Doc: "CapWatts is what the domain may draw. For a dual-fed rack it is what\none feed carries alone, derated as the facility derates it (usually\n80%), since that is what the rack has to survive.", Nested: ""},
|
|
{Key: "other_watts", Type: "float64", Doc: "OtherWatts is drawn in the domain by what llmgw does not place on:\nswitches, storage, machines outside the cluster.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Federation", Title: "Federation", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "Name is this cluster as a peer reads it, in the federated_from tag\nsent with a request to a peer set to receive tags (`ctl federation add\n--send-tags`). Empty is this machine's host name.", Nested: ""},
|
|
{Key: "interval", Type: "time.Duration", Doc: "Interval is how often each peer is asked whether it is ready, timed,\nand what it offers. A peer that does not answer is taken out of\nrouting until it does. Zero is 10s.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "FirewallSettings", Title: "FirewallSettings", Doc: "", Fields: []Field{
|
|
{Key: "enabled", Type: "bool", Doc: "Enabled keeps llmgw's internal ports — engines, node exporter, the\nagents' stream, the mirror, the store — to the cluster's networks on\nevery machine. False removes llmgw's nftables table everywhere, for a\nsite whose own firewall does that job; the rules are kept, and true\nputs them back. Changed in the cluster's settings, it is in force on\nevery node and control plane within seconds.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "GPUNeed", Title: "GPUNeed", Doc: "", Fields: []Field{
|
|
{Key: "min_compute_capability", Type: "float64", Doc: "MinComputeCapability is the oldest architecture that runs it: 7.5 is\nTuring, 8.0 Ampere (bf16, flash-attention 2), 8.9 Ada (fp8), 9.0 Hopper.", Nested: ""},
|
|
{Key: "min_memory_gb", Type: "int", Doc: "MinMemoryGB is the card's own memory it needs.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "GuardModel", Title: "GuardModel", Doc: "", Fields: []Field{
|
|
{Key: "format", Type: "string", Doc: "Format is qwen3guard (Qwen3Guard-Gen: \"Safety: Safe\", \"Unsafe\" or\n\"Controversial\", and the categories), llamaguard3 (Meta's Llama Guard\n3: \"safe\", or \"unsafe\" and the categories), shieldgemma (Google's\nShieldGemma: Yes or No), or json ({\"safe\": false, \"categories\":\n[\"...\"]}).", Nested: ""},
|
|
{Key: "categories", Type: "[]string", Doc: "Categories the model is asked about, for shieldgemma — each a policy\nsentence — and the names reported for llamaguard3's S1, S2, ….", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Input", Title: "Input", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "Name is the placeholder's, and the request's key.", Nested: ""},
|
|
{Key: "type", Type: "string", Doc: "Type is one of text, int, float, bool, choice, image, mask, audio,\nvideo, file. The last five are files, sent as base64, a data URL, or\nthe id of a file uploaded to /v1/files, and put in ComfyUI's input\ndirectory; the placeholder gets the name they were given there.", Nested: ""},
|
|
{Key: "required", Type: "bool", Doc: "Required inputs must be in every request; the others take Default,\nor the workflow's own {{name|default}}.", Nested: ""},
|
|
{Key: "default", Type: "any", Doc: "", Nested: ""},
|
|
{Key: "min", Type: "*float64", Doc: "Min and Max bound a number; for text, Max bounds its length.", Nested: ""},
|
|
{Key: "max", Type: "*float64", Doc: "", Nested: ""},
|
|
{Key: "choices", Type: "[]string", Doc: "Choices are what a choice may be.", Nested: ""},
|
|
{Key: "max_mb", Type: "int", Doc: "MaxMB bounds a file input; zero is the type's own bound.", Nested: ""},
|
|
{Key: "doc", Type: "string", Doc: "Doc says what the input is for, to whoever fills it in.", Nested: ""},
|
|
}},
|
|
{Package: "guard", Name: "JudgePolicy", Title: "JudgePolicy", Doc: "", Fields: []Field{
|
|
{Key: "model", Type: "string", Doc: "Model is a catalog model with role: guard.", Nested: ""},
|
|
{Key: "models", Type: "[]string", Doc: "Models are judges asked in turn, instead of Model: the first that\nanswers gives the verdict. A judge on a GPU first and one on CPUs\nafter it is the GPU's speed while it is there, and the CPUs' when it\nis not — busy, failed, or the node down.", Nested: ""},
|
|
{Key: "input", Type: "string", Doc: "Input and Output are what an unsafe verdict does: flag or block.\nEmpty does not ask the judge about that direction.", Nested: ""},
|
|
{Key: "output", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "categories", Type: "[]string", Doc: "Categories, when set, are the only ones that count: Llama Guard's\ncodes (S1, S10) or the names the model reports.", Nested: ""},
|
|
{Key: "controversial", Type: "string", Doc: "Controversial is what Qwen3Guard's third verdict counts as: safe\n(the default) or unsafe.", Nested: ""},
|
|
{Key: "latency_budget", Type: "time.Duration", Doc: "LatencyBudget is how long each judge is waited for before the next is\nasked, or, after the last, the policy's fail applies. Zero is 250ms.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "KVIndex", Title: "KVIndex", Doc: "", Fields: []Field{
|
|
{Key: "tokenize_timeout", Type: "time.Duration", Doc: "TokenizeTimeout bounds the call that turns a request into token ids,\nmade to one of the model's replicas before the request is placed. Past\nit the request is routed as tier 1 would. Zero takes 250ms.", Nested: ""},
|
|
{Key: "min_blocks", Type: "int", Doc: "MinBlocks is the fewest leading blocks a replica must hold to be\npreferred for them; fewer is coincidence — a chat template's opening\ntokens — rather than a prefix worth following. Zero takes 4, which is\n64 tokens at vLLM's usual block size of 16.", Nested: ""},
|
|
{Key: "max_blocks", Type: "int", Doc: "MaxBlocks bounds how many of a prompt's blocks are hashed and looked\nup. Beyond it a longer match still helps, but the difference no longer\ndecides anything. Zero takes 4096.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "LDAP", Title: "LDAP", Doc: "", Fields: []Field{
|
|
{Key: "urls", Type: "[]string", Doc: "URLs are the directory's servers, tried in turn: ldaps:// (636), or\nldap:// (389) with StartTLS.", Nested: ""},
|
|
{Key: "start_tls", Type: "bool", Doc: "StartTLS upgrades an ldap:// connection before anything is sent. A\npassword never crosses a connection without TLS: ldap:// without it\nis refused.", Nested: ""},
|
|
{Key: "ca_file", Type: "string", Doc: "CAFile is the authority the servers' certificates are checked against,\nwhen it is not the system's: an AD Certificate Services root.", Nested: ""},
|
|
{Key: "ca", Type: "string", Doc: "CA is the same authority written out, PEM: kept in the cluster's\nsettings, so no file has to be put on every control plane. Used in\nplace of CAFile when both are set.", Nested: ""},
|
|
{Key: "bind_dn", Type: "string", Doc: "BindDN and the file holding its password: a service account that may\nread users and groups. Nothing is written to the directory.", Nested: ""},
|
|
{Key: "bind_password_file", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "base_dn", Type: "string", Doc: "BaseDN is where people are searched for, subtree included.", Nested: ""},
|
|
{Key: "group_base_dn", Type: "string", Doc: "GroupBaseDN is where their groups are; empty is BaseDN's domain root.", Nested: ""},
|
|
{Key: "user_filter", Type: "string", Doc: "UserFilter narrows who may sign in, ANDed with the login match: a\ngroup (memberOf=...) or an OU's people only. Empty is every enabled\nuser object.", Nested: ""},
|
|
{Key: "label", Type: "string", Doc: "Label is what the form says to sign in with: \"Sign in with your\n<label> account\". Empty is \"Active Directory\".", Nested: ""},
|
|
{Key: "create_accounts", Type: "bool", Doc: "CreateAccounts makes an account, active, for a person the directory\nvouches for and the gateway does not know. Off, only existing accounts\nsign in this way.", Nested: ""},
|
|
{Key: "roles", Type: "map[string][]string", Doc: "Roles maps a staff role (viewer, operator, billing, admin) to the\ngroups that give it, by name (CN) or whole DN. Set at every sign-in\nand sync: someone taken out of the group loses the role.", Nested: ""},
|
|
{Key: "memberships", Type: "map[string]string", Doc: "Memberships maps a group, by CN or DN, to a place in the tree —\norg:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT — joined at sign-in\nand kept by the sync, which also takes away a place whose group the\nperson has left.", Nested: ""},
|
|
{Key: "sync_interval", Type: "time.Duration", Doc: "SyncInterval is how often the leader reads the directory. Zero is\n15m; negative turns the sync off, leaving only sign-in.", Nested: ""},
|
|
{Key: "timeout", Type: "time.Duration", Doc: "Timeout bounds each round trip to a server. Zero is 10s.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Landing", Title: "Landing", Doc: "", Fields: []Field{
|
|
{Key: "contact", Type: "string", Doc: "Contact is where someone without a key should go — an address, a URL, a\nchannel name. Rendered as a link when it is one, as text otherwise.", Nested: ""},
|
|
{Key: "notice", Type: "string", Doc: "Notice is an optional line of your own, for whatever the reader needs to\nknow that this page cannot guess.", Nested: ""},
|
|
{Key: "language", Type: "string", Doc: "Language is what the web portal speaks to someone who has not chosen\na language with the switcher at the foot of every page and whose\nbrowser asks for none it speaks: \"en\" or \"tr\". Empty is English.", Nested: ""},
|
|
{Key: "docs_sign_in", Type: "bool", Doc: "DocsSignIn serves the docs portal at /docs only to someone signed in.\nOff, the portal is public like this page: its client guide is for the\nperson who has no account yet, and the operator documents describe\nllmgw in general — no address, model or key of this installation —\nwhich the project's repository publishes anyway. On, a visitor is sent\nto sign in, and without accounts the portal needs an API key.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "License", Title: "License", Doc: "", Fields: []Field{
|
|
{Key: "id", Type: "string", Doc: "ID names the licence for a machine: an SPDX identifier where there is\none (MIT, Apache-2.0, CC-BY-NC-4.0), else a lowercase name of the\nlicence's own (stability-community, tencent-hunyuan-community). What a\nperson's or a site's choice of licences matches on. Name is for people.", Nested: ""},
|
|
{Key: "name", Type: "string", Doc: "Name is the licence's own: MIT, Apache-2.0.", Nested: ""},
|
|
{Key: "url", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "notice", Type: "string", Doc: "Notice is what must accompany what was made: attribution, terms on\nthe output, what the weights were trained on.", Nested: ""},
|
|
{Key: "commercial", Type: "*bool", Doc: "Commercial says whether what is made may be used commercially: nil\nis not said, which is not the same as yes.", Nested: ""},
|
|
{Key: "excluded_regions", Type: "[]string", Doc: "ExcludedRegions are where the licence grants nothing: ISO 3166 codes,\nand EU for the union (Hunyuan's: EU, UK, KR).", Nested: ""},
|
|
{Key: "revenue_cap_usd", Type: "int64", Doc: "RevenueCapUSD is the yearly revenue above which the licence asks for\nanother agreement (Stability's community licence: 1000000). Zero is\nnone.", Nested: ""},
|
|
{Key: "attribution", Type: "string", Doc: "Attribution is what a product using the output must show: \"Powered by\nStability AI\".", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "LlamaCppConfig", Title: "LlamaCppConfig", Doc: "", Fields: []Field{
|
|
{Key: "bin", Type: "string", Doc: "Bin is the llama-server binary. It is the server, not the CLI: the\ngateway talks OpenAI over HTTP and needs the one that serves it.\nEmpty means the agent installs llama.cpp itself, at the version the\ncontrol plane asks for. Setting it opts out of that.", Nested: ""},
|
|
{Key: "parallel", Type: "int", Doc: "Parallel is how many requests a server handles at once, when a profile\ndoes not say. llama.cpp defaults to one, which would make every second\nrequest wait behind the first no matter what the router decided.", Nested: ""},
|
|
{Key: "threads", Type: "int", Doc: "Threads is how many CPU threads a shard with no GPU computes on. Zero\ntakes half this machine's CPUs.\n\nNot all of them, which is llama.cpp's own default: two busy servers\nthat each spin on every CPU fight for them, and on an 8-vCPU node\nthat made 7B generation fall from 10 tokens a second to under 1 each.\nAt half, one server alone is about a third slower and two together\nkeep 5.5 each. A node that only ever runs one CPU replica does better\nwith all of them; say so here.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Mirror", Title: "Mirror", Doc: "", Fields: []Field{
|
|
{Key: "api", Type: "string", Doc: "API is the mirror's management address, scheme included, for example\nhttps://models.internal:8082. Empty means no mirror is configured.\n\nThe management port, not the one nodes fetch bytes from. Those are two\nlisteners on the mirror precisely so this one can be locked down without\nre-certifying every agent's fetcher.", Nested: ""},
|
|
{Key: "tls", Type: "tlsconf.Config", Doc: "TLS authenticates to the mirror's management API, with the same CA the\nagents use. Empty leaves the connection in plain HTTP, which is only\nacceptable on a network where nothing untrusted can reach that port —\nthe mirror says as much in its own log when it starts that way.", Nested: "tlsconf.Config"},
|
|
{Key: "registry", Type: "string", Doc: "Registry is the model registry the mirror page browses, empty for the\npublic Hugging Face. Set it to an internal mirror of the registry where\nthe control plane cannot reach the internet — or leave the whole mirror\nblock out, which turns the explorer off rather than giving it a search\nbox that can only time out.", Nested: ""},
|
|
{Key: "timeout", Type: "time.Duration", Doc: "Timeout bounds a management call. It does not bound a download: the\nmirror answers a fetch as soon as it has started one, and the bytes keep\narriving long after this deadline has passed.", Nested: ""},
|
|
{Key: "update_check", Type: "time.Duration", Doc: "UpdateCheck is how often the registry is asked whether the files of\nthe models served from it have changed there: a repository can publish\nnew weights under the same name. Only told, never applied — moving a\nmodel to them is `llmgw ctl model update`. 0 never asks.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Model", Title: "Model", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "aliases", Type: "[]string", Doc: "", Nested: ""},
|
|
{Key: "description", Type: "string", Doc: "Description is what this model is for, in a line.\n\nWritten by whoever added it, because nobody else can say it. A registry\noffers a pipeline tag and a wall of README, and neither answers the\nquestion a person picking from a list is asking: which of these nine\nnames should I send this request to.\n\nIt reaches clients through /v1/models, which is the only view most of\nthem ever see. A gateway serving nine models by name alone makes that\nchoice by guesswork.", Nested: ""},
|
|
{Key: "source", Type: "string", Doc: "Source is the weights path or a HuggingFace id.", Nested: ""},
|
|
{Key: "profiles", Type: "[]Profile", Doc: "", Nested: ""},
|
|
{Key: "scaling", Type: "Scaling", Doc: "", Nested: "catalog.Scaling"},
|
|
{Key: "class", Type: "string", Doc: "Class is how much this model's capacity is worth against another's,\nwhen there is not enough for both: critical, standard or opportunistic.\nEmpty is standard, or opportunistic for a model loaded on demand. See\nEffectiveClass.", Nested: ""},
|
|
{Key: "owner", Type: "string", Doc: "Owner is the part of the tenancy tree whose capacity quota this model\ncounts against, written org:ID, team:ID or project:ID. Empty is shared:\nheld by the cluster, counted against nobody.", Nested: ""},
|
|
{Key: "external", Type: "*External", Doc: "External is a model served by someone else — an OpenAI-compatible API\noutside the cluster — rather than by replicas placed here. Such a model\nhas no profiles and nothing is placed for it; the router sends its\nrequests to the provider. Most useful named in another model's\nfallbacks: used only when the cluster has no capacity left. Its\nprompts leave the cluster, so which keys may name it is worth an\nallowlist.", Nested: "catalog.External"},
|
|
{Key: "comfyui", Type: "*ComfyUI", Doc: "ComfyUI is how a model whose profiles run engine comfyui makes its\npictures: the workflow, and the file it loads.", Nested: "catalog.ComfyUI"},
|
|
{Key: "license", Type: "*License", Doc: "License is the terms of what the model makes, for a model whose\noutput somebody ships: shown with every result and written into a\nmesh.", Nested: "catalog.License"},
|
|
{Key: "adapters", Type: "[]Adapter", Doc: "Adapters are LoRA adapters served on this model's replicas, each asked\nfor by its own name as though it were a model. Every replica loads\nevery adapter, applying one only to a request that names it — unless\nMaxLoadedAdapters is set.", Nested: ""},
|
|
{Key: "max_loaded_adapters", Type: "int", Doc: "MaxLoadedAdapters, above zero, has each replica hold at most this many\nadapters at once, loaded when a request first asks for one and unloaded\nleast recently used first to make room. The router sends a request for\nan adapter to a replica that already holds it while that one has room\nto take it. vLLM profiles only: llama-server loads adapters at start\nand cannot load one afterwards. Zero loads every adapter on every\nreplica at start.", Nested: ""},
|
|
{Key: "fallbacks", Type: "[]string", Doc: "", Nested: ""},
|
|
{Key: "canary", Type: "*Canary", Doc: "Canary sends a share of this model's requests to another catalog\nmodel — the same model at new weights — to compare the two before\nmoving to it (`llmgw ctl model canary`).", Nested: "catalog.Canary"},
|
|
{Key: "eval", Type: "*EvalGate", Doc: "Eval is a quality suite new weights must pass before they replace\nthis model's: `llmgw ctl model update` and `model promote` run it\nagainst them first, and stop below its score.", Nested: "catalog.EvalGate"},
|
|
{Key: "routing_profile", Type: "string", Doc: "RoutingProfile selects how the router scores endpoints for this model.", Nested: ""},
|
|
{Key: "slo", Type: "SLO", Doc: "", Nested: "catalog.SLO"},
|
|
{Key: "context_length", Type: "int", Doc: "ContextLength is how many tokens one request may use, prompt and\ncompletion together, and MaxOutput bounds the completion alone. Zero\nmeans unknown, which is what every model was until now.\n\nThe gateway knew neither, so an over-long prompt was not refused: it was\nadmitted, given a replica, and failed at the engine, and a client sizing\nits prompts from /v1/models had nothing to read. Worse, the number is not\nthe one an operator writes in the engine's arguments — llama.cpp divides\n--ctx-size across its parallel slots, so a pool of 8192 across four slots\nis 2048 a request, and nothing said so.", Nested: ""},
|
|
{Key: "max_output", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "capabilities", Type: "Capabilities", Doc: "Capabilities is what this model can do beyond plain completion.", Nested: "catalog.Capabilities"},
|
|
{Key: "query_prefix", Type: "string", Doc: "QueryPrefix is put before a question an embedding model embeds for a\nknowledge-base search; documents are embedded without it. Models\ntrained with an instruction on the query side want it, Qwen3-Embedding\n\"Instruct: Given a web search query, retrieve relevant passages that\nanswer the query\\nQuery: \" for one.", Nested: ""},
|
|
{Key: "role", Type: "string", Doc: "Role \"guard\" is a model that judges the gateway's traffic for the\nguardrails rather than serving clients: hidden from /v1/models and the\nchat page, refused to any key that names it, critical by default, and\nalways running. Guard says how its answers are read.", Nested: ""},
|
|
{Key: "guard", Type: "*GuardModel", Doc: "", Nested: "catalog.GuardModel"},
|
|
{Key: "origin", Type: "*Origin", Doc: "Origin is the registry repository the weights came from, at which\ncommit, and each file's checksum there. Written when a model is served\nfrom the registry; for one that was not, worked out from the mirror's\nrecord of where it fetched the file. It is what lets the control plane\nsay that the repository has published different weights since, and\nmove the model to them.", Nested: "catalog.Origin"},
|
|
}},
|
|
{Package: "config", Name: "Monitoring", Title: "Monitoring", Doc: "", Fields: []Field{
|
|
{Key: "prometheus", Type: "string", Doc: "Prometheus is the server's address, for example\nhttp://monitoring.internal:9090. Empty leaves the charts off the\ndashboard rather than showing an error where one should be.", Nested: ""},
|
|
{Key: "window", Type: "time.Duration", Doc: "Window is how far back the charts look. Zero takes an hour.", Nested: ""},
|
|
{Key: "timeout", Type: "time.Duration", Doc: "Timeout bounds one query. It runs while somebody waits for a page, so a\nmonitoring system that is slow today is skipped rather than waited for.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "OIDC", Title: "OIDC", Doc: "", Fields: []Field{
|
|
{Key: "issuer", Type: "string", Doc: "Issuer is the provider's address; its /.well-known/openid-configuration\nsays the rest.", Nested: ""},
|
|
{Key: "client_id", Type: "string", Doc: "ClientID and the file holding its secret, as registered with the\nprovider.", Nested: ""},
|
|
{Key: "client_secret_file", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "redirect_url", Type: "string", Doc: "RedirectURL is where the provider sends people back:\nhttps://<gateway>/login/oidc/callback. Empty works it out from the\nrequest, which is right unless a proxy changes the address.", Nested: ""},
|
|
{Key: "label", Type: "string", Doc: "Label is what the sign-in button says: \"Sign in with <label>\".", Nested: ""},
|
|
{Key: "scopes", Type: "[]string", Doc: "Scopes asked for besides openid; empty is email and profile, and\ngroups when Roles or Memberships use them.", Nested: ""},
|
|
{Key: "groups_claim", Type: "string", Doc: "GroupsClaim names the ID token claim holding groups; empty is\n\"groups\".", Nested: ""},
|
|
{Key: "create_accounts", Type: "bool", Doc: "CreateAccounts makes an account, active, for a person the provider\nvouches for and the gateway does not know. Off, only existing accounts\nsign in this way.", Nested: ""},
|
|
{Key: "roles", Type: "map[string][]string", Doc: "Roles maps a staff role (viewer, operator, billing, admin) to the\ngroups that give it. Set at every sign-in: someone taken out of the\ngroup loses the role the next time.", Nested: ""},
|
|
{Key: "memberships", Type: "map[string]string", Doc: "Memberships maps a group to a place in the tree — org:NAME,\nteam:ORG/TEAM, project:ORG/TEAM/PROJECT — joined at sign-in.", Nested: ""},
|
|
{Key: "no_passwords", Type: "bool", Doc: "PasswordSignIn off refuses passwords on the sign-in form: people come\nin through the provider only. Administrators still can, so a broken\nprovider does not lock everybody out.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Offline", Title: "Offline", Doc: "", Fields: []Field{
|
|
{Key: "mirror", Type: "string", Doc: "Mirror is the mirror's file address, as nodes reach it:\nhttp://10.0.0.5:8081 — the port model files are served on, not the\nmanagement API. Set, the cluster is offline: nodes install engines\nfrom it and never from their publishers, releases come from it unless\nreleases names another place, and no registry is asked whether a\nmodel has newer weights. Python engines (vLLM, SGLang) come from PyPI,\nwhich no bundle carries; on an offline node they are installed by hand.", Nested: ""},
|
|
{Key: "checksums", Type: "string", Doc: "Checksums is the file of engine checksums this control plane trusts,\nkept by llmgw bundle import and llmgw bundle trust; every engine file\na node fetches from the mirror is checked against it. Read again\nwhenever it changes. Empty is /etc/llmgw/offline.sha256.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "OnDemandSettings", Title: "OnDemandSettings", Doc: "", Fields: []Field{
|
|
{Key: "evict_after_idle", Type: "time.Duration", Doc: "EvictAfterIdle is how long an on-demand replica must go unasked-for\nbefore a model that wants its card may have it. Zero keeps 2m.", Nested: ""},
|
|
{Key: "min_lifetime", Type: "time.Duration", Doc: "MinLifetime is how long a replica that has just come up is safe from\nthat, so two models do not take turns paying for loads. Zero keeps\n5m. On one card shared by a picture model and a 3D model, 1m each\nbrings the wait between a picture and its mesh to about a minute.\nBoth are in force on the leader within seconds.", Nested: ""},
|
|
}},
|
|
{Package: "discovery", Name: "Options", Title: "Options", Doc: "", Fields: []Field{
|
|
{Key: "nvidia_smi", Type: "string", Doc: "NvidiaSMI is the path to nvidia-smi; empty means look it up on PATH.\nWorth configuring on nodes where the driver lives outside the default\npaths, which containerized driver installs often do.", Nested: ""},
|
|
{Key: "ib_root", Type: "string", Doc: "IBRoot is the sysfs InfiniBand directory.", Nested: ""},
|
|
{Key: "prefer_nvml", Type: "bool", Doc: "PreferNVML uses the NVML backend when this binary has one. NVLink\ntopology is read from nvidia-smi topo -m on every build; NVML, where it\nis used, is the more exact source of the peers.", Nested: ""},
|
|
{Key: "skip_gpus", Type: "bool", Doc: "SkipGPUs omits GPU discovery. A node without nvidia-smi is already\nfound to have none, so this is for one whose driver is broken.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Origin", Title: "Origin", Doc: "", Fields: []Field{
|
|
{Key: "repo", Type: "string", Doc: "Repo is the repository, owner/name.", Nested: ""},
|
|
{Key: "revision", Type: "string", Doc: "Revision is the commit the files were fetched at.", Nested: ""},
|
|
{Key: "files", Type: "[]OriginFile", Doc: "Files are the weights fetched from it, with what the registry\npublished as each one's sha256. A new commit that leaves them alone —\na README edited — is not an update.", Nested: ""},
|
|
{Key: "previous", Type: "*PreviousOrigin", Doc: "Previous is what the last update replaced, kept so it can be gone back\nto while its files are still on the mirror.", Nested: "catalog.PreviousOrigin"},
|
|
}},
|
|
{Package: "catalog", Name: "OriginFile", Title: "OriginFile", Doc: "", Fields: []Field{
|
|
{Key: "path", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "sha256", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "bytes", Type: "int64", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "guard", Name: "Pattern", Title: "Pattern", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "regex", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "action", Type: "string", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Personal", Title: "Personal", Doc: "", Fields: []Field{
|
|
{Key: "disabled", Type: "bool", Doc: "Disabled refuses personal keys and personal chat; people then use the\ngateway only through projects they are members of.", Nested: ""},
|
|
{Key: "rpm", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "tpm", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "budget", Type: "float64", Doc: "Budget is in whole units per Period (\"day\" or \"month\").", Nested: ""},
|
|
{Key: "period", Type: "string", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "guard", Name: "Policy", Title: "Policy", Doc: "", Fields: []Field{
|
|
{Key: "mode", Type: "string", Doc: "Mode: enforce acts, shadow only records what it would have done, off\ndoes nothing. Empty is enforce.", Nested: ""},
|
|
{Key: "fail", Type: "string", Doc: "Fail: what happens when a policy cannot judge — a judge model down or\nover its latency budget. open serves, closed refuses. Empty is open.", Nested: ""},
|
|
{Key: "no_store", Type: "bool", Doc: "NoStore keeps nothing of the request: no stored conversation, no\nmemory notes, no prefix routing, no fallback outside the cluster.", Nested: ""},
|
|
{Key: "input", Type: "Rules", Doc: "", Nested: "guard.Rules"},
|
|
{Key: "output", Type: "Rules", Doc: "", Nested: "guard.Rules"},
|
|
{Key: "tools", Type: "Tools", Doc: "", Nested: "guard.Tools"},
|
|
{Key: "max_output_tokens", Type: "int", Doc: "MaxOutputTokens is put in a request that asks for no limit; one\nasking for more is lowered to it.", Nested: ""},
|
|
{Key: "system_prompt", Type: "string", Doc: "SystemPrompt is put before every request's messages.", Nested: ""},
|
|
{Key: "drop_client_system", Type: "bool", Doc: "DropClientSystem removes the caller's own system and developer\nmessages, so only the pinned prompt instructs the model.", Nested: ""},
|
|
{Key: "require_capabilities", Type: "bool", Doc: "RequireCapabilities refuses a request offering tools, or sending\nimages, to a model whose catalog entry does not say it takes them.", Nested: ""},
|
|
{Key: "validate_structured", Type: "bool", Doc: "ValidateStructured checks an answer against the JSON schema the\nrequest asked for in response_format: one that does not fit is\nstopped — or, streamed a sentence at a time, flagged, having been\nsent already.", Nested: ""},
|
|
{Key: "judge", Type: "*JudgePolicy", Doc: "Judge has a judge model on the cluster read the text too.", Nested: "guard.JudgePolicy"},
|
|
{Key: "stream", Type: "string", Doc: "Stream: sentence scans a streamed answer a sentence at a time,\nreleasing each once it passes; buffer_all holds the whole answer\nuntil it is judged. Empty is sentence.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Prefix", Title: "Prefix", Doc: "", Fields: []Field{
|
|
{Key: "enabled", Type: "bool", Doc: "", Nested: ""},
|
|
{Key: "max_nodes", Type: "int", Doc: "MaxNodes bounds the tree, MinMatch is the shortest match worth acting on\nin bytes, and TTL is how long an entry is believed — an engine evicts on\nits own schedule, so an old record may describe a cache that is gone.", Nested: ""},
|
|
{Key: "min_match", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "ttl", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "match_ratio", Type: "float64", Doc: "MatchRatio is the share of the prompt that must match to be worth acting\non. BalanceAbs and BalanceRel are the imbalance guard, and both must trip\nbefore affinity is abandoned.", Nested: ""},
|
|
{Key: "balance_abs", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "balance_rel", Type: "float64", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "PresetSettings", Title: "PresetSettings", Doc: "", Fields: []Field{
|
|
{Key: "auto", Type: "bool", Doc: "Auto has the leading control plane add a picture model (flux-schnell)\nand a 3D model (triposr) by itself, on demand, once a node has a card\nthat fits one and the catalog has no model that makes the same kind of\nthing — so a new cluster's portal has its pictures and 3D pages\nwithout anybody writing a definition. Only where the cluster already\ninstalls ComfyUI (engines.comfyui). A preset somebody removed is not\nadded again by itself. In force on the leader within a minute.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "PreviousOrigin", Title: "PreviousOrigin", Doc: "", Fields: []Field{
|
|
{Key: "source", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "revision", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "files", Type: "[]OriginFile", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Profile", Title: "Profile", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "engine", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "engine_version", Type: "string", Doc: "EngineVersion restricts the profile to nodes whose installed engine's\nversion starts with it: \"0.11\" takes vLLM 0.11.0 and 0.11.1, not\n0.10.2. Empty takes whatever version the node has.", Nested: ""},
|
|
{Key: "gpu_class", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "source", Type: "string", Doc: "Source is the weights this profile loads, when not the model's: an\nembedding model served by vLLM from its repository on a card and by\nllama.cpp from a GGUF on CPUs is one model with two sources. Empty\nloads the model's. Updates and canaries move the model's source; a\nprofile with its own keeps it.", Nested: ""},
|
|
{Key: "tp", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "pp", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "gpus_per_node", Type: "int", Doc: "GPUsPerNode defaults to TP for a single-node profile.", Nested: ""},
|
|
{Key: "nodes", Type: "int", Doc: "Nodes above 1 makes this a multi-node profile, which requires a fabric.", Nested: ""},
|
|
{Key: "args", Type: "[]string", Doc: "", Nested: ""},
|
|
{Key: "env", Type: "map[string]string", Doc: "", Nested: ""},
|
|
{Key: "min_memory_gb", Type: "int", Doc: "MinMemoryGB is the host RAM this profile needs to run.\n\nThe number that matters is the model's working set: the weights, plus the\nKV cache for the whole context pool, plus the engine's own overhead. For\na GPU profile that lives in the cards and this is small; for a CPU\nprofile it is the whole thing, and getting it wrong means an OOM on a\nnode that reported itself healthy.\n\nZero means unconstrained, which is every profile that existed before\nthis. A node that does not report its memory is not excluded either —\nan older agent must not have its nodes quietly emptied by an upgrade of\nthe control plane.", Nested: ""},
|
|
{Key: "cpu_cores", Type: "int", Doc: "CPUCores is the cores a replica of a profile with no GPU is given:\nreserved out of its node's CPUs by placement, as memory is, and what\nits engine computes on (llama.cpp's --threads).\n\nReserved because an engine takes every core it is told about, and two\non one node that each think they have all of them spin against each\nother: on an 8-vCPU node, two 7B replicas fell from 10 tokens a second\nto under one each. Zero reserves nothing and leaves the thread count to\nthe node, which is every profile from before this.", Nested: ""},
|
|
{Key: "min_disk_gb", Type: "int", Doc: "MinDiskGB is the free space a node needs before it can hold these\nweights.\n\nOnly checked when the node does not already have them: a node holding\nthe file needs no room for it, and requiring the space twice would keep\na model off the one node best placed to run it.\n\nDeclared rather than derived, because the size of a model that has never\nbeen fetched anywhere is not knowable from here — and a wrong guess\nfails late, halfway through a download, which is the failure this exists\nto prevent. Zero means unconstrained, and a node that reports no storage\nis not excluded: unknown is not full.", Nested: ""},
|
|
{Key: "node_selector", Type: "map[string]string", Doc: "NodeSelector restricts this profile to nodes carrying every one of these\nlabels, with these values.\n\nA CPU profile otherwise fits everywhere, because there is no card to be\nfree — which is right for a 0.5B model and wrong for a 14B one. Nothing\nin the inventory says how much memory a node has, so the operator says\nwhich nodes are meant for a model instead of the scheduler inferring it.\nAgents have reported labels since the beginning; this is what reads them.", Nested: ""},
|
|
{Key: "power_watts", Type: "float64", Doc: "PowerWatts is what one replica of this profile draws, in watts, split\nevenly over its nodes, counted against the power cap of every rack,\nPDU or other domain its nodes are in (facility.budgets in server.yaml).\nZero derives it: each card at the power limit it reports, each\nreserved core at facility.cpu_core_watts. Only matters where a domain\nhas a cap.", Nested: ""},
|
|
{Key: "keep_within", Type: "string", Doc: "KeepWithin keeps a multi-node replica's nodes inside one rack, row,\nroom or site, read from their llmgw.io/rack (and so on) labels: for a\nfabric whose leaves do not reach across racks, or cables that do not.\nNodes without the label are passed over for such a profile. Empty is\nno constraint.", Nested: ""},
|
|
{Key: "weight", Type: "float64", Doc: "Weight is the router's capacity weight. It should come from ctl bench\nrather than a guess (plan §6.2.1).", Nested: ""},
|
|
{Key: "priority", Type: "int", Doc: "Priority orders profiles during placement; highest that fits wins.", Nested: ""},
|
|
{Key: "startup_timeout", Type: "time.Duration", Doc: "StartupTimeout bounds model loading. A 70B model across two nodes takes\ntens of minutes, so this is per profile rather than global.", Nested: ""},
|
|
{Key: "kv_events", Type: "bool", Doc: "KVEvents has vLLM publish what its prefix cache holds, and the router\nroute this model's requests by it: to the replica holding the most of\na prompt's leading blocks (plan §7.5 tier 2). vLLM only.\n\nOff by default, because it costs something on every request: the\nrouter asks a replica to tokenize the prompt before placing it, a\nround trip and the chat template run twice. It pays where prompts\nshare long prefixes that the approximate tree guesses wrong — many\nreplicas, conversations that move between them, evictions. The engine\nis started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,\nwhich the router's hashing assumes; arguments overriding either leave\nthe index empty, and the agent says so in its log.", Nested: ""},
|
|
{Key: "gpu", Type: "*GPUNeed", Doc: "GPU says what card the profile needs, in place of naming its class:\nit stands for every class in the cluster that meets it. A catalog\nentry written once then places sensibly on any site's cards.", Nested: "catalog.GPUNeed"},
|
|
{Key: "comfyui", Type: "*ProfileComfyUI", Doc: "ComfyUI is this profile's own workflow, checkpoint and nodes, over the\nmodel's: an fp8 workflow where the card has fp8, fp16 where it has not.", Nested: "catalog.ProfileComfyUI"},
|
|
}},
|
|
{Package: "catalog", Name: "ProfileComfyUI", Title: "ProfileComfyUI", Doc: "", Fields: []Field{
|
|
{Key: "workflow", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "checkpoint", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "nodes", Type: "[]CustomNode", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Router", Title: "Router", Doc: "", Fields: []Field{
|
|
{Key: "activation_wait", Type: "time.Duration", Doc: "ActivationWait is how long a request waits for an on-demand model to\nload before the client is told to come back. Zero takes the default.\n\nThe one real decision in on-demand loading. Refusing outright makes the\nfeature useless, since the first request for every model would fail;\nwaiting without a bound turns a request into a job, and a client that\nbudgeted thirty seconds ends up holding a connection for ten minutes.", Nested: ""},
|
|
{Key: "scrape_interval", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "scrape_timeout", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "stale_after", Type: "time.Duration", Doc: "StaleAfter is how long an endpoint may go unscraped before being scored\ndown (plan §7.6).", Nested: ""},
|
|
{Key: "retries", Type: "int", Doc: "Retries is how many *additional* endpoints may be tried after a failure,\nand only while nothing has been written to the client (plan §7.7).", Nested: ""},
|
|
{Key: "cooldown", Type: "time.Duration", Doc: "Cooldown is how long a failing endpoint leaves the candidate pool.", Nested: ""},
|
|
{Key: "failure_threshold", Type: "int", Doc: "FailureThreshold is the consecutive failure count that opens a breaker.\n1 turns the breaker into a pure cooldown.", Nested: ""},
|
|
{Key: "default_profile", Type: "string", Doc: "DefaultProfile names the routing profile used when a model picks none.", Nested: ""},
|
|
{Key: "prefix", Type: "Prefix", Doc: "Prefix turns on the approximate prefix tree (plan §7.5 tier 1). Off, the\nrouter uses session affinity alone, which is tier 0 and already most of\nthe win for multi-turn traffic.", Nested: "config.Prefix"},
|
|
{Key: "kv_index", Type: "KVIndex", Doc: "KVIndex tunes the precise KV index (plan §7.5 tier 2). Which models\nuse it is the catalog's to say, by kv_events on a vLLM profile; this\nis only how.", Nested: "config.KVIndex"},
|
|
{Key: "session_header", Type: "string", Doc: "SessionHeader carries the session id for affinity (plan §7.5 tier 0).", Nested: ""},
|
|
{Key: "inject_usage", Type: "bool", Doc: "InjectUsage asks engines for usage even when the client did not, so token\naccounting works regardless of client behaviour.", Nested: ""},
|
|
{Key: "tenant_label", Type: "string", Doc: "TenantLabel is the level of the tree the per-tenant request series are\nlabelled by — llmgw_tenant_requests_total and its siblings: \"org\",\n\"team\" or \"project\", or \"none\" for no such series. A label per project\non a cluster of thousands is more series than Prometheus wants, and a\nteam is what an alert is usually about; \"team\" is the default.", Nested: ""},
|
|
{Key: "tag_labels", Type: "[]string", Doc: "TagLabels are the request tags (metadata, x-llmgw-tags) counted in\nllmgw_tagged_requests_total and llmgw_tagged_tokens_total, by value.\nOnly these: a tag is the client's to set, and every value of one is\na series. The first 100 values of each are counted apart, the rest as\n\"other\". Empty counts none; the usage records keep every tag anyway.", Nested: ""},
|
|
{Key: "max_body_bytes", Type: "int64", Doc: "MaxBodyBytes bounds the request body the router will read.", Nested: ""},
|
|
{Key: "batch", Type: "Batch", Doc: "Batch is the Files and Batch APIs (/v1/files, /v1/batches): a file of\nrequests served at the lowest priority, its answers written to\nanother. Needs the store.", Nested: "config.Batch"},
|
|
{Key: "cache", Type: "Cache", Doc: "Cache bounds the answers kept for the projects that reuse them\n(llmgw keys cache). Which projects do, and for how long, is theirs\nto set; this is only how much memory it may take.", Nested: "config.Cache"},
|
|
{Key: "trusted_proxies", Type: "[]string", Doc: "TrustedProxies are the reverse proxies in front of the client port,\nas addresses or CIDR prefixes. Their X-Forwarded-For, -Proto and -Host\nare believed; anyone else's are not, since they decide the address\nwritten into the setup scripts and whose attempts are counted where.", Nested: ""},
|
|
{Key: "timeouts", Type: "Timeouts", Doc: "", Nested: "config.Timeouts"},
|
|
{Key: "scoring", Type: "Scoring", Doc: "", Nested: "config.Scoring"},
|
|
{Key: "admission", Type: "Admission", Doc: "", Nested: "config.Admission"},
|
|
}},
|
|
{Package: "guard", Name: "Rules", Title: "Rules", Doc: "", Fields: []Field{
|
|
{Key: "pii", Type: "map[string]string", Doc: "PII maps a kind (email, phone, iban, card, tckn, ip, secret) to what is\ndone when it is found: flag, redact or block.", Nested: ""},
|
|
{Key: "injection", Type: "string", Doc: "Injection is what is done on signs of prompt injection: flag or block.", Nested: ""},
|
|
{Key: "deny", Type: "[]Pattern", Doc: "Deny are patterns, matched against the text as a model reads it.", Nested: ""},
|
|
{Key: "max_chars", Type: "int", Doc: "MaxChars refuses text longer than this; zero is no limit.", Nested: ""},
|
|
{Key: "restore", Type: "bool", Doc: "Restore, on input, puts what redact rewrote as placeholders back into\nthe answer: the model reads \"[EMAIL_1]\", the client reads its own\naddress. Only the values that request sent, only into its answer.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "SCIM", Title: "SCIM", Doc: "", Fields: []Field{
|
|
{Key: "token_file", Type: "string", Doc: "TokenFile holds the bearer token the provider sends. Empty turns\nprovisioning off.", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "SGLangConfig", Title: "SGLangConfig", Doc: "", Fields: []Field{
|
|
{Key: "python", Type: "string", Doc: "Python is the interpreter of the SGLang virtualenv. SGLang is launched as\na module rather than a console script, which is the documented entry\npoint and avoids depending on the script being on PATH.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "SLO", Title: "SLO", Doc: "", Fields: []Field{
|
|
{Key: "ttft_p99", Type: "time.Duration", Doc: "TTFTp99 is the time to the first token 99 requests in 100 are within.", Nested: ""},
|
|
{Key: "tpot_p99", Type: "time.Duration", Doc: "TPOTp99 is the time per output token, after the first, 99 streamed\nrequests in 100 are within.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "ScaleWindow", Title: "ScaleWindow", Doc: "", Fields: []Field{
|
|
{Key: "days", Type: "[]string", Doc: "Days are mon … sun, or weekdays, weekends; empty is every day.", Nested: ""},
|
|
{Key: "from", Type: "string", Doc: "From and To are HH:MM; a window whose To is before its From runs\npast midnight.", Nested: ""},
|
|
{Key: "to", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "min", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "max", Type: "int", Doc: "Max, when set, is the window's ceiling for load scaling; zero keeps\nthe model's own.", Nested: ""},
|
|
}},
|
|
{Package: "catalog", Name: "Scaling", Title: "Scaling", Doc: "", Fields: []Field{
|
|
{Key: "min", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "max", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "idle_timeout", Type: "time.Duration", Doc: "IdleTimeout is how long an on-demand model may go unasked-for before its\nreplicas are stopped. Zero takes the cluster default.\n\nMinutes rather than seconds, always. Stopping a replica throws away the\nKV cache prefix routing spent its effort filling, and a model that flaps\nin and out loses that entirely — so the saving has to be worth more than\nthe cache, which it only is over a real idle period.", Nested: ""},
|
|
{Key: "schedule", Type: "[]ScaleWindow", Doc: "Schedule changes min and max by the time of day, in the control\nplane's control.timezone: more replicas kept through working hours,\nfewer at night, an on-demand model kept loaded while people are at\nwork. The first window that holds now wins; outside them all, Min and\nMax apply.", Nested: ""},
|
|
{Key: "prewarm", Type: "time.Duration", Doc: "Prewarm starts a window this long before it opens, so its replicas\nhave loaded by then: loading a model is minutes, and the people who\narrive at nine are the reason for the window.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Scoring", Title: "Scoring", Doc: "", Fields: []Field{
|
|
{Key: "target_waiting", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "target_inflight", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "kv_ceiling", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "session_load_factor", Type: "float64", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Secrets", Title: "Secrets", Doc: "", Fields: []Field{
|
|
{Key: "vault", Type: "secrets.Vault", Doc: "Vault is how vault: references and vault-transit: are reached.", Nested: "secrets.Vault"},
|
|
{Key: "master_key", Type: "string", Doc: "MasterKey seals the secrets the store keeps for itself — what every\nengine's key is derived from — so a copy of the store, or of a backup,\ndoes not hold them: vault-transit:KEY seals with Vault's transit engine,\nwhich never lets the key out; otherwise a reference to 32 random bytes\n(openssl rand -hex 32), which then must be kept as safely as the store.\nValues from before it was set are sealed the first time they are read.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Server", Title: "Server", Doc: "", Fields: []Field{
|
|
{Key: "listen", Type: "string", Doc: "Listen is the client-facing OpenAI API address.", Nested: ""},
|
|
{Key: "admin_listen", Type: "string", Doc: "AdminListen serves the dashboard, the management API, health and pprof.\n\nLoopback by default, and it should stay there: pprof can dump the process\nheap, and API keys are in that heap in plaintext. Every comment beside\nthis setting has always said not to expose it, while the default bound\nevery interface — a default that contradicts its own documentation is a\nfootgun, so the default now matches the advice.", Nested: ""},
|
|
{Key: "metrics_listen", Type: "string", Doc: "MetricsListen serves Prometheus metrics and nothing else.\n\nSeparate from AdminListen because the reason that one is loopback is\npprof, not metrics — and metrics are the one thing on it that something\nelse genuinely needs to reach. Sharing a listener meant a fleet could\neither be monitored or keep its heap private, which is not a choice\nanybody should have to make.\n\nEmpty means off, and metrics remain available on AdminListen either way,\nso nothing that scrapes it today has to change.", Nested: ""},
|
|
{Key: "agent_listen", Type: "string", Doc: "AgentListen is where node agents connect. Separate from the client port\nso the fleet-facing surface can be firewalled independently.", Nested: ""},
|
|
{Key: "auth", Type: "Auth", Doc: "Auth turns on API keys on the client-facing port.", Nested: "config.Auth"},
|
|
{Key: "threads", Type: "Threads", Doc: "Threads keeps conversations and agent memory without API keys, for a\ngateway on one trusted machine. With auth on they live in the auth store,\nper project, and this block is ignored.", Nested: "config.Threads"},
|
|
{Key: "landing", Type: "Landing", Doc: "Landing configures the page served at \"/\".", Nested: "config.Landing"},
|
|
{Key: "agent_tls", Type: "tlsconf.Config", Doc: "AgentTLS authenticates the fleet-facing port. It deliberately does not\ncover the client-facing one: that surface is fronted by a publicly\ntrusted certificate, while this one is mutual and private.", Nested: "tlsconf.Config"},
|
|
{Key: "max_connections", Type: "int", Doc: "MaxConnections bounds the client port's open connections; past it new\nones wait in the kernel's backlog. 0 is no bound.", Nested: ""},
|
|
{Key: "admin_token_file", Type: "string", Doc: "AdminTokenFile holds the token the admin API's /api and /debug paths\ndemand. Empty is admin-token in the systemd unit's state directory\n(/var/lib/llmgw), or secrets/admin-token beside this file for a server\nrun by hand. Made, 0600, when the server starts and it does not exist;\n`sudo llmgw ctl` and the console read it from there.", Nested: ""},
|
|
{Key: "admin_remote", Type: "bool", Doc: "AdminRemote allows admin_listen on an address other than loopback. The\ntoken still guards the API, but it then crosses the network in clear.", Nested: ""},
|
|
{Key: "debug_pprof", Type: "bool", Doc: "DebugPprof serves Go's profiler on the admin listener, behind the token.\nOff by default: a heap profile holds the API keys requests presented.", Nested: ""},
|
|
{Key: "agent_insecure", Type: "bool", Doc: "AgentInsecure lets a control plane run its agent port without\nagent_tls: plaintext, and any machine that reaches it may register as\nany node — be given replicas, and so be sent users' prompts — or, in\nthe other direction, a machine in the path may tell every agent what\nto run. Refused unless said here, and then the port listens on\nloopback unless agent_listen names an address. For one machine, a\ntest, or a network nothing untrusted reaches.", Nested: ""},
|
|
{Key: "monitoring", Type: "Monitoring", Doc: "Monitoring is where the dashboard reads its history from.", Nested: "config.Monitoring"},
|
|
{Key: "tracing", Type: "obs.Tracing", Doc: "Tracing sends traces to an OpenTelemetry collector: each inference\nrequest, and on a control plane each placement, replica start and\nmodel job. Off unless an endpoint is set here or in\nOTEL_EXPORTER_OTLP_ENDPOINT.", Nested: "obs.Tracing"},
|
|
{Key: "mirror", Type: "Mirror", Doc: "Mirror points the control plane at a model mirror it can administer.\nEmpty means there is none, and /admin/mirror stays unmounted.", Nested: "config.Mirror"},
|
|
{Key: "offline", Type: "Offline", Doc: "Offline is a cluster that reaches nothing outside itself: its nodes\ninstall engines, and upgrades fetch releases, from the mirror, out of\na bundle carried in (llmgw bundle create, llmgw bundle import).", Nested: "config.Offline"},
|
|
{Key: "engines", Type: "map[string]string", Doc: "Engines are the engines every node installs, and at which version. The\nagents install them themselves — the right build for their hardware,\nchecked against its publisher's checksum and seen to run — so a node is\nnever trusted to already have what somebody once put there.\n\nUnset keeps DefaultEngines. A version replaces the default; an empty\none leaves that engine to be installed by hand.", Nested: ""},
|
|
{Key: "engine_overrides", Type: "[]EngineOverride", Doc: "EngineOverrides give some nodes different engines from the rest: a\nvLLM built for CUDA 12 on the Volta cards CUDA 13 dropped, or none at\nall on nodes meant only for llama.cpp. Each picks nodes and lays its\nengines over the cluster's, in order, later ones last; an empty version\nremoves that engine from the nodes it picks.", Nested: ""},
|
|
{Key: "backup_dir", Type: "string", Doc: "BackupDir is where `llmgw server backup` writes, and where the newest\narchive's age is read for llmgw_backup_last_success_timestamp_seconds.\nEmpty means /var/backups/llmgw-server.", Nested: ""},
|
|
{Key: "guard", Type: "guard.Config", Doc: "Guard is the library of guardrail policies: what requests and answers\nare checked for, and what is done. Applied to every request through\nguard.default, and to parts of the tenancy tree by attaching them\n(llmgw ctl guard attach). Empty checks nothing.", Nested: "guard.Config"},
|
|
{Key: "secrets", Type: "Secrets", Doc: "Secrets is where secrets kept outside files are read from, and the key\nthe store's own secrets are sealed under.", Nested: "config.Secrets"},
|
|
{Key: "notify", Type: "notify.Config", Doc: "Notify sends what needs a person — a model parked, a node lost, a\nbudget spent, new weights published — to webhooks, signed, without a\nPrometheus and Alertmanager in between. Empty sends nothing.", Nested: "notify.Config"},
|
|
{Key: "mail", Type: "mail.Config", Doc: "Mail is where the gateway's few emails go out from: a link to set a\nforgotten password. Unset, none is sent and people ask an operator.", Nested: "mail.Config"},
|
|
{Key: "serving", Type: "ServingSettings", Doc: "Serving stops and starts the whole cluster's service (llmgw stop, llmgw\nstart).", Nested: "config.ServingSettings"},
|
|
{Key: "firewall", Type: "FirewallSettings", Doc: "Firewall turns llmgw's firewall on every machine on or off.", Nested: "config.FirewallSettings"},
|
|
{Key: "presets", Type: "PresetSettings", Doc: "Presets is how the models that come with llmgw reach the catalog.", Nested: "config.PresetSettings"},
|
|
{Key: "on_demand", Type: "OnDemandSettings", Doc: "OnDemand is how models that load when asked for take turns on the\ncards.", Nested: "config.OnDemandSettings"},
|
|
{Key: "agents", Type: "AgentSettings", Doc: "Agents are the nodes' agent settings: agent.yaml's keys but those an\nagent needs to start, for every node (default) and for one\n(nodes.<id>). Sent to the agents as they change; an agent whose\nsettings changed starts again on them, its engines serving\nthroughout.", Nested: "config.AgentSettings"},
|
|
{Key: "firewall_networks", Type: "[]string", Doc: "FirewallNetworks are networks, besides those each machine reaches the\ncluster on, that llmgw's internal ports answer: a monitoring host on\nanother subnet, nodes behind a router. Written into every machine's\nfirewall by `server init`, `node add` and `server add`.", Nested: ""},
|
|
{Key: "backup_recipients", Type: "[]string", Doc: "BackupRecipients encrypt every archive `llmgw server backup` writes, to\nwhoever holds one of their keys: age recipients (age1…) or SSH public\nkeys (ssh-ed25519 …, ssh-rsa …), as `age` takes them. An archive holds\nthe certificate authority's key and every secret in the store, and is\ncopied off the machine; encrypted, the copy is safe wherever it lands.\nEmpty writes archives unencrypted, readable by root only.", Nested: ""},
|
|
{Key: "backup_copy_to", Type: "[]string", Doc: "BackupCopyTo is where each archive is copied once written, so it\nsurvives this machine: a directory (a mounted share, another disk) or\nssh://user@host[:port]/path, reached with BackupCopyKey and checked\nagainst /etc/llmgw/backup_known_hosts, which learns a host the first\ntime. The same number of archives is kept there as here. Refused\nwithout backup_recipients: an archive leaves the machine encrypted.", Nested: ""},
|
|
{Key: "backup_copy_key", Type: "string", Doc: "BackupCopyKey is the SSH private key for ssh:// copies.", Nested: ""},
|
|
{Key: "releases", Type: "string", Doc: "Releases is where `llmgw ctl upgrade` fetches release binaries from,\nfor this machine and — through their agents — every node. Empty means\nthe project's own releases.", Nested: ""},
|
|
{Key: "router", Type: "Router", Doc: "", Nested: "config.Router"},
|
|
{Key: "control", Type: "Control", Doc: "", Nested: "config.Control"},
|
|
{Key: "models", Type: "[]catalog.Model", Doc: "", Nested: ""},
|
|
{Key: "facility", Type: "Facility", Doc: "Facility is the power caps of racks, PDUs and other domains, and how\na replica's draw is estimated against them. Where a node is comes from\nits llmgw.io/site, room, row, rack and pdu labels; holding a domain for\nmaintenance is `llmgw ctl facility hold`, not configuration.", Nested: "config.Facility"},
|
|
{Key: "federation", Type: "Federation", Doc: "Federation is how this cluster takes part in federation with other\nllmgw clusters. The peers themselves are added at runtime, with\n`llmgw ctl federation add`, and kept in the store.", Nested: "config.Federation"},
|
|
{Key: "endpoints", Type: "[]Endpoint", Doc: "Endpoints is the phase-1 static replica list. It is only consulted when\nno model declares a profile, which is what makes a router-only smoke test\nagainst sim-engines possible without a control plane.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "ServingSettings", Title: "ServingSettings", Doc: "", Fields: []Field{
|
|
{Key: "stopped", Type: "bool", Doc: "Stopped takes every model down and answers every request for one with\n503 (system_stopped), leaving the control planes, agents and mirror\nrunning and every model's definition as it is: llmgw stop sets it,\nllmgw start clears it, and the models come back as their catalog\nentries ask. In force on every control plane within seconds.", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "SimConfig", Title: "SimConfig", Doc: "", Fields: []Field{
|
|
{Key: "bin", Type: "string", Doc: "Bin is the llmgw binary itself; the engine is one of its subcommands.\nEmpty means whatever started this agent, which is right almost always.", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "TRTLLMConfig", Title: "TRTLLMConfig", Doc: "", Fields: []Field{
|
|
{Key: "python", Type: "string", Doc: "Python is the interpreter of the environment TensorRT-LLM is installed\nin: <venv>/bin/python for a virtual environment, python3 in NVIDIA's\ncontainer. Naming it is what makes this node offer the engine; empty\noffers nothing.\n\nThe interpreter rather than trtllm-serve, because llmgw starts the\nserver through a few lines of its own that make it demand the\ngateway's key, which trtllm-serve has no option for.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Threads", Title: "Threads", Doc: "", Fields: []Field{
|
|
{Key: "store", Type: "string", Doc: "Store is the SQLite file. Empty means nothing is kept.", Nested: ""},
|
|
{Key: "retain_hours", Type: "int", Doc: "RetainHours is how long idle conversations and notes are kept. Required\nwith Store: a store with no expiry keeps everything anyone typed, by\nomission.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Timeouts", Title: "Timeouts", Doc: "", Fields: []Field{
|
|
{Key: "dial", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "response_header", Type: "time.Duration", Doc: "ResponseHeader bounds the wait for headers on a streaming request. It is\na liveness check there, because headers arrive before generation starts.", Nested: ""},
|
|
{Key: "generation", Type: "time.Duration", Doc: "Generation bounds a non-streaming request, where the engine sends no\nheaders until the completion is done. Setting response_header alone used\nto cap both, so 60s read as a liveness guard and behaved as a limit of\nabout a hundred tokens on a slow model.", Nested: ""},
|
|
{Key: "first_token", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "idle", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "total", Type: "time.Duration", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "guard", Name: "Tools", Title: "Tools", Doc: "", Fields: []Field{
|
|
{Key: "allow", Type: "[]string", Doc: "Allow, when set, are the only tool names a request may offer and a\nmodel may call.", Nested: ""},
|
|
{Key: "deny", Type: "[]string", Doc: "Deny are tool names never offered or called.", Nested: ""},
|
|
{Key: "on_denied", Type: "string", Doc: "OnDenied is strip (the tool is removed from the request) or block.\nEmpty is strip.", Nested: ""},
|
|
{Key: "max_calls_per_turn", Type: "int", Doc: "MaxCallsPerTurn stops an answer calling more tools than this.", Nested: ""},
|
|
{Key: "validate_arguments", Type: "bool", Doc: "ValidateArguments checks a call's arguments against the parameters\nschema the request declared for the tool.", Nested: ""},
|
|
}},
|
|
{Package: "obs", Name: "Tracing", Title: "Tracing", Doc: "", Fields: []Field{
|
|
{Key: "endpoint", Type: "string", Doc: "Endpoint is the collector, for example http://otel.internal:4318.\nEmpty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, and to no tracing.", Nested: ""},
|
|
{Key: "sample_ratio", Type: "float64", Doc: "SampleRatio is the share of traces kept, 0 to 1, of those nobody\nupstream already decided. Zero takes 1. A request with a sampled\ntraceparent of its own is kept, and an agent keeps every start the\ncontrol plane kept, whatever their own ratio says.", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "VLLMConfig", Title: "VLLMConfig", Doc: "", Fields: []Field{
|
|
{Key: "bin", Type: "string", Doc: "Bin is the vllm executable.", Nested: ""},
|
|
{Key: "launch_mode", Type: "string", Doc: "LaunchMode selects the multi-node mechanism: \"mp\", the default, or\n\"ray\".\n\nmp is what vLLM documents today — every node runs serve with its rank,\nthe workers headless — and needs nothing but vLLM. ray is the older way,\nkept for a vLLM too old for mp's multi-node flags; it needs Ray on every\nnode and has not been checked against a real cluster.", Nested: ""},
|
|
{Key: "ray_bin", Type: "string", Doc: "RayBin is used when LaunchMode is \"ray\".", Nested: ""},
|
|
}},
|
|
{Package: "secrets", Name: "Vault", Title: "Vault", Doc: "", Fields: []Field{
|
|
{Key: "addr", Type: "string", Doc: "Addr is Vault's address, like https://vault.internal:8200; empty\ntakes VAULT_ADDR.", Nested: ""},
|
|
{Key: "token_file", Type: "string", Doc: "TokenFile holds a token; empty takes VAULT_TOKEN, or logs in with\nAppRole when RoleIDFile and SecretIDFile are set.", Nested: ""},
|
|
{Key: "role_id_file", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "secret_id_file", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "ca_file", Type: "string", Doc: "CAFile verifies Vault's certificate; empty uses the system's.", Nested: ""},
|
|
{Key: "namespace", Type: "string", Doc: "Namespace is Vault Enterprise's or OpenBao's namespace.", Nested: ""},
|
|
}},
|
|
{Package: "notify", Name: "Webhook", Title: "Webhook", Doc: "", Fields: []Field{
|
|
{Key: "url", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "format", Type: "string", Doc: "Format is json (the event, with X-Llmgw-Signature when SecretFile is\nset) or slack ({\"text\": …}, which Slack, Mattermost and Rocket.Chat\nincoming webhooks take).", Nested: ""},
|
|
{Key: "events", Type: "[]string", Doc: "Events are the kinds sent here; empty is all.", Nested: ""},
|
|
{Key: "secret_file", Type: "string", Doc: "SecretFile holds the key the body is signed with: X-Llmgw-Signature\nis sha256=<hex HMAC of the body>.", Nested: ""},
|
|
}},
|
|
{Package: "accounting", Name: "Price", Title: "accounting.Price", Doc: "", Fields: []Field{
|
|
{Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""},
|
|
{Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""},
|
|
{Key: "per_image", Type: "int64", Doc: "PerImage is what one picture costs, for a model that makes them: a\npicture's cost is the card's time, which its prompt's few tokens do\nnot measure.", Nested: ""},
|
|
{Key: "per_mesh", Type: "int64", Doc: "PerMesh is what one 3D model costs: minutes of a card, which no\ntoken count measures.", Nested: ""},
|
|
}},
|
|
{Package: "admission", Name: "Band", Title: "admission.Band", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "priority", Type: "int", Doc: "Priority orders bands; higher is served first.", Nested: ""},
|
|
{Key: "saturation_ceiling", Type: "float64", Doc: "SaturationCeiling is the pool saturation above which this band waits.\n1.0 means \"never hold this band back\".", Nested: ""},
|
|
{Key: "queue_ttl", Type: "time.Duration", Doc: "QueueTTL, set, is how long this band's requests may wait instead of\nthe controller's: a band for bulk work — a knowledge-base import, a\nbackfill — whose requests had rather wait ten minutes behind people\nthan be turned away after thirty seconds.", Nested: ""},
|
|
}},
|
|
{Package: "admission", Name: "Config", Title: "admission.Config", Doc: "", Fields: []Field{
|
|
{Key: "bands", Type: "[]Band", Doc: "", Nested: ""},
|
|
{Key: "default_band", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "queue_ttl", Type: "time.Duration", Doc: "QueueTTL drops a request that has waited this long. Sending the GPU work\nthe client has already given up on helps nobody.", Nested: ""},
|
|
{Key: "queue_capacity_per_band", Type: "int", Doc: "QueueCapacityPerBand bounds each band's queue. Rejecting immediately\nbeats accumulating silently.", Nested: ""},
|
|
{Key: "queue_capacity_total", Type: "int", Doc: "QueueCapacityTotal bounds everything queued, across bands. When it is\nreached, a request is not refused while a lower band has requests\nqueued: the newest of the lowest band is turned away instead\n(displacement), since the oldest have waited longest and are nearest\ntheir TTL. Zero takes half of every band's capacity together.", Nested: ""},
|
|
{Key: "interval", Type: "time.Duration", Doc: "Interval is how often the dispatcher re-checks saturation. Releases also\nwake it, so this is only the backstop for saturation falling on its own.", Nested: ""},
|
|
}},
|
|
{Package: "agent", Name: "Config", Title: "agent.Config", Doc: "", Fields: []Field{
|
|
{Key: "node_id", Type: "string", Doc: "NodeID defaults to the hostname. It must equal the common name of the\nagent's certificate, so set it only where the two differ.", Nested: ""},
|
|
{Key: "server", Type: "string", Doc: "Server is the control plane, and Servers is the same thing when there is\nmore than one. Both are accepted: every configuration written before\nthere could be a second control plane names one address, and none of\nthem should have to change.\n\nThe order is tried in order, and a refusal moves to the next. Only the\nleader accepts an agent, so a standby is skipped in the time it takes to\nbe told so — which is what makes failover a reconnect rather than an\nedit to every node's configuration.", Nested: ""},
|
|
{Key: "servers", Type: "[]string", Doc: "", Nested: ""},
|
|
{Key: "hostname", Type: "string", Doc: "Hostname defaults to the machine's.", Nested: ""},
|
|
{Key: "mgmt_addr", Type: "string", Doc: "MgmtAddr is the address engines are reached on. It defaults to this\nmachine's address on the route to the control plane, which is the one\nthe control plane and the router can reach.", Nested: ""},
|
|
{Key: "labels", Type: "map[string]string", Doc: "Labels are free-form facts about a node for a profile's node_selector.\nNothing needs them: memory, GPUs, engines and disk are measured, and a\nlabel restating one of those goes stale the day the machine changes.", Nested: ""},
|
|
{Key: "mgmt_interface", Type: "string", Doc: "MgmtInterface is the ethernet interface NCCL bootstraps over. It\ndefaults to the interface holding MgmtAddr, then eth0.", Nested: ""},
|
|
{Key: "fabrics", Type: "[]inventory.FabricAttachment", Doc: "Fabrics labels this node's HCAs. It is explicit on purpose: OpenSM's\ndefault subnet prefix is the same on every fabric, so nothing can be\ninferred from the wire (plan §5.2).", Nested: ""},
|
|
{Key: "heartbeat_interval", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "reconnect_base", Type: "time.Duration", Doc: "ReconnectBase is the first reconnect delay; it backs off from there.", Nested: ""},
|
|
{Key: "reconnect_max", Type: "time.Duration", Doc: "", Nested: ""},
|
|
{Key: "adopt_ready_timeout", Type: "time.Duration", Doc: "AdoptReadyTimeout bounds how long an adopted engine is given to answer a\nhealth check before the agent stops waiting on it.\n\nShorter than a startup timeout on purpose: this engine is already\nrunning, so it is not loading a model, it is either serving or it is not.", Nested: ""},
|
|
{Key: "tls", Type: "tlsconf.Config", Doc: "TLS authenticates both ends of the control-plane stream. Leaving it unset\nkeeps the plaintext behaviour, which only belongs on a trusted network.", Nested: "tlsconf.Config"},
|
|
{Key: "model_dirs", Type: "[]string", Doc: "ModelDirs are the directories this node keeps model weights in, beyond\nWeightsCache, which is always looked in. Storage differs at every installation,\nso llmgw observes it rather than owning it: a Lustre mount, an image\nexported over the fabric, local NVMe and a copy made by hand all look\nidentical from here, and placement only needs to know whether the file\nis present.", Nested: ""},
|
|
{Key: "engines_dir", Type: "string", Doc: "EnginesDir is where the agent installs engines, one directory per\nbuild. Defaults to /opt/llmgw/engines.", Nested: ""},
|
|
{Key: "weights_cache", Type: "string", Doc: "WeightsCache is where a remote source is downloaded to. Defaults to the\nfirst entry of ModelDirs, because a node that says where its models live\nhas already answered this question, and otherwise to\n/var/lib/llmgw-weights.\n\nFetching is the simple arrangement: every node keeps its own copy. The\nexotic ones need no code — weights on a shared filer or an image exported\nover the fabric are simply present, and present means nothing is fetched.", Nested: ""},
|
|
{Key: "discovery", Type: "discovery.Options", Doc: "", Nested: "discovery.Options"},
|
|
{Key: "engines", Type: "engines.Config", Doc: "", Nested: "engines.Config"},
|
|
{Key: "supervisor", Type: "supervisor.Config", Doc: "", Nested: "supervisor.Config"},
|
|
{Key: "tracing", Type: "obs.Tracing", Doc: "Tracing sends this node's part of each replica start — the weights\nfetched, the engine launched, the wait for it to answer — to an\nOpenTelemetry collector, inside the control plane's trace of that\nstart. Give it the collector server.yaml's tracing names. Off unless\nan endpoint is set here or in OTEL_EXPORTER_OTLP_ENDPOINT.", Nested: "obs.Tracing"},
|
|
}},
|
|
{Package: "config", Name: "Band", Title: "config.Band", Doc: "", Fields: []Field{
|
|
{Key: "name", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "priority", Type: "int", Doc: "", Nested: ""},
|
|
{Key: "saturation_ceiling", Type: "float64", Doc: "", Nested: ""},
|
|
{Key: "queue_ttl", Type: "time.Duration", Doc: "QueueTTL is how long this band's requests may wait for room; the\nadmission's queue_ttl unless set. Long for a band of bulk work, which\nhad rather wait behind people than be turned away.", Nested: ""},
|
|
}},
|
|
{Package: "config", Name: "Price", Title: "config.Price", Doc: "", Fields: []Field{
|
|
{Key: "prompt_per_1k", Type: "int64", Doc: "", Nested: ""},
|
|
{Key: "completion_per_1k", Type: "int64", Doc: "", Nested: ""},
|
|
{Key: "per_image", Type: "int64", Doc: "PerImage is the cost of one picture, for a model that makes them\n(/v1/images/generations), on top of its prompt's tokens.", Nested: ""},
|
|
{Key: "per_mesh", Type: "int64", Doc: "PerMesh is the cost of one 3D model (/v1/3d/generations).", Nested: ""},
|
|
}},
|
|
{Package: "engines", Name: "Config", Title: "engines.Config", Doc: "", Fields: []Field{
|
|
{Key: "disabled", Type: "[]string", Doc: "Disabled are engines this node never installs or advertises, whatever\nthe control plane asks: vLLM on a node of cards no build of it runs on,\nthat is meant for llama.cpp alone.", Nested: ""},
|
|
{Key: "vllm", Type: "VLLMConfig", Doc: "", Nested: "engines.VLLMConfig"},
|
|
{Key: "sglang", Type: "SGLangConfig", Doc: "", Nested: "engines.SGLangConfig"},
|
|
{Key: "llamacpp", Type: "LlamaCppConfig", Doc: "", Nested: "engines.LlamaCppConfig"},
|
|
{Key: "trtllm", Type: "TRTLLMConfig", Doc: "TRTLLM is TensorRT-LLM, which the agent does not install: a node\noffers it only when its interpreter is named here.", Nested: "engines.TRTLLMConfig"},
|
|
{Key: "sim", Type: "SimConfig", Doc: "", Nested: "engines.SimConfig"},
|
|
{Key: "comfyui", Type: "ComfyUIConfig", Doc: "ComfyUI makes pictures; offered by a node whose ComfyUI is named here.", Nested: "engines.ComfyUIConfig"},
|
|
}},
|
|
{Package: "guard", Name: "Config", Title: "guard.Config", Doc: "", Fields: []Field{
|
|
{Key: "enabled", Type: "*bool", Doc: "Enabled turns every policy on or off at once: false checks nothing,\nwhatever is defined or attached. Unset is on.", Nested: ""},
|
|
{Key: "default", Type: "[]string", Doc: "", Nested: ""},
|
|
{Key: "policies", Type: "map[string]Policy", Doc: "", Nested: ""},
|
|
}},
|
|
{Package: "mail", Name: "Config", Title: "mail.Config", Doc: "", Fields: []Field{
|
|
{Key: "smtp", Type: "string", Doc: "SMTP is the relay, host:port: 587 with STARTTLS, 465 with TLS from the\nstart, 25 as the relay allows. Empty sends nothing.", Nested: ""},
|
|
{Key: "from", Type: "string", Doc: "From is the sender, llmgw@example.com or \"llmgw <llmgw@example.com>\".", Nested: ""},
|
|
{Key: "username", Type: "string", Doc: "Username and PasswordFile log in to the relay, when it wants that.\nThe password is a secret reference: a file, env:NAME or vault:path.", Nested: ""},
|
|
{Key: "password_file", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "base_url", Type: "string", Doc: "BaseURL is the gateway's public address, which links in mail point\nat: https://llm.example.com. Never taken from a request, whose Host a\nsender chooses and could point a reset link at their own site.", Nested: ""},
|
|
}},
|
|
{Package: "notify", Name: "Config", Title: "notify.Config", Doc: "", Fields: []Field{
|
|
{Key: "webhooks", Type: "[]Webhook", Doc: "", Nested: ""},
|
|
{Key: "host", Type: "string", Doc: "Host names the gateway in messages; empty is this machine's name.", Nested: ""},
|
|
}},
|
|
{Package: "supervisor", Name: "Config", Title: "supervisor.Config", Doc: "", Fields: []Field{
|
|
{Key: "work_dir", Type: "string", Doc: "WorkDir holds pid files and per-shard scratch space.", Nested: ""},
|
|
{Key: "log_lines", Type: "int", Doc: "LogLines is how many lines of engine output to retain per shard.", Nested: ""},
|
|
{Key: "stop_timeout", Type: "time.Duration", Doc: "StopTimeout is how long a process gets to exit after SIGTERM before it is\nkilled. Engines need time to free GPU memory cleanly.", Nested: ""},
|
|
{Key: "health_interval", Type: "time.Duration", Doc: "HealthInterval is how often a starting shard is polled.", Nested: ""},
|
|
{Key: "rendezvous_timeout", Type: "time.Duration", Doc: "RendezvousTimeout bounds the wait for a multi-node rendezvous point to\nstart listening.", Nested: ""},
|
|
{Key: "alive_grace", Type: "time.Duration", Doc: "AliveGrace is how long a non-serving rank must stay up before it counts\nas ready. There is no external signal for a worker, so staying alive is\nthe only evidence available.", Nested: ""},
|
|
{Key: "run_as", Type: "string", Doc: "RunAs is the user engines run as when the agent is root; empty is\nllmgw-engine, \"root\" keeps them root.", Nested: ""},
|
|
}},
|
|
{Package: "tlsconf", Name: "Config", Title: "tlsconf.Config", Doc: "", Fields: []Field{
|
|
{Key: "ca", Type: "string", Doc: "CA is the bundle used to verify the other side. In a real deployment this\nis whichever authority issued the fleet's certificates.", Nested: ""},
|
|
{Key: "cert", Type: "string", Doc: "Cert and Key are this component's own identity.", Nested: ""},
|
|
{Key: "key", Type: "string", Doc: "", Nested: ""},
|
|
{Key: "insecure", Type: "bool", Doc: "Insecure disables TLS entirely. It exists for local runs against\nsim-engines; anything using it must stay on a trusted network.", Nested: ""},
|
|
}},
|
|
}
|