llmgw/internal/docgen/openapi/openapi.go

// Package openapi generates the admin API's OpenAPI 3.1 document from its
// route table, adminapi.Routes.
//
// The schemas are read from the Go types the handlers decode and write, by
// reflection, following encoding/json's rules — tag names, omitted fields,
// embedded structs flattened — so the document says what goes over the wire
// rather than what someone remembered to write down. Their descriptions are
// the types' and fields' doc comments, read from the source, which is why
// generating needs the repository and is done by `go generate` rather than
// at run time: the binary serves the document built into it.
package openapi

import (
	"bytes"
	"encoding"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path"
	"path/filepath"
	"reflect"
	"slices"
	"sort"
	"strings"
	"time"
	"unicode"

	"gitea.mynodes.xyz/ilker/llmgw/internal/clientapi"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)

// Module is the import path the repository's packages are under; a type
// from any other is described without comments.
const Module = "gitea.mynodes.xyz/ilker/llmgw"

// Doc is what one document says of itself and its API: the two APIs llmgw
// has, the admin listener's and the client's, are described by one
// generator from their route tables.
type Doc struct {
	Title, Description string
	// ServerURL and ServerDescription are where it is served by default.
	ServerURL, ServerDescription string
	// Scheme is the bearer token's name in the document, SchemeDescription
	// what it is.
	Scheme, SchemeDescription string
	// Error is a value of the type every refusal answers with.
	Error any
	// Prefix is taken off paths to name operations and tags.
	Prefix string
	// Home is the package whose types are named alone; others are named
	// with their package.
	Home string
	// Actor is whether changes take the admin API's X-Llmgw-Actor header.
	Actor bool
}

// Admin is the admin API's document.
var Admin = Doc{
	Title: "llmgw admin API", Description: infoDescription,
	ServerURL: "http://127.0.0.1:9090", ServerDescription: "The admin listener at its default address, admin_listen.",
	Scheme: "adminToken",
	SchemeDescription: "The admin token: the contents of /var/lib/llmgw/admin-token on the " +
		"control plane (admin_token_file), readable by root and the server. " +
		"Sent as Authorization: Bearer TOKEN.",
	Error: adminapi.ErrorView{}, Prefix: "/api/v1/", Home: reflect.TypeOf(adminapi.Route{}).PkgPath(), Actor: true,
}

// Client is the client API's document: the API on :8080 clients use.
var Client = Doc{
	Title: "llmgw client API", Description: clientDescription,
	ServerURL: "http://localhost:8080", ServerDescription: "A gateway at its default address, listen.",
	Scheme: "apiKey",
	SchemeDescription: "An API key, sk-llmgw-…, made with `llmgw keys create` or on the portal's profile page. " +
		"Sent as Authorization: Bearer KEY; /v1/messages also takes it as x-api-key.",
	Error: clientapi.Error{}, Prefix: "/v1/", Home: reflect.TypeOf(clientapi.Error{}).PkgPath(),
}

const clientDescription = "OpenAI's API, as llmgw serves it, and what llmgw adds: pictures and 3D models from " +
	"ComfyUI, knowledge bases, memory notes, stored conversations.\n\n" +
	"The SDKs of OpenAI and Anthropic work against it unchanged: set the base URL to the gateway's /v1 and the " +
	"key to an llmgw key. A request carries more than is listed here: what the gateway does not read goes to " +
	"the engine as it came. Headers llmgw reads: x-llmgw-thread (continue a stored conversation), " +
	"x-llmgw-knowledge (answer from these collections), x-llmgw-tags (tags kept with the usage), " +
	"x-session-id (stay on one replica). Headers it sends back: x-request-id, x-llmgw-served-model, " +
	"x-llmgw-hardware, x-llmgw-seed, and Retry-After when waiting helps.\n\n" +
	"Generated from the route table in internal/clientapi; see docs/clients.md."

// Generate renders the admin API's document, reading doc comments from the
// repository at root.
func Generate(root string, routes []adminapi.Route, pathParams map[string]string) ([]byte, error) {
	return GenerateDoc(root, Admin, routes, pathParams)
}

// GenerateDoc renders one API's document.
func GenerateDoc(root string, d Doc, routes []adminapi.Route, pathParams map[string]string) ([]byte, error) {
	errorType := reflect.TypeOf(d.Error)
	g := &generator{root: root, docs: map[string]map[string]typeDoc{}, names: map[reflect.Type]string{}, d: d}

	// Every named struct reachable from a body first, so a name two
	// packages share can be told apart before either is written: by the name
	// alone where it is unique, with its package where it is not.
	for _, r := range routes {
		for _, v := range []any{r.Request, r.Response} {
			if v != nil {
				g.collect(reflect.TypeOf(v))
			}
		}
	}
	g.collect(errorType)
	g.assignNames()
	errorSchema := g.schema(errorType)

	paths := map[string]*object{}
	var order []string
	tags := map[string]bool{}
	for _, r := range routes {
		item, ok := paths[r.Path]
		if !ok {
			item = &object{}
			paths[r.Path] = item
			order = append(order, r.Path)
		}
		op, err := g.operation(r, pathParams)
		if err != nil {
			return nil, err
		}
		tags[g.tagOf(r.Path)] = true
		item.set(strings.ToLower(r.Method), op)
	}
	slices.Sort(order)
	var pathsObj object
	for _, p := range order {
		pathsObj.set(p, *paths[p])
	}

	var schemas object
	names := make([]string, 0, len(g.schemas))
	for n := range g.schemas {
		names = append(names, n)
	}
	sort.Strings(names)
	for _, n := range names {
		schemas.set(n, g.schemas[n])
	}

	var tagList []any
	for _, t := range sortedKeys(tags) {
		tagList = append(tagList, obj("name", t))
	}

	components := obj(
		"securitySchemes", obj(d.Scheme, obj(
			"type", "http",
			"scheme", "bearer",
			"description", d.SchemeDescription,
		)),
	)
	if d.Actor {
		components.set("parameters", obj("Actor", obj(
			"name", "X-Llmgw-Actor",
			"in", "header",
			"required", false,
			"description", "Who the change is recorded as made by in the audit log: letters, digits "+
				"and ._-@, at most 64. unknown when absent.",
			"schema", obj("type", "string"),
		)))
	}
	components.set("responses", obj(
		"Error", obj("description", "Refused: the error says why.",
			"content", obj("application/json", obj("schema", errorSchema))),
		"Unauthorized", obj("description", "The token is missing or wrong.",
			"content", obj("application/json", obj("schema", errorSchema))),
	))
	components.set("schemas", schemas)
	doc := obj(
		"openapi", "3.1.0",
		"info", obj(
			"title", d.Title,
			"version", "v1",
			"description", d.Description,
		),
		"servers", []any{obj("url", d.ServerURL, "description", d.ServerDescription)},
		"security", []any{obj(d.Scheme, []any{})},
		"tags", tagList,
		"paths", pathsObj,
		"components", components,
	)
	b, err := marshal(doc)
	if err != nil {
		return nil, err
	}
	var out bytes.Buffer
	if err := json.Indent(&out, b, "", "  "); err != nil {
		return nil, err
	}
	out.WriteByte('\n')
	return out.Bytes(), nil
}

const infoDescription = "The control plane's management API: what `llmgw ctl` and the console use.\n\n" +
	"It is served on the admin listener (admin_listen, 127.0.0.1:9090 by default), which stays on " +
	"loopback unless admin_remote is set: reach it from elsewhere through an SSH tunnel. Every path " +
	"under /api needs the admin token as a bearer token.\n\n" +
	"The listener also refuses, before the token is looked at: a Host header that is a name rather " +
	"than an address or localhost; a request another site's page sent; and any method but GET and " +
	"HEAD whose Content-Type is not application/json, even one with no body.\n\n" +
	"Durations in bodies are integer nanoseconds, as Go encodes them. Generated from the route " +
	"table in internal/control/adminapi; see docs/admin-api.md."

// operation builds one route's operation object.
func (g *generator) operation(r adminapi.Route, pathParams map[string]string) (object, error) {
	var op object
	op.set("operationId", g.operationID(r.Method, r.Path))
	op.set("summary", r.Summary)
	if r.Description != "" {
		op.set("description", r.Description)
	}
	op.set("tags", []any{g.tagOf(r.Path)})

	var params []any
	segs := strings.Split(strings.Trim(r.Path, "/"), "/")
	for i, s := range segs {
		if !strings.HasPrefix(s, "{") {
			continue
		}
		name := strings.Trim(s, "{}")
		key := s
		if i > 0 {
			key = segs[i-1] + "/" + s
		}
		desc, ok := pathParams[key]
		if !ok {
			return nil, fmt.Errorf("%s %s: no description of %s in the path parameters", r.Method, r.Path, key)
		}
		params = append(params, obj("name", name, "in", "path", "required", true,
			"description", desc, "schema", obj("type", "string")))
	}
	for _, q := range r.Query {
		t := q.Type
		if t == "" {
			t = "string"
		}
		p := obj("name", q.Name, "in", "query")
		if q.Required {
			p.set("required", true)
		}
		p.set("description", q.Description)
		p.set("schema", obj("type", t))
		params = append(params, p)
	}
	if g.d.Actor && r.Method != "GET" && r.Method != "HEAD" {
		params = append(params, obj("$ref", "#/components/parameters/Actor"))
	}
	if len(params) > 0 {
		op.set("parameters", params)
	}

	if r.Request != nil {
		ct := "application/json"
		if r.Form {
			ct = "multipart/form-data"
		}
		body := obj("required", !r.OptionalBody,
			"content", obj(ct, obj("schema", g.schema(reflect.TypeOf(r.Request)))))
		op.set("requestBody", body)
	}

	status := r.Status
	if status == 0 {
		status = 200
	}
	var content object
	switch {
	case r.Binary != "":
		content = obj(r.Binary, obj("schema", obj("type", "string", "format", "binary")))
	case r.Response == nil:
		content = obj("application/json", obj("schema", obj()))
	default:
		content = obj("application/json", obj("schema", g.schema(reflect.TypeOf(r.Response))))
	}
	if r.CSV {
		content.set("text/csv", obj("schema", obj("type", "string")))
	}
	if r.Stream {
		content.set("text/event-stream", obj("schema", obj("type", "string",
			"description", "With stream: true, server-sent events: one JSON chunk per data: line, then data: [DONE].")))
	}
	desc := "OK"
	if status == 202 {
		desc = "Accepted: started, and running on."
	}
	op.set("responses", obj(
		fmt.Sprint(status), obj("description", desc, "content", content),
		"401", obj("$ref", "#/components/responses/Unauthorized"),
		"default", obj("$ref", "#/components/responses/Error"),
	))
	return op, nil
}

// operationID is the method and the path's words, getModelsNameTarget.
func (g *generator) operationID(method, p string) string {
	var b strings.Builder
	b.WriteString(strings.ToLower(method))
	for _, s := range strings.Split(strings.TrimPrefix(p, g.d.Prefix), "/") {
		for _, w := range strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) {
			b.WriteString(strings.ToUpper(w[:1]) + w[1:])
		}
	}
	return b.String()
}

// tagOf groups an operation by the first word of its path.
func (g *generator) tagOf(p string) string {
	first, _, _ := strings.Cut(strings.TrimPrefix(strings.TrimPrefix(p, g.d.Prefix), "/"), "/")
	return strings.TrimSuffix(first, ".json")
}

// --- schemas ---

type typeDoc struct {
	doc    string
	fields map[string]string
}

type generator struct {
	root    string
	docs    map[string]map[string]typeDoc
	names   map[reflect.Type]string
	schemas map[string]any
	seen    []reflect.Type
	d       Doc
}

var (
	timeType      = reflect.TypeOf(time.Time{})
	durationType  = reflect.TypeOf(time.Duration(0))
	rawType       = reflect.TypeOf(json.RawMessage{})
	marshalerType = reflect.TypeOf((*json.Marshaler)(nil)).Elem()
	textType      = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
)

// collect records every named struct reachable from t.
func (g *generator) collect(t reflect.Type) {
	for t.Kind() == reflect.Pointer || t.Kind() == reflect.Slice || t.Kind() == reflect.Array || t.Kind() == reflect.Map {
		if t == rawType {
			return
		}
		t = t.Elem()
	}
	if t.Kind() != reflect.Struct || special(t) {
		return
	}
	if t.Name() != "" {
		if slices.Contains(g.seen, t) {
			return
		}
		g.seen = append(g.seen, t)
	}
	for _, f := range fields(t) {
		g.collect(f.typ)
	}
}

// assignNames names each struct's schema: by its name alone when it is the
// admin API's own, with its package's otherwise — catalog.Model,
// modelserve.Job. Qualified always rather than only on a collision, so a
// type added elsewhere one day cannot rename a schema clients already use.
func (g *generator) assignNames() {
	own := g.d.Home
	for _, t := range g.seen {
		n := t.Name()
		if t.PkgPath() != own {
			n = path.Base(t.PkgPath()) + "." + n
		}
		g.names[t] = n
	}
	g.schemas = map[string]any{}
}

func (g *generator) nameOf(t reflect.Type) string { return g.names[t] }

// special is a type encoded other than by its fields.
func special(t reflect.Type) bool {
	if t == timeType || t == durationType || t == rawType {
		return true
	}
	return t.Implements(marshalerType) || reflect.PointerTo(t).Implements(marshalerType) ||
		t.Implements(textType) || reflect.PointerTo(t).Implements(textType)
}

func ref(name string) object { return obj("$ref", "#/components/schemas/"+name) }

// schema describes t, adding the named structs it reaches to the components.
func (g *generator) schema(t reflect.Type) any {
	switch {
	case t == timeType:
		return obj("type", "string", "format", "date-time")
	case t == durationType:
		return obj("type", "integer", "format", "int64", "description", "A duration, in nanoseconds.")
	case t == rawType:
		return obj("description", "Any JSON value.")
	case t.Implements(textType) || reflect.PointerTo(t).Implements(textType):
		return obj("type", "string")
	case t.Implements(marshalerType) || reflect.PointerTo(t).Implements(marshalerType):
		return obj("description", "Encoded by the type's own JSON encoding.")
	}
	switch t.Kind() {
	case reflect.Bool:
		return obj("type", "boolean")
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Uint8, reflect.Uint16, reflect.Uint32:
		return obj("type", "integer")
	case reflect.Int64, reflect.Uint, reflect.Uint64:
		return obj("type", "integer", "format", "int64")
	case reflect.Float32, reflect.Float64:
		return obj("type", "number")
	case reflect.String:
		return obj("type", "string")
	case reflect.Pointer:
		return g.schema(t.Elem())
	case reflect.Slice, reflect.Array:
		if t.Elem().Kind() == reflect.Uint8 {
			return obj("type", "string", "contentEncoding", "base64")
		}
		return obj("type", "array", "items", g.schema(t.Elem()))
	case reflect.Map:
		return obj("type", "object", "additionalProperties", g.schema(t.Elem()))
	case reflect.Interface:
		return obj()
	case reflect.Struct:
		if t.Name() == "" {
			return g.structSchema(t, "")
		}
		name := g.names[t]
		if name == "" {
			// Reached only through a body collect did not walk, which would be
			// a bug here; named anyway rather than inlined so it shows.
			name = t.Name()
			g.names[t] = name
		}
		if _, ok := g.schemas[name]; !ok {
			g.schemas[name] = nil // placeholder: a type that contains itself
			g.schemas[name] = g.structSchema(t, g.typeDocOf(t).doc)
		}
		return ref(name)
	}
	return obj()
}

func (g *generator) structSchema(t reflect.Type, doc string) object {
	s := obj("type", "object")
	if doc != "" {
		s.set("description", doc)
	}
	var props object
	for _, f := range fields(t) {
		var p any
		if f.asString {
			p = obj("type", "string")
		} else {
			p = g.schema(f.typ)
		}
		if d := g.typeDocOf(f.owner).fields[f.goName]; d != "" {
			p = withDescription(p, d)
		}
		props.set(f.name, p)
	}
	s.set("properties", props)
	return s
}

// withDescription puts a field's comment on its schema. A reference keeps
// its own description where it is declared; in 3.1 a sibling description
// beside $ref is allowed and is the field's.
func withDescription(p any, d string) any {
	o, ok := p.(object)
	if !ok {
		return p
	}
	if prev, ok := o.get("description"); ok {
		d = d + "\n\n" + prev.(string)
	}
	o = slices.Clone(o)
	o.set("description", d)
	return o
}

// field is one key of a struct's JSON.
type field struct {
	name     string
	goName   string
	owner    reflect.Type // the struct it is declared in, for its comment
	typ      reflect.Type
	asString bool
	depth    int
}

// fields lists a struct's JSON keys as encoding/json writes them: exported
// fields, by tag name, "-" left out, embedded structs without a name of
// their own flattened, and a key at a shallower depth winning over the same
// key from deeper in.
func fields(t reflect.Type) []field {
	var out []field
	var walk func(t reflect.Type, depth int)
	walk = func(t reflect.Type, depth int) {
		for i := 0; i < t.NumField(); i++ {
			sf := t.Field(i)
			tag := sf.Tag.Get("json")
			if tag == "-" {
				continue
			}
			name, opts, _ := strings.Cut(tag, ",")
			ft := sf.Type
			if sf.Anonymous && name == "" {
				et := ft
				if et.Kind() == reflect.Pointer {
					et = et.Elem()
				}
				if et.Kind() == reflect.Struct && !special(et) {
					walk(et, depth+1)
					continue
				}
			}
			if !sf.IsExported() {
				continue
			}
			if name == "" {
				name = sf.Name
			}
			out = append(out, field{name: name, goName: sf.Name, owner: t, typ: ft, depth: depth,
				asString: slices.Contains(strings.Split(opts, ","), "string")})
		}
	}
	walk(t, 0)
	// The shallowest of each name, in the order they were met.
	best := map[string]int{}
	for _, f := range out {
		if d, ok := best[f.name]; !ok || f.depth < d {
			best[f.name] = f.depth
		}
	}
	var kept []field
	taken := map[string]bool{}
	for _, f := range out {
		if f.depth == best[f.name] && !taken[f.name] {
			taken[f.name] = true
			kept = append(kept, f)
		}
	}
	return kept
}

// typeDocOf reads a type's comment and its fields' from its package's source.
func (g *generator) typeDocOf(t reflect.Type) typeDoc {
	pkg := t.PkgPath()
	if pkg != Module && !strings.HasPrefix(pkg, Module+"/") {
		return typeDoc{}
	}
	docs, ok := g.docs[pkg]
	if !ok {
		docs = parseDocs(filepath.Join(g.root, filepath.FromSlash(strings.TrimPrefix(strings.TrimPrefix(pkg, Module), "/"))))
		g.docs[pkg] = docs
	}
	return docs[t.Name()]
}

// parseDocs reads the struct types declared in a directory and their
// comments. A directory it cannot read gives no comments rather than an
// error: the document is still right, only quieter.
func parseDocs(dir string) map[string]typeDoc {
	out := map[string]typeDoc{}
	entries, err := os.ReadDir(dir)
	if err != nil {
		return out
	}
	fset := token.NewFileSet()
	for _, e := range entries {
		n := e.Name()
		if e.IsDir() || !strings.HasSuffix(n, ".go") || strings.HasSuffix(n, "_test.go") {
			continue
		}
		f, err := parser.ParseFile(fset, filepath.Join(dir, n), nil, parser.ParseComments)
		if err != nil {
			continue
		}
		for _, d := range f.Decls {
			gd, ok := d.(*ast.GenDecl)
			if !ok || gd.Tok != token.TYPE {
				continue
			}
			for _, sp := range gd.Specs {
				ts := sp.(*ast.TypeSpec)
				doc := ts.Doc
				if doc == nil && len(gd.Specs) == 1 {
					doc = gd.Doc
				}
				td := typeDoc{doc: text(doc), fields: map[string]string{}}
				if st, ok := ts.Type.(*ast.StructType); ok {
					for _, fl := range st.Fields.List {
						c := fl.Doc
						if c == nil {
							c = fl.Comment
						}
						for _, name := range fieldNames(fl) {
							td.fields[name] = text(c)
						}
					}
				}
				out[ts.Name.Name] = td
			}
		}
	}
	return out
}

func fieldNames(fl *ast.Field) []string {
	if len(fl.Names) > 0 {
		var out []string
		for _, n := range fl.Names {
			out = append(out, n.Name)
		}
		return out
	}
	// Embedded: named by its type.
	t := fl.Type
	if s, ok := t.(*ast.StarExpr); ok {
		t = s.X
	}
	switch x := t.(type) {
	case *ast.Ident:
		return []string{x.Name}
	case *ast.SelectorExpr:
		return []string{x.Sel.Name}
	}
	return nil
}

// text is a comment as prose: the comment markers gone, lines within a
// paragraph joined, paragraphs kept apart.
func text(g *ast.CommentGroup) string {
	if g == nil {
		return ""
	}
	var paras []string
	var cur []string
	for _, line := range strings.Split(g.Text(), "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			if len(cur) > 0 {
				paras = append(paras, strings.Join(cur, " "))
				cur = nil
			}
			continue
		}
		cur = append(cur, line)
	}
	if len(cur) > 0 {
		paras = append(paras, strings.Join(cur, " "))
	}
	return strings.Join(paras, "\n\n")
}

// --- an object that keeps its keys in order ---

type kv struct {
	k string
	v any
}

// object is a JSON object written in the order its keys were set, so the
// document reads in the order it was meant to — openapi, info, paths — and
// a struct's properties in the order its fields are declared.
type object []kv

func obj(pairs ...any) object {
	var o object
	for i := 0; i+1 < len(pairs); i += 2 {
		o.set(pairs[i].(string), pairs[i+1])
	}
	return o
}

func (o *object) set(k string, v any) {
	for i := range *o {
		if (*o)[i].k == k {
			(*o)[i].v = v
			return
		}
	}
	*o = append(*o, kv{k, v})
}

func (o object) get(k string) (any, bool) {
	for _, p := range o {
		if p.k == k {
			return p.v, true
		}
	}
	return nil, false
}

// MarshalJSON writes the keys in order.
func (o object) MarshalJSON() ([]byte, error) {
	var b bytes.Buffer
	b.WriteByte('{')
	for i, p := range o {
		if i > 0 {
			b.WriteByte(',')
		}
		k, err := marshal(p.k)
		if err != nil {
			return nil, err
		}
		b.Write(k)
		b.WriteByte(':')
		v, err := marshal(p.v)
		if err != nil {
			return nil, err
		}
		b.Write(v)
	}
	b.WriteByte('}')
	return b.Bytes(), nil
}

// marshal is json.Marshal without escaping <, > and &, which are common in
// the descriptions and harmless in a document nobody puts in a page.
func marshal(v any) ([]byte, error) {
	var b bytes.Buffer
	enc := json.NewEncoder(&b)
	enc.SetEscapeHTML(false)
	if err := enc.Encode(v); err != nil {
		return nil, err
	}
	return bytes.TrimRight(b.Bytes(), "\n"), nil
}

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}