# Following a format: asked for JSON of a given shape, does the model give
# it — the thing an application parsing its answers depends on, and the
# first to go when a chat template drifts. Each answer is read as JSON (a
# code fence or a sentence around it is forgiven) and held to a schema.
#
#   llmgw ctl eval MODEL --suite json-format
#
# Copy it into control.eval_dir (/etc/llmgw/eval) to change it; a suite
# there of the same name replaces this one.
name: json-format
description: Answers asked for as JSON of a given shape, held to a schema.
threshold: 0.7
system: You answer with a single JSON value and nothing else.
max_tokens: 200
cases:
  - id: person
    prompt: >-
      Give a JSON object for a person named Ada Lovelace, born in 1815, with the
      keys "name" (a string) and "born" (a number).
    json_schema:
      type: object
      required: [name, born]
      properties:
        name: {type: string, pattern: "Ada"}
        born: {type: integer, minimum: 1815, maximum: 1815}

  - id: list-of-colours
    prompt: Give a JSON array of exactly three colour names, as strings.
    json_schema:
      type: array
      minItems: 3
      maxItems: 3
      items: {type: string, minLength: 1}

  - id: boolean-answer
    prompt: >-
      Is water wet? Answer as a JSON object with one key, "answer", whose
      value is true or false.
    json_schema:
      type: object
      required: [answer]
      properties:
        answer: {type: boolean}

  - id: nested
    prompt: >-
      Give a JSON object describing a book: "title" (a string), "author" (an
      object with "first" and "last", both strings) and "pages" (a whole number
      above zero). Use any book.
    json_schema:
      type: object
      required: [title, author, pages]
      properties:
        title: {type: string, minLength: 1}
        author:
          type: object
          required: [first, last]
          properties:
            first: {type: string}
            last: {type: string}
        pages: {type: integer, exclusiveMinimum: 0}
    weight: 2

  - id: enum-choice
    prompt: >-
      Classify the sentence "I love this product!" as positive, negative or
      neutral. Answer as a JSON object {"sentiment": "..."} using one of those
      three words in lower case.
    json_schema:
      type: object
      required: [sentiment]
      properties:
        sentiment: {enum: [positive]}
    weight: 2

  - id: numbers
    prompt: 'Give a JSON object {"sum": N} where N is the sum of 4, 5 and 6.'
    json_schema:
      type: object
      required: [sum]
      properties:
        sum: {const: 15}

  - id: no-extra-keys
    prompt: >-
      Give a JSON object with exactly two keys, "city" and "country", for
      Paris. No other keys.
    json_schema:
      type: object
      required: [city, country]
      additionalProperties: false
      properties:
        city: {type: string}
        country: {type: string}