# Arithmetic a small instruction model gets right: sums, products and a
# little reading, each answer scored by the last number in it. A quantisation
# that has gone wrong, or a chat template that no longer matches the
# weights, shows up here as a falling score long before anyone notices it in
# conversation.
#
#   llmgw ctl eval MODEL --suite arithmetic
#   llmgw ctl model update MODEL --eval arithmetic --min-score 0.8
#
# Copy it into control.eval_dir (/etc/llmgw/eval) to change it; a suite
# there of the same name replaces this one.
name: arithmetic
description: Sums, products and word problems, each scored by the number in its answer.
threshold: 0.7
system: You are a careful calculator. Answer with the number only, without words or units.
max_tokens: 64
cases:
  - id: add-small
    prompt: What is 17 + 25?
    number: 42
  - id: add-carry
    prompt: What is 389 + 476?
    number: 865
  - id: subtract
    prompt: What is 1000 - 357?
    number: 643
  - id: subtract-negative
    prompt: What is 12 - 30?
    number: -18
  - id: multiply
    prompt: What is 12 * 12?
    number: 144
  - id: multiply-larger
    prompt: What is 23 * 17?
    number: 391
    weight: 2
  - id: divide
    prompt: What is 144 / 6?
    number: 24
  - id: decimal
    prompt: What is 2.5 * 4?
    number: 10
  - id: percent
    prompt: What is 15% of 200?
    number: 30
  - id: words-apples
    prompt: Ana has 7 apples and buys 5 more, then gives 3 away. How many apples does she have?
    number: 9
    weight: 2
  - id: words-time
    prompt: A train leaves at 14:00 and the trip takes 150 minutes. How many hours does the trip take?
    number: 2.5
    tolerance: 0.01
    weight: 2
  - id: order-of-operations
    prompt: What is 2 + 3 * 4?
    number: 14
    weight: 2