diff --git a/internal/store/dialect.go b/internal/store/dialect.go index 6e32b19e..99ac72ce 100644 --- a/internal/store/dialect.go +++ b/internal/store/dialect.go @@ -44,6 +44,18 @@ func (d Dialect) rebind(query string) string { inSingle, inDouble := false, false for i := 0; i < len(query); i++ { c := query[i] + // A line comment is copied as it is. Read as SQL, the apostrophe in + // "-- the key's owner" opened a string that swallowed every + // placeholder after it, and PostgreSQL refused the query. + if c == '-' && !inSingle && !inDouble && i+1 < len(query) && query[i+1] == '-' { + end := strings.IndexByte(query[i:], '\n') + if end < 0 { + end = len(query) - i + } + b.WriteString(query[i : i+end]) + i += end - 1 + continue + } switch { case c == '\'' && !inDouble: inSingle = !inSingle diff --git a/internal/store/dialect_test.go b/internal/store/dialect_test.go index a15af61d..329e5777 100644 --- a/internal/store/dialect_test.go +++ b/internal/store/dialect_test.go @@ -42,3 +42,17 @@ func TestAQueryWithNoPlaceholdersIsUntouched(t *testing.T) { t.Errorf("rebind = %q", got) } } + +func TestACommentDoesNotOpenAString(t *testing.T) { + // An apostrophe in a comment is not a quote. Read as one, it swallowed + // every placeholder after it. + got := PostgresDialect.rebind("SELECT a -- the key's owner\nFROM t WHERE x = ? AND y = ?") + want := "SELECT a -- the key's owner\nFROM t WHERE x = $1 AND y = $2" + if got != want { + t.Errorf("got %q", got) + } + // A comment's own question mark is not a placeholder either. + if got := PostgresDialect.rebind("SELECT 1 -- why?\nWHERE x = ?"); got != "SELECT 1 -- why?\nWHERE x = $1" { + t.Errorf("got %q", got) + } +} diff --git a/internal/store/keyusage_test.go b/internal/store/keyusage_test.go new file mode 100644 index 00000000..9190a6b4 --- /dev/null +++ b/internal/store/keyusage_test.go @@ -0,0 +1,61 @@ +package store + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +// checkUsageNamesWhoAndWhere is run against both dialects: a report that +// named people by id and projects by id was a table nobody could read. +func checkUsageNamesWhoAndWhere(t *testing.T, s *DB) { + t.Helper() + ctx := context.Background() + must := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + must(s.PutOrg(ctx, Org{ID: "o", Name: "Acme"})) + must(s.PutTeam(ctx, Team{ID: "t", OrgID: "o", Name: "ml"})) + must(s.PutProject(ctx, Project{ID: "p", TeamID: "t", Name: "training"})) + must(s.CreateUser(ctx, User{ID: "usr-1", Email: "ada@example.com", CreatedAt: time.Now()})) + now := time.Now() + must(s.WriteUsage(ctx, []Usage{ + {At: now, KeyID: "session:usr-1", UserID: "usr-1", ProjectID: "p", Model: "m", ServedModel: "m", Status: 200}, + {At: now, KeyID: "session:usr-1", UserID: "usr-1", Model: "m", ServedModel: "m", Status: 200}, + })) + + rows, err := s.UsageSince(ctx, now.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(rows) != 2 { + t.Fatalf("expected two rows, got %+v", rows) + } + places := map[string]bool{} + for _, r := range rows { + if r.Email != "ada@example.com" || !r.Chat() { + t.Errorf("row %+v should name ada and read as the chat page", r) + } + places[r.ProjectPath] = true + } + if !places["Acme / ml / training"] || !places[""] { + t.Errorf("places = %v, expected the project's path and personal", places) + } +} + +func TestUsageNamesWhoAndWhere(t *testing.T) { + s, err := OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "u.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + checkUsageNamesWhoAndWhere(t, s) +} + +func TestPostgresUsageNamesWhoAndWhere(t *testing.T) { + checkUsageNamesWhoAndWhere(t, openPG(t)) +} diff --git a/internal/store/sqlite.go b/internal/store/sqlite.go index 849592a6..bb00cdf9 100644 --- a/internal/store/sqlite.go +++ b/internal/store/sqlite.go @@ -621,21 +621,30 @@ func (s *DB) UsageSince(ctx context.Context, since time.Time) ([]KeyUsage, error return queryAll(ctx, s.db, ` SELECT u.key_id, COALESCE(k.name, ''), COALESCE(k.key_prefix, ''), u.project_id, + COALESCE(us.email, ''), + COALESCE(o.name || ' / ' || t.name || ' / ' || p.name, ''), COUNT(*), SUM(u.prompt_tokens), SUM(u.completion_tokens), SUM(u.cost_micros), SUM(CASE WHEN u.status >= 400 THEN 1 ELSE 0 END), MAX(u.ts) FROM usage u LEFT JOIN api_keys k ON k.id = u.key_id + -- Who, as recorded with the spend; the owner of the key for rows + -- written before the person was. + LEFT JOIN users us ON us.id = COALESCE(NULLIF(u.user_id, ''), k.user_id) + LEFT JOIN projects p ON p.id = NULLIF(u.project_id, '') + LEFT JOIN teams t ON t.id = p.team_id + LEFT JOIN organizations o ON o.id = t.org_id WHERE u.ts >= ? - -- k.name and k.key_prefix are selected, so PostgreSQL wants them - -- grouped. Both are determined by the key id already in the list. - GROUP BY u.key_id, u.project_id, k.name, k.key_prefix + -- Everything selected beside the sums is grouped, which PostgreSQL + -- wants; each is determined by the key and project already there. + GROUP BY u.key_id, u.project_id, k.name, k.key_prefix, us.email, o.name, t.name, p.name ORDER BY SUM(u.cost_micros) DESC, COUNT(*) DESC`, []any{millis(since)}, func(rows *sql.Rows) (KeyUsage, error) { var k KeyUsage var last int64 err := rows.Scan(&k.KeyID, &k.KeyName, &k.KeyPrefix, &k.ProjectID, + &k.Email, &k.ProjectPath, &k.Requests, &k.Prompt, &k.Completion, &k.CostMicros, &k.Errors, &last) k.LastAt = fromMillis(last) return k, err diff --git a/internal/store/store.go b/internal/store/store.go index a89f10b4..b0f4141d 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,6 +17,7 @@ package store import ( "context" "errors" + "strings" "time" ) @@ -292,18 +293,25 @@ type Store interface { // KeyUsage summarizes one key's activity in a period, for the dashboard and // `ctl`. It is a reporting query and never runs on the request path. type KeyUsage struct { - KeyID string - KeyName string - KeyPrefix string - ProjectID string - Requests int64 - Prompt int64 - Completion int64 - CostMicros int64 - Errors int64 - LastAt time.Time + KeyID string + KeyName string + KeyPrefix string + ProjectID string + // Who spent it and where, as names: the person's email, and the + // project's path. Empty for a key with no owner, and for personal use. + Email string + ProjectPath string + Requests int64 + Prompt int64 + Completion int64 + CostMicros int64 + Errors int64 + LastAt time.Time } +// Chat reports whether this is the chat page's traffic, which has no key. +func (k KeyUsage) Chat() bool { return strings.HasPrefix(k.KeyID, "session:") } + // Bucket is one time slice of usage, for the dashboard's charts. type Bucket struct { Start time.Time diff --git a/internal/web/templates/admin_usage.html b/internal/web/templates/admin_usage.html index ff5c29a1..31ac345a 100644 --- a/internal/web/templates/admin_usage.html +++ b/internal/web/templates/admin_usage.html @@ -48,14 +48,14 @@ {{if .Rows}} - + {{range .Rows}} - - - - + + + + {{end}}
GroupRequestsTokensSpendErrors
GroupRequestsTokensSpendErrors
{{.Name}}{{compact .Requests}}{{compact .Tokens}}{{.Spend}}{{if .Errors}}{{.Errors}}{{else}}—{{end}}{{compact .Requests}}{{compact .Tokens}}{{.Spend}}{{if .Errors}}{{.Errors}}{{else}}—{{end}}
diff --git a/internal/web/templates/dashboard.html b/internal/web/templates/dashboard.html index 59a68de4..9c1bc246 100644 --- a/internal/web/templates/dashboard.html +++ b/internal/web/templates/dashboard.html @@ -201,14 +201,15 @@ requested whenever a fallback served it.

Usage — last 24 hours

{{if .AuthEnabled}} {{if .Usage}} - +
KeyProjectRequestsErrorsPromptCompletionSpendLast
{{range .Usage}} - - + + + - - + + {{end}}
PersonThroughSpends againstRequestsErrorsPromptCompletionSpendLast
{{if .KeyName}}{{.KeyName}} {{.KeyPrefix}}{{else}}{{.KeyID}}{{end}}{{.ProjectID}}{{or .Email "—"}}{{if .Chat}}chat page{{else if .KeyName}}{{.KeyName}} {{.KeyPrefix}}{{else}}{{.KeyID}}{{end}}{{if .ProjectPath}}{{.ProjectPath}}{{else if .ProjectID}}{{.ProjectID}}{{else}}personal{{end}} {{.Requests}} {{if .Errors}}{{.Errors}}{{else}}0{{end}}{{.Prompt}}{{.Completion}}{{compact .Prompt}}{{compact .Completion}} {{units .CostMicros}} {{ago .LastAt}}
diff --git a/internal/web/timeseries.go b/internal/web/timeseries.go index 8d2062f4..4d01533e 100644 --- a/internal/web/timeseries.go +++ b/internal/web/timeseries.go @@ -61,7 +61,7 @@ type LegendEntry struct { // Ranked by peak, which is what a gauge asks: the model that hit the ceiling is // the one worth a colour. For a counter, see LineChartTotals. func LineChart(series []promq.Series, unit string, maxSeries int) TimeSeries { - return lineChart(series, unit, maxSeries, promq.Series.Max) + return lineChart(series, unit, maxSeries, promq.Series.Max, promq.Series.Last) } // LineChartTotals is LineChart ranked by the sum over the window. @@ -70,12 +70,16 @@ func LineChart(series []promq.Series, unit string, maxSeries int) TimeSeries { // the total, and ranking by peak would give a colour to the team that made one // large burst over the team that ran steadily all week — the opposite of what a // usage report is read for. +// +// Its legend gives each line's total too. The latest slice, which is what a +// gauge's legend shows, is usually a quiet one for a counter: a legend reading +// "0 · 0 · 0" under a table of hundreds said nothing true. func LineChartTotals(series []promq.Series, unit string, maxSeries int) TimeSeries { - return lineChart(series, unit, maxSeries, promq.Series.Sum) + return lineChart(series, unit, maxSeries, promq.Series.Sum, promq.Series.Sum) } func lineChart(series []promq.Series, unit string, maxSeries int, - weight func(promq.Series) float64, + weight, legend func(promq.Series) float64, ) TimeSeries { if maxSeries <= 0 || maxSeries > len(vizColours) { maxSeries = len(vizColours) @@ -111,7 +115,7 @@ func lineChart(series []promq.Series, unit string, maxSeries int, } writeLine(&b, s, minT, span, maxV, colour) out.Legend = append(out.Legend, LegendEntry{ - Name: s.Name, Colour: colour, Last: formatValue(s.Last(), unit), + Name: s.Name, Colour: colour, Last: formatValue(legend(s), unit), }) } b.WriteString(``) diff --git a/internal/web/timeseries_test.go b/internal/web/timeseries_test.go index 7dd926b6..2d515bd9 100644 --- a/internal/web/timeseries_test.go +++ b/internal/web/timeseries_test.go @@ -183,3 +183,16 @@ func TestWithNoMetricsStoreThereAreNoPanels(t *testing.T) { t.Errorf("expected nothing, got %+v", got) } } + +func TestAUsageLegendGivesTotalsNotTheLastSlice(t *testing.T) { + // A counter's latest slice is usually quiet; "0" beside a line that + // carried hundreds read as a broken chart. + c := LineChartTotals([]promq.Series{ts("ml", 5, 7, 0), ts("vision", 1, 2, 0)}, "", 4) + got := map[string]string{} + for _, l := range c.Legend { + got[l.Name] = l.Last + } + if got["ml"] != "12" || got["vision"] != "3" { + t.Errorf("legend = %v, expected the totals", got) + } +} diff --git a/internal/web/web.go b/internal/web/web.go index a143c306..35cc5748 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -80,12 +80,7 @@ var funcs = template.FuncMap{ } return p }, - "ago": func(t time.Time) string { - if t.IsZero() { - return "never" - } - return time.Since(t).Round(time.Second).String() + " ago" - }, + "ago": ago, // Spend is stored in micro-units so accumulation is exact; it is shown in // whole units because that is the number an operator set a budget in. // compact renders a large count the way someone says it aloud: 1.2k rather @@ -226,3 +221,33 @@ func humanLatency(ms float64) string { return fmt.Sprintf("%dm %02ds", int(d.Minutes()), int(d.Seconds())%60) } } + +// ago says how long ago something was the way a person would: to the unit +// that matters and no finer. "3h7m25s ago" is precise about seconds nobody +// asked about and slower to read than "3 hours ago". +func ago(t time.Time) string { + if t.IsZero() { + return "never" + } + d := time.Since(t) + unit := func(n int, one string) string { + if n == 1 { + return "1 " + one + " ago" + } + return fmt.Sprintf("%d %ss ago", n, one) + } + switch { + case d < 45*time.Second: + return "just now" + case d < time.Hour: + return unit(int(d.Minutes()+0.5), "minute") + case d < 36*time.Hour: + return unit(int(d.Hours()+0.5), "hour") + case d < 14*24*time.Hour: + return unit(int(d.Hours()/24+0.5), "day") + case d < 60*24*time.Hour: + return unit(int(d.Hours()/24/7+0.5), "week") + default: + return unit(int(d.Hours()/24/30+0.5), "month") + } +}