Stop a one-in-sixty-two test from failing once in a while
TestRejections built a "wrong" key by replacing a valid one's last character with X. Keys are base62, so one in sixty-two already ends in X and the wrong key was the right one, which authenticated and failed the assertion. It passed locally all day and failed in CI, which is how long a flake of that shape takes to look like a real bug — long enough that the instinct on seeing it is to go looking in the authenticator. The replacement is now chosen to differ from what it replaces, with a test on the helper itself: a mutation that can return its input turns the test it serves into one that quietly checks nothing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
79e459d9ff
commit
ec943f9587
1 file changed
+35
-1
@@ -74,7 +74,7 @@ func TestRejections(t *testing.T) {
|
||||
if _, err := a.Authenticate(ctx, "ghp_0123456789abcdefghij"); !errors.Is(err, ErrInvalid) {
|
||||
t.Errorf("foreign token: %v", err)
|
||||
}
|
||||
if _, err := a.Authenticate(ctx, secret[:len(secret)-1]+"X"); !errors.Is(err, ErrInvalid) {
|
||||
if _, err := a.Authenticate(ctx, alter(secret)); !errors.Is(err, ErrInvalid) {
|
||||
t.Errorf("wrong key: %v", err)
|
||||
}
|
||||
|
||||
@@ -100,6 +100,21 @@ func (c *countingStore) KeyByHash(ctx context.Context, hash []byte) (store.Key,
|
||||
return c.Store.KeyByHash(ctx, hash)
|
||||
}
|
||||
|
||||
// alter changes a key by one character, guaranteeing the result differs.
|
||||
//
|
||||
// Replacing the last character with a fixed one does not: keys are base62, so
|
||||
// one in sixty-two already ends in that character and the "wrong" key is the
|
||||
// right one. That test passed locally for a day and failed in CI, which is
|
||||
// exactly how long a one-in-sixty-two flake takes to look like a real bug.
|
||||
func alter(secret string) string {
|
||||
last := secret[len(secret)-1]
|
||||
replacement := byte('a')
|
||||
if last == replacement {
|
||||
replacement = 'b'
|
||||
}
|
||||
return secret[:len(secret)-1] + string(replacement)
|
||||
}
|
||||
|
||||
func TestTheRequestPathDoesNotReadTheDatabase(t *testing.T) {
|
||||
// The rule the whole package is shaped around. Without the cache this is a
|
||||
// database read per request, on every request.
|
||||
@@ -404,3 +419,22 @@ func TestExemptingTheLandingPageDoesNotExemptEverything(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlterAlwaysChangesTheKey(t *testing.T) {
|
||||
// The helper the test above depends on. If it can return its input, the
|
||||
// test it serves silently stops testing anything.
|
||||
for range 500 {
|
||||
secret, _, err := store.NewKey("id", "n", "p")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if alter(secret) == secret {
|
||||
t.Fatalf("alter returned its input for %q", secret)
|
||||
}
|
||||
}
|
||||
// Including the case that made the original flaky: a key ending in the
|
||||
// replacement character.
|
||||
if got := alter("sk-llmgw-aaaa"); got == "sk-llmgw-aaaa" {
|
||||
t.Error("alter is a no-op when the key already ends in the replacement")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user