1
0
mirror of https://github.com/quay/quay.git synced 2026-10-23 09:49:06 +03:00
Files
quay/internal/auth/credential_cache.go
Harish Govindarajulu 79e230b183 PROJQUAY-13204: fix(omr): remember verified passwords so token issuance skips bcrypt (#7151)
Every GET /v2/auth with Basic credentials ran bcrypt.CompareHashAndPassword
against the user's $2b$12$ hash, 325 ms of CPU per token on a 2-vCPU host.
Registry clients re-authenticate constantly: skopeo requests two tokens
per image and oc-mirror one per repository. During a 1,000-tag push the
registry process ran at 1.4 cores, about 650 ms per push of it bcrypt and
under 15 ms actual registry work, so push throughput capped at roughly two
images per second regardless of image size and a burst of fresh pull
clients queued for seconds on their first token. Quay's Python service
verifies passwords the same way but is rarely hit because production
clients use robot accounts, whose secret is compared without bcrypt, and
gunicorn spreads logins across workers; the mirror registry is a single
process whose only client is a mirror tool using the bootstrap user's
password.

Remember a successful verification for a short TTL (default five
minutes) and serve repeat logins with the same credentials without
bcrypt. The cache never holds the secret: it keeps an HMAC-SHA256 of the
presented secret under a random per-process key together with the
password hash that was current when the verification succeeded. A hit
needs the same username, an unexpired entry, the same stored hash, so a
password change invalidates immediately, and a matching HMAC. The user
row is still read on every request, so a disabled account is rejected at
once. Only successes are cached; a wrong password always pays the full
bcrypt cost and gains no timing information. The cache is bounded to
1,024 entries.

Add PASSWORD_AUTH_CACHE_TTL_S (default 300, 0 disables, validated
non-negative) and plumb it to both the registry token endpoint and the
API v1 authenticator. Unit tests cover repeat login, wrong password,
password change, disabled user, TTL expiry, disabled cache, unknown
user, absence of plaintext in the cache and the size bound; config tests
cover parsing, the default and validation.

On the same host, ten sequential token requests now cost 386 ms of
registry CPU instead of 3,251 ms, and a two-way parallel skopeo push loop
dropped from 664 ms to 25 ms of registry CPU per push, from 139 percent to
16 percent CPU, and rose from 2.1 to 6.5 pushes per second.
2026-09-17 15:56:18 -05:00

134 lines
3.8 KiB
Go

package auth
import (
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"crypto/subtle"
"sync"
"time"
)
// DefaultPasswordCacheTTL is how long a successful password verification is
// remembered so repeat logins with the same credentials skip bcrypt.
const DefaultPasswordCacheTTL = 5 * time.Minute
// maxCredentialCacheEntries bounds the cache; the mirror registry has a
// handful of users, so this is far above any realistic working set.
const maxCredentialCacheEntries = 1024
// credentialCache remembers recent successful password verifications.
//
// A bcrypt comparison at the cost factor Quay uses takes roughly a third of a
// CPU second on a small host, and registry clients request a new token for
// every image (skopeo asks twice per push), so on a single-process mirror
// registry token issuance was the dominant CPU cost of a push. Quay's Python
// service has the same per-login cost but is rarely hit by it because
// production clients use robot accounts, whose secret is compared without
// bcrypt.
//
// The cache never stores the secret itself: it keeps an HMAC-SHA256 of the
// presented secret under a random per-process key, together with the password
// hash that was current when the verification succeeded. A hit requires the
// same username, an unexpired entry, the same stored hash (so a password
// change invalidates immediately) and a matching HMAC. Only successful
// verifications are cached, so a wrong password always pays the full bcrypt
// cost and gains no timing information.
type credentialCache struct {
ttl time.Duration
now func() time.Time
key []byte
mu sync.Mutex
entries map[string]credentialCacheEntry
}
type credentialCacheEntry struct {
passwordHash string
secretMAC []byte
expiresAt time.Time
}
func newCredentialCache(ttl time.Duration) *credentialCache {
if ttl <= 0 {
return nil
}
key := make([]byte, sha256.Size)
if _, err := rand.Read(key); err != nil {
// Without a random key the cache cannot be made safe; run without it.
return nil
}
return &credentialCache{
ttl: ttl,
now: time.Now,
key: key,
entries: make(map[string]credentialCacheEntry),
}
}
func (c *credentialCache) mac(secret string) []byte {
h := hmac.New(sha256.New, c.key)
h.Write([]byte(secret))
return h.Sum(nil)
}
// matches reports whether a recent successful verification exists for the
// same username, stored password hash and secret.
func (c *credentialCache) matches(username, passwordHash, secret string) bool {
if c == nil {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
entry, ok := c.entries[username]
if !ok {
return false
}
if !c.now().Before(entry.expiresAt) {
delete(c.entries, username)
return false
}
if subtle.ConstantTimeCompare([]byte(entry.passwordHash), []byte(passwordHash)) != 1 {
delete(c.entries, username)
return false
}
return hmac.Equal(entry.secretMAC, c.mac(secret))
}
// remember records a successful verification.
func (c *credentialCache) remember(username, passwordHash, secret string) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
now := c.now()
if len(c.entries) >= maxCredentialCacheEntries {
c.evictLocked(now)
}
c.entries[username] = credentialCacheEntry{
passwordHash: passwordHash,
secretMAC: c.mac(secret),
expiresAt: now.Add(c.ttl),
}
}
// evictLocked drops expired entries and, if the cache is still full, the
// entry closest to expiry.
func (c *credentialCache) evictLocked(now time.Time) {
var oldestKey string
var oldest time.Time
for key, entry := range c.entries {
if !now.Before(entry.expiresAt) {
delete(c.entries, key)
continue
}
if oldestKey == "" || entry.expiresAt.Before(oldest) {
oldestKey, oldest = key, entry.expiresAt
}
}
if len(c.entries) >= maxCredentialCacheEntries && oldestKey != "" {
delete(c.entries, oldestKey)
}
}