Files
shields/core/token-pooling/token-pool.spec.js
T
DCjanusandcopilot-swe-agent[bot] bd23e5fce6 [GitHub] Store accepted token scopes (#11915)
* feat(github): store token scopes

Assisted-by: Codex <gpt-5.5>

* perf(github): index token scopes

Assisted-by: Codex <gpt-5.5>

* feat(github): persist accepted token scopes

Assisted-by: Codex <gpt-5.5>

* docs(github): clarify OAuth scope capture

Assisted-by: Codex <gpt-5.5>

* docs(github): shorten OAuth scope note

Assisted-by: Codex <gpt-5.5>

* docs(github): clarify default OAuth scopes

Assisted-by: Codex <gpt-5.5>

* docs(github): link OAuth scope discussion

Assisted-by: Codex <gpt-5.5>

* docs(github): mark OAuth scopes as TODO

Assisted-by: Codex <gpt-5.5>

* fix(token-pool): refresh existing token data

Assisted-by: Codex <gpt-5.5>

* fix(token-pool): avoid duplicate revalidated tokens

Assisted-by: Codex:gpt-5.5

* fix(token-pool): remove revalidated tokens from priority queue

Assisted-by: Codex:gpt-5.5

* fix(token-pool): preserve token metadata on re-add

Assisted-by: Codex:gpt-5.5

* chore(github): defer token scopes index

Assisted-by: Codex:gpt-5.5

* fix(github): trim accepted token scopes

Assisted-by: Codex:gpt-5.5

* chore(github): keep token scopes index

Assisted-by: Codex:gpt-5.5

* docs(token-pool): document metadata update semantics

Assisted-by: Codex:gpt-5.5

* chore(github): store token scopes without index

Assisted-by: Codex:gpt-5.5

* chore: rerun ci

* test(token-pool): use mixed scope persistence fixture

Assisted-by: Codex:gpt-5.5

* fix(token-pool): replace invalid tokens on re-add

Keep stale queue references invalid so they can be removed lazily without rebuilding the priority queue.

Assisted-by: Codex:gpt-5.6-sol

* chore: apply Prettier 3.9 formatting

Assisted-by: Codex:gpt-5.6-sol

* chore: retry CI

Assisted-by: Codex:gpt-5.6-sol

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
2026-07-20 15:31:01 +02:00

332 lines
9.9 KiB
JavaScript

import { expect } from 'chai'
import sinon from 'sinon'
import { Token, TokenPool } from './token-pool.js'
const times = (n, fn) => [...Array(n)].map(() => fn())
function expectPoolToBeExhausted(pool) {
expect(() => {
pool.next()
}).to.throw(Error, /^Token pool is exhausted$/)
}
describe('The token pool', function () {
const ids = ['1', '2', '3', '4', '5']
const batchSize = 3
let tokenPool
beforeEach(function () {
tokenPool = new TokenPool({ batchSize })
ids.forEach(id => tokenPool.add(id))
})
it('should yield the expected tokens', function () {
ids.forEach(id =>
times(batchSize, () => expect(tokenPool.next().id).to.equal(id)),
)
})
it('should repeat when reaching the end', function () {
ids.forEach(id =>
times(batchSize, () => expect(tokenPool.next().id).to.equal(id)),
)
ids.forEach(id =>
times(batchSize, () => expect(tokenPool.next().id).to.equal(id)),
)
})
it('updates data when an existing token is added again', function () {
const tokenPool = new TokenPool()
expect(tokenPool.add('token', { scopes: [] })).to.equal(true)
expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal(
false,
)
expect(tokenPool.next().data).to.deep.equal({
scopes: ['read:packages'],
})
})
it('preserves existing data when an existing token is added again without data', function () {
const tokenPool = new TokenPool()
expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal(true)
expect(tokenPool.add('token')).to.equal(false)
expect(tokenPool.next().data).to.deep.equal({
scopes: ['read:packages'],
})
})
it('replaces invalid tokens when they are added again', function () {
const tokenPool = new TokenPool()
expect(tokenPool.add('token', { scopes: [] })).to.equal(true)
tokenPool.next().invalidate()
expectPoolToBeExhausted(tokenPool)
expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal(
false,
)
const token = tokenPool.next()
expect(token.id).to.equal('token')
expect(token.data).to.deep.equal({ scopes: ['read:packages'] })
})
it('isolates re-added tokens from stale references', function () {
const tokenPool = new TokenPool()
expect(tokenPool.add('token', { scopes: [] })).to.equal(true)
const staleToken = tokenPool.next()
staleToken.invalidate()
expect(tokenPool.add('token', { scopes: ['read:packages'] })).to.equal(
false,
)
staleToken.update(0, Token.nextResetNever)
const token = tokenPool.next()
expect(token).to.not.equal(staleToken)
expect(token.data).to.deep.equal({ scopes: ['read:packages'] })
})
it('preserves data when an invalid token is replaced without new data', function () {
const tokenPool = new TokenPool()
tokenPool.add('token', { scopes: ['read:packages'] })
tokenPool.next().invalidate()
expect(tokenPool.add('token')).to.equal(false)
expect(tokenPool.next().data).to.deep.equal({
scopes: ['read:packages'],
})
})
it('does not duplicate queued tokens when they are replaced', function () {
const tokenPool = new TokenPool({ batchSize: 2 })
tokenPool.add('first')
tokenPool.add('second')
tokenPool.next().invalidate()
tokenPool.add('first')
expect(tokenPool.next().id).to.equal('second')
expect(tokenPool.next().id).to.equal('second')
expect(tokenPool.next().id).to.equal('first')
expect(tokenPool.next().id).to.equal('first')
})
it('does not duplicate exhausted tokens when they are replaced', function () {
const tokenPool = new TokenPool()
tokenPool.add('first')
tokenPool.add('second')
const first = tokenPool.next()
first.update(0, Token.nextResetNever)
tokenPool.next()
first.invalidate()
tokenPool.add('first')
const { allTokenDebugInfo } = tokenPool.serializeDebugInfo({
sanitize: false,
})
const tokenIds = allTokenDebugInfo.map(({ id }) => id)
expect(tokenIds).to.have.members(['first', 'second'])
expect(
allTokenDebugInfo.find(({ id }) => id === 'first').isFrozen,
).to.equal(false)
})
it('removes invalid priority queue entries on access', function () {
const tokenPool = new TokenPool()
tokenPool.add('token', undefined, 0, Token.nextResetNever)
const token = tokenPool.tokensById.get('token')
expectPoolToBeExhausted(tokenPool)
token.invalidate()
expect(tokenPool.priorityQueue.size()).to.equal(1)
expectPoolToBeExhausted(tokenPool)
expect(tokenPool.priorityQueue.size()).to.equal(0)
})
context('tokens are marked exhausted immediately', function () {
it('should be exhausted', function () {
ids.forEach(() => {
const token = tokenPool.next()
token.update(0, Token.nextResetNever)
})
expectPoolToBeExhausted(tokenPool)
})
})
context('tokens are marked after the last request', function () {
it('should be exhausted', function () {
ids.forEach(() => {
const token = times(batchSize, () => tokenPool.next()).pop()
token.update(0, Token.nextResetNever)
})
expectPoolToBeExhausted(tokenPool)
})
})
context('tokens are renewed', function () {
it('should keep using them', function () {
const tokensToRenew = ['2', '4']
const renewalCount = 3
ids.forEach(id => {
const token = times(batchSize, () => tokenPool.next()).pop()
const usesRemaining = tokensToRenew.includes(token.id)
? renewalCount
: 0
token.update(usesRemaining, Token.nextResetNever)
})
tokensToRenew.forEach(id => {
let token
times(renewalCount, () => {
token = tokenPool.next()
expect(token.id).to.equal(id)
}).pop()
token.update(0, Token.nextResetNever)
})
expectPoolToBeExhausted(tokenPool)
})
})
context('tokens reset', function () {
let clock
beforeEach(function () {
clock = sinon.useFakeTimers()
})
afterEach(function () {
clock.restore()
})
it('should start using them', function () {
const tokensToReset = ['2', '4']
const futureTime = 1440
ids.forEach(id => {
const token = times(batchSize, () => tokenPool.next()).pop()
const nextReset = tokensToReset.includes(token.id)
? futureTime
: Token.nextResetNever
token.update(0, nextReset)
})
expectPoolToBeExhausted(tokenPool)
clock.tick(1000 * futureTime)
tokensToReset.forEach(id => {
const token = times(batchSize, () => tokenPool.next()).pop()
token.update(0, Token.nextResetNever)
})
expectPoolToBeExhausted(tokenPool)
})
})
context('when empty', function () {
it('next() should return the expected error', function () {
const tokenPool = new TokenPool()
expect(() => tokenPool.next()).to.throw('Token pool is exhausted')
})
})
context('failed attempts', function () {
it('should count and reset consecutive failed attempts', function () {
const token = new Token('1')
expect(token.failedAttempts).to.equal(0)
token.recordFailedAttempt()
token.recordFailedAttempt()
expect(token.failedAttempts).to.equal(2)
token.resetFailedAttempts()
expect(token.failedAttempts).to.equal(0)
})
})
context('endBatchFor()', function () {
it('rotates to a different token on the next call', function () {
const first = tokenPool.next()
tokenPool.endBatchFor(first)
expect(tokenPool.next().id).to.not.equal(first.id)
})
it('keeps the abandoned token in the rotation', function () {
const first = tokenPool.next()
tokenPool.endBatchFor(first)
// Cycle through the rest of the pool; the abandoned token should reappear
// once the other tokens' batches have been served.
const seen = times(ids.length * batchSize, () => tokenPool.next().id)
expect(seen).to.include(first.id)
})
it('does nothing when the token is not the current batch', function () {
const first = tokenPool.next()
const other = new Token('not-in-batch')
tokenPool.endBatchFor(other)
// first's batch is untouched, so it is returned again.
expect(tokenPool.next().id).to.equal(first.id)
})
})
context('serializeDebugInfo()', function () {
let clock
beforeEach(function () {
clock = sinon.useFakeTimers({ now: 1544307744484 })
})
afterEach(function () {
clock.restore()
})
it('should return sanitized debug info', function () {
const debugInfo = tokenPool.serializeDebugInfo()
expect(debugInfo.utcEpochSeconds).to.equal(1544307744)
expect(debugInfo.totalUsesRemaining).to.equal(ids.length * batchSize)
expect(debugInfo.allTokenDebugInfo).to.have.lengthOf(ids.length)
expect(debugInfo.sanitized).to.equal(true)
debugInfo.allTokenDebugInfo.forEach(tokenInfo => {
expect(tokenInfo.id).to.be.a('string')
expect(tokenInfo.id).to.have.lengthOf(64) // SHA-256 hex is 64 chars
expect(tokenInfo.data).to.equal('[redacted]')
expect(tokenInfo.usesRemaining).to.equal(batchSize)
expect(tokenInfo.nextReset).to.equal(Token.nextResetNever)
expect(tokenInfo.isValid).to.equal(true)
expect(tokenInfo.isFrozen).to.equal(false)
})
})
it('should return unsanitized debug info', function () {
const debugInfo = tokenPool.serializeDebugInfo({ sanitize: false })
expect(debugInfo.sanitized).to.equal(false)
debugInfo.allTokenDebugInfo.forEach((tokenInfo, index) => {
expect(tokenInfo.id).to.equal(ids[index])
})
})
it('should exclude invalidated tokens', function () {
const token = tokenPool.next()
token.invalidate()
const debugInfo = tokenPool.serializeDebugInfo()
expect(debugInfo.allTokenDebugInfo).to.have.lengthOf(ids.length - 1)
expect(debugInfo.totalUsesRemaining).to.equal(
(ids.length - 1) * batchSize,
)
})
})
})