mirror of
https://github.com/badges/shields.git
synced 2026-11-21 16:06:35 +03:00
1 parent
39d393028d
commit
9007658fd0
10 files changed
+212
-59
No files matched your search
+27
-45
@@ -5,40 +5,21 @@ const log = require('./log')
|
||||
const secretIsValid = require('./sys/secret-is-valid')
|
||||
const queryString = require('query-string')
|
||||
const request = require('request')
|
||||
const autosave = require('json-autosave')
|
||||
const serverSecrets = require('./server-secrets')
|
||||
const mapKeys = require('lodash.mapkeys')
|
||||
|
||||
// This is an initial value which makes the code work while the initial data
|
||||
// is loaded. In the then() callback of scheduleAutosaving(), it's reassigned
|
||||
// with a JsonSave object.
|
||||
let githubUserTokens = { data: [] }
|
||||
const userTokenRateLimit = 12500
|
||||
|
||||
function scheduleAutosaving(githubUserTokensFile) {
|
||||
autosave(githubUserTokensFile, { data: [] })
|
||||
.then(save => {
|
||||
githubUserTokens = save
|
||||
for (let i = 0; i < githubUserTokens.data.length; i++) {
|
||||
addGithubToken(githubUserTokens.data[i])
|
||||
}
|
||||
// Personal tokens allow access to GitHub private repositories.
|
||||
// You can manage your personal GitHub token at
|
||||
// <https://github.com/settings/tokens>.
|
||||
if (serverSecrets && serverSecrets.gh_token) {
|
||||
addGithubToken(serverSecrets.gh_token)
|
||||
}
|
||||
})
|
||||
.catch(e => {
|
||||
console.error('Could not create ' + githubUserTokensFile)
|
||||
})
|
||||
}
|
||||
let githubUserTokens = []
|
||||
// Ideally, we would want priority queues here.
|
||||
const reqRemaining = new Map() // From token to requests remaining.
|
||||
const reqReset = new Map() // From token to timestamp.
|
||||
|
||||
function cancelAutosaving() {
|
||||
if (githubUserTokens.stop) {
|
||||
githubUserTokens.stop()
|
||||
githubUserTokens.save()
|
||||
githubUserTokens = { data: [] }
|
||||
}
|
||||
// Personal tokens allow access to GitHub private repositories.
|
||||
// You can manage your personal GitHub token at
|
||||
// <https://github.com/settings/tokens>.
|
||||
if (serverSecrets && serverSecrets.gh_token) {
|
||||
addGithubToken(serverSecrets.gh_token)
|
||||
}
|
||||
|
||||
function setRoutes(server) {
|
||||
@@ -166,12 +147,6 @@ function sendTokenToAllServers(token) {
|
||||
)
|
||||
}
|
||||
|
||||
// Track rate limit requests remaining.
|
||||
|
||||
// Ideally, we would want priority queues here.
|
||||
const reqRemaining = new Map() // From token to requests remaining.
|
||||
const reqReset = new Map() // From token to timestamp.
|
||||
|
||||
// token: client token as a string.
|
||||
// reqs: number of requests remaining.
|
||||
// reset: timestamp when the number of remaining requests is reset.
|
||||
@@ -189,8 +164,6 @@ function utcEpochSeconds() {
|
||||
return (Date.now() / 1000) >>> 0
|
||||
}
|
||||
|
||||
const userTokenRateLimit = 12500
|
||||
|
||||
// Return false if the token cannot reasonably be expected to perform
|
||||
// a GitHub request.
|
||||
function isTokenUsable(token, now) {
|
||||
@@ -206,7 +179,7 @@ function isTokenUsable(token, now) {
|
||||
// with a reasonable chance that the request will succeed.
|
||||
function usableTokens() {
|
||||
const now = utcEpochSeconds()
|
||||
return githubUserTokens.data.filter(function(token) {
|
||||
return githubUserTokens.filter(function(token) {
|
||||
return isTokenUsable(token, now)
|
||||
})
|
||||
}
|
||||
@@ -235,17 +208,17 @@ function addGithubToken(token) {
|
||||
// A reset date of 0 has to be in the past.
|
||||
setReqRemaining(token, userTokenRateLimit, 0)
|
||||
// Insert it only if it is not registered yet.
|
||||
if (githubUserTokens.data.indexOf(token) === -1) {
|
||||
githubUserTokens.data.push(token)
|
||||
if (githubUserTokens.indexOf(token) === -1) {
|
||||
githubUserTokens.push(token)
|
||||
}
|
||||
}
|
||||
|
||||
function rmGithubToken(token) {
|
||||
rmReqRemaining(token)
|
||||
// Remove it only if it is in there.
|
||||
const idx = githubUserTokens.data.indexOf(token)
|
||||
const idx = githubUserTokens.indexOf(token)
|
||||
if (idx >= 0) {
|
||||
githubUserTokens.data.splice(idx, 1)
|
||||
githubUserTokens.splice(idx, 1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,12 +239,20 @@ function sha(str) {
|
||||
.digest('hex')
|
||||
}
|
||||
|
||||
function getAllTokenIds() {
|
||||
return githubUserTokens.slice()
|
||||
}
|
||||
|
||||
function removeAllTokens() {
|
||||
githubUserTokens = []
|
||||
}
|
||||
|
||||
function serializeDebugInfo(options) {
|
||||
// Apply defaults.
|
||||
const { sanitize } = Object.assign({ sanitize: true }, options)
|
||||
|
||||
const unsanitized = {
|
||||
tokens: githubUserTokens.data,
|
||||
tokens: githubUserTokens,
|
||||
reqRemaining: mapToObject(reqRemaining),
|
||||
reqReset: mapToObject(reqReset),
|
||||
utcEpochSeconds: utcEpochSeconds(),
|
||||
@@ -347,9 +328,10 @@ function githubRequest(request, url, query, cb) {
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scheduleAutosaving,
|
||||
cancelAutosaving,
|
||||
request: githubRequest,
|
||||
setRoutes,
|
||||
serializeDebugInfo,
|
||||
addGithubToken,
|
||||
getAllTokenIds,
|
||||
removeAllTokens,
|
||||
}
|
||||
@@ -56,9 +56,9 @@ function reset(server) {
|
||||
*
|
||||
* @param {Object} server instance
|
||||
*/
|
||||
function stop(server) {
|
||||
async function stop(server) {
|
||||
if (server) {
|
||||
server.stop()
|
||||
await server.stop()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -57,8 +57,8 @@ before('Start running the server', function() {
|
||||
this.timeout(5000)
|
||||
server = serverHelpers.start()
|
||||
})
|
||||
after('Shut down the server', function() {
|
||||
serverHelpers.stop(server)
|
||||
after('Shut down the server', async function() {
|
||||
await serverHelpers.stop(server)
|
||||
})
|
||||
|
||||
const runner = new Runner()
|
||||
|
||||
@@ -83,8 +83,12 @@ describe('Text formatters', function() {
|
||||
})
|
||||
|
||||
context('in october', function() {
|
||||
let clock
|
||||
beforeEach(function() {
|
||||
sinon.useFakeTimers(new Date(2017, 9, 15).getTime())
|
||||
clock = sinon.useFakeTimers(new Date(2017, 9, 15).getTime())
|
||||
})
|
||||
afterEach(function() {
|
||||
clock.restore()
|
||||
})
|
||||
|
||||
test(formatDate, () => {
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
'use strict'
|
||||
|
||||
const autosave = require('json-autosave')
|
||||
const githubAuth = require('./github-auth')
|
||||
|
||||
// This is currently bound to the legacy github auth code. That will be
|
||||
// replaced with a dependency-injected token provider.
|
||||
class TokenPersistence {
|
||||
constructor({ path }) {
|
||||
this.path = path
|
||||
this.save = undefined
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
// This code is a bit difficult to understand, in part because it's
|
||||
// working against the interface of `json-autosave` which wants to own the
|
||||
// data structure.
|
||||
const save = await autosave(this.path, { data: [] })
|
||||
this.save = save
|
||||
|
||||
save.data.forEach(tokenString => {
|
||||
githubAuth.addGithubToken(tokenString)
|
||||
})
|
||||
|
||||
// Override the autosave handler to refresh the token data before
|
||||
// saving.
|
||||
save.autosave = () => {
|
||||
save.data = githubAuth.getAllTokenIds()
|
||||
return save.save()
|
||||
}
|
||||
// Put the change in autosave handler into effect.
|
||||
save.stop()
|
||||
save.start()
|
||||
}
|
||||
|
||||
async stop() {
|
||||
if (this.save) {
|
||||
this.save.stop()
|
||||
await this.save.autosave()
|
||||
this.save = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = TokenPersistence
|
||||
@@ -0,0 +1,93 @@
|
||||
'use strict'
|
||||
|
||||
const fs = require('fs')
|
||||
const tmp = require('tmp')
|
||||
const readFile = require('fs-readfile-promise')
|
||||
const sinon = require('sinon')
|
||||
const { sleep } = require('wait-promise')
|
||||
const { expect } = require('chai')
|
||||
const TokenPersistence = require('./token-persistence')
|
||||
const githubAuth = require('./github-auth')
|
||||
|
||||
describe('Token persistence', function() {
|
||||
// Fake timers must be set up before any timers are scheduled.
|
||||
let clock
|
||||
beforeEach(function() {
|
||||
clock = sinon.useFakeTimers()
|
||||
})
|
||||
afterEach(function() {
|
||||
clock.restore()
|
||||
})
|
||||
|
||||
beforeEach(githubAuth.removeAllTokens)
|
||||
afterEach(githubAuth.removeAllTokens)
|
||||
|
||||
let path, persistence
|
||||
beforeEach(function() {
|
||||
path = tmp.tmpNameSync()
|
||||
persistence = new TokenPersistence({ path })
|
||||
})
|
||||
afterEach(function() {
|
||||
if (persistence) {
|
||||
persistence.stop()
|
||||
persistence = null
|
||||
}
|
||||
})
|
||||
|
||||
context('when the file does not exist', function() {
|
||||
it('creates it with an empty array', async function() {
|
||||
await persistence.initialize()
|
||||
const json = JSON.parse(await readFile(path))
|
||||
|
||||
expect(json).to.deep.equal([])
|
||||
})
|
||||
})
|
||||
|
||||
context('when the file exists', function() {
|
||||
it('loads the contents', async function() {
|
||||
const initialTokens = ['a', 'b', 'c'].map(char => char.repeat(40))
|
||||
fs.writeFileSync(path, JSON.stringify(initialTokens))
|
||||
|
||||
await persistence.initialize()
|
||||
|
||||
expect(githubAuth.getAllTokenIds()).to.deep.equal(initialTokens)
|
||||
})
|
||||
})
|
||||
|
||||
context('when shutting down', function() {
|
||||
it('writes added tokens to the file', async function() {
|
||||
const initialTokens = ['a', 'b', 'c'].map(char => char.repeat(40))
|
||||
fs.writeFileSync(path, JSON.stringify(initialTokens))
|
||||
|
||||
const newToken = 'e'.repeat(40)
|
||||
const expected = initialTokens.slice()
|
||||
expected.push(newToken)
|
||||
|
||||
await persistence.initialize()
|
||||
githubAuth.addGithubToken(newToken)
|
||||
await persistence.stop()
|
||||
|
||||
const json = JSON.parse(await readFile(path))
|
||||
expect(json).to.deep.equal(expected)
|
||||
})
|
||||
})
|
||||
|
||||
context('time has elapsed', function() {
|
||||
it('writes added tokens to the file', async function() {
|
||||
const addedTokens = ['d', 'e'].map(char => char.repeat(40))
|
||||
|
||||
await persistence.initialize()
|
||||
|
||||
addedTokens.forEach(githubAuth.addGithubToken)
|
||||
|
||||
// Fake time passing to trigger autosaving, then give the save a brief
|
||||
// moment of real time to complete before reading.
|
||||
clock.tick(5000)
|
||||
clock.restore()
|
||||
await sleep(200)
|
||||
|
||||
const json = JSON.parse(await readFile(path))
|
||||
expect(json).to.deep.equal(addedTokens)
|
||||
})
|
||||
})
|
||||
})
|
||||
Generated
+15
@@ -5762,6 +5762,15 @@
|
||||
"integrity": "sha1-mC1ok6+RjnLQjeyehnP/K1qNat0=",
|
||||
"dev": true
|
||||
},
|
||||
"fs-readfile-promise": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fs-readfile-promise/-/fs-readfile-promise-3.0.1.tgz",
|
||||
"integrity": "sha512-LsSxMeaJdYH27XrW7Dmq0Gx63mioULCRel63B5VeELYLavi1wF5s0XfsIdKDFdCL9hsfQ2qBvXJszQtQJ9h17A==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"graceful-fs": "^4.1.11"
|
||||
}
|
||||
},
|
||||
"fs-write-stream-atomic": {
|
||||
"version": "1.0.10",
|
||||
"resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz",
|
||||
@@ -14648,6 +14657,12 @@
|
||||
"integrity": "sha512-8Xz4H3vhYRGbFupLtl6dHwMx0ojUcjt0HYkqZ9oBCfipd/5mD7Md58m2/dq7uPuZU/0T3Gb1m66KS9jn+I+14Q==",
|
||||
"dev": true
|
||||
},
|
||||
"wait-promise": {
|
||||
"version": "0.4.1",
|
||||
"resolved": "https://registry.npmjs.org/wait-promise/-/wait-promise-0.4.1.tgz",
|
||||
"integrity": "sha1-Hq2PgtKRolAQwLHu+NRxEyPMTas=",
|
||||
"dev": true
|
||||
},
|
||||
"walk": {
|
||||
"version": "2.3.9",
|
||||
"resolved": "https://registry.npmjs.org/walk/-/walk-2.3.9.tgz",
|
||||
|
||||
+4
-1
@@ -133,6 +133,7 @@
|
||||
"eslint-plugin-react": "^7.6.1",
|
||||
"eslint-plugin-standard": "^3.0.1",
|
||||
"fetch-ponyfill": "^6.0.0",
|
||||
"fs-readfile-promise": "^3.0.1",
|
||||
"icedfrisby": "2.0.0-alpha.2",
|
||||
"icedfrisby-nock": "^1.0.0",
|
||||
"is-png": "^1.1.0",
|
||||
@@ -163,7 +164,9 @@
|
||||
"sinon": "^6.0.0",
|
||||
"sinon-chai": "^3.0.0",
|
||||
"snap-shot-it": "^5.0.1",
|
||||
"url": "^0.11.0"
|
||||
"tmp": "0.0.33",
|
||||
"url": "^0.11.0",
|
||||
"wait-promise": "^0.4.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8.x",
|
||||
|
||||
@@ -132,10 +132,12 @@ function reset() {
|
||||
clearRegularUpdateCache();
|
||||
}
|
||||
|
||||
function stop(callback) {
|
||||
githubConstellation.stop();
|
||||
async function stop() {
|
||||
await githubConstellation.stop();
|
||||
analytics.cancelAutosaving();
|
||||
camp.close(callback);
|
||||
return new Promise(resolve => {
|
||||
camp.close(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
||||
@@ -4,6 +4,7 @@ const path = require('path')
|
||||
const githubAuth = require('../../lib/github-auth')
|
||||
const serverSecrets = require('../../lib/server-secrets')
|
||||
const log = require('../../lib/log')
|
||||
const TokenPersistence = require('../../lib/token-persistence')
|
||||
const GithubApiProvider = require('./github-api-provider')
|
||||
const { setRoutes: setAdminRoutes } = require('./auth/admin')
|
||||
|
||||
@@ -13,10 +14,12 @@ class GithubConstellation {
|
||||
constructor(config) {
|
||||
this._debugEnabled = config.service.debug.enabled
|
||||
this._debugIntervalSeconds = config.service.debug.intervalSeconds
|
||||
this._userTokensPath = path.resolve(
|
||||
|
||||
const userTokensPath = path.resolve(
|
||||
config.persistence.dir,
|
||||
'github-user-tokens.json'
|
||||
)
|
||||
this.persistence = new TokenPersistence({ path: userTokensPath })
|
||||
|
||||
const baseUrl = process.env.GITHUB_URL || 'https://api.github.com'
|
||||
this.apiProvider = new GithubApiProvider({ baseUrl })
|
||||
@@ -33,8 +36,12 @@ class GithubConstellation {
|
||||
async initialize(server) {
|
||||
this.scheduleDebugLogging()
|
||||
|
||||
githubAuth.scheduleAutosaving(this._userTokensPath)
|
||||
// TODO Catch errors and send them to Sentry.
|
||||
try {
|
||||
await this.persistence.initialize()
|
||||
} catch (e) {
|
||||
// TODO Send to sentry.
|
||||
console.error(e)
|
||||
}
|
||||
|
||||
setAdminRoutes(server)
|
||||
|
||||
@@ -43,12 +50,14 @@ class GithubConstellation {
|
||||
}
|
||||
}
|
||||
|
||||
stop() {
|
||||
githubAuth.cancelAutosaving()
|
||||
async stop() {
|
||||
if (this.debugInterval) {
|
||||
clearInterval(this.debugInterval)
|
||||
this.debugInterval = undefined
|
||||
}
|
||||
|
||||
await this.persistence.stop()
|
||||
this.persistence = undefined
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user