updating use of indices and queries

Signed-off-by: David Lawrence <david.lawrence@docker.com> (github: endophage)

cmd/notary-signer/main.go

@@ -16,6 +16,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 
+	"github.com/dancannon/gorethink"
 	"github.com/docker/distribution/health"
 	"github.com/docker/notary"
 	"github.com/docker/notary/cryptoservice"
@@ -23,6 +24,7 @@ import (
 	"github.com/docker/notary/signer"
 	"github.com/docker/notary/signer/api"
 	"github.com/docker/notary/signer/keydbstore"
+	"github.com/docker/notary/storage/rethinkdb"
 	"github.com/docker/notary/trustmanager"
 	"github.com/docker/notary/tuf/data"
 	"github.com/docker/notary/utils"
@@ -42,10 +44,11 @@ const (
 )
 
 var (
-	debug      bool
-	logFormat  string
-	configFile string
-	mainViper  = viper.New()
+	debug       bool
+	logFormat   string
+	configFile  string
+	mainViper   = viper.New()
+	doBootstrap bool
 )
 
 func init() {
@@ -54,6 +57,7 @@ func init() {
 	flag.StringVar(&configFile, "config", "", "Path to configuration file")
 	flag.BoolVar(&debug, "debug", false, "show the version and exit")
 	flag.StringVar(&logFormat, "logf", "json", "Set the format of the logs. Only 'json' and 'logfmt' are supported at the moment.")
+	flag.BoolVar(&doBootstrap, "bootstrap", false, "Do any necessary setup of configured backend storage services")
 
 	// this needs to be in init so that _ALL_ logs are in the correct format
 	if logFormat == jsonLogFormat {
@@ -100,9 +104,9 @@ func setUpCryptoservices(configuration *viper.Viper, allowedBackends []string) (
 				return nil, fmt.Errorf("must provide a default alias for the key DB")
 			}
 			logrus.Debug("Default Alias: ", defaultAlias)
-			s, err := keydbstore.NewKeyRethinkDBStore(passphraseRetriever, defaultAlias, sess)
+			s := keydbstore.NewRethinkDBKeyStore(passphraseRetriever, defaultAlias, sess)
 			keyStore = s
-			hRegister("DB operational", s.CheckHealth, time.Second*60)
+			health.RegisterPeriodicFunc("DB operational", s.CheckHealth, time.Minute)
 		}
 
 	case notary.MySQLBackend, notary.SQLiteBackend:
@@ -128,10 +132,18 @@ func setUpCryptoservices(configuration *viper.Viper, allowedBackends []string) (
 		}
 
 		health.RegisterPeriodicFunc(
-			"DB operational", dbStore.HealthCheck, time.Second*60)
+			"DB operational", dbStore.HealthCheck, time.Minute)
 		keyStore = dbStore
 	}
 
+	if doBootstrap {
+		err := bootstrap(keyStore)
+		if err != nil {
+			logrus.Fatal(err.Error())
+		}
+		os.Exit(0)
+	}
+
 	cryptoService := cryptoservice.NewCryptoService(keyStore)
 	cryptoServices := make(signer.CryptoServiceIndex)
 	cryptoServices[data.ED25519Key] = cryptoService

cmd/notary-signer/main_test.go

@@ -12,7 +12,6 @@ import (
 	"github.com/docker/notary/signer"
 	"github.com/docker/notary/signer/keydbstore"
 	"github.com/docker/notary/tuf/data"
-	"github.com/docker/notary/utils"
 	"github.com/jinzhu/gorm"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/spf13/viper"
@@ -106,8 +105,8 @@ func TestSetupCryptoServicesDBStoreNoDefaultAlias(t *testing.T) {
 	_, err = setUpCryptoservices(
 		configure(fmt.Sprintf(
 			`{"storage": {"backend": "%s", "db_url": "%s"}}`,
-			notary.SqliteBackend, tmpFile.Name())),
-		[]string{notary.SqliteBackend})
+			notary.SQLiteBackend, tmpFile.Name())),
+		[]string{notary.SQLiteBackend})
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "must provide a default alias for the key DB")
 }
@@ -137,8 +136,8 @@ func TestSetupCryptoServicesDBStoreSuccess(t *testing.T) {
 		configure(fmt.Sprintf(
 			`{"storage": {"backend": "%s", "db_url": "%s"},
 			"default_alias": "timestamp"}`,
-			notary.SqliteBackend, tmpFile.Name())),
-		[]string{notary.SqliteBackend})
+			notary.SQLiteBackend, tmpFile.Name())),
+		[]string{notary.SQLiteBackend})
 	require.NoError(t, err)
 	require.Len(t, cryptoServices, 2)
 
@@ -167,9 +166,9 @@ func TestSetupCryptoServicesMemoryStore(t *testing.T) {
 	config := configure(fmt.Sprintf(`{"storage": {"backend": "%s"}}`,
 		notary.MemoryBackend))
 	cryptoServices, err := setUpCryptoservices(config,
-		[]string{notary.SqliteBackend, notary.MemoryBackend})
+		[]string{notary.SQLiteBackend, notary.MemoryBackend})
 	require.NoError(t, err)
-	require.Len(t, cryptoServices, 2)
+	assert.Len(t, cryptoServices, 2)
 
 	edService, ok := cryptoServices[data.ED25519Key]
 	require.True(t, ok)

server/storage/rethinkdb.go

@@ -57,11 +57,11 @@ func NewRethinkDBStorage(dbName string, sess *gorethink.Session) RethinkDB {
 // If the GUN+role don't exist, returns an error.
 func (rdb RethinkDB) GetKey(gun, role string) (cipher string, public []byte, err error) {
 	var key RDBKey
-	res, err := gorethink.DB(rdb.dbName).Table(key.TableName()).Get(
-		RDBKey{
-			Gun:  gun,
-			Role: role,
-		}).Run(rdb.sess)
+	res, err := gorethink.DB(rdb.dbName).Table(key.TableName()).GetAllByIndex(
+		rdbGunRoleIdx,
+		gun,
+		role,
+	).Run(rdb.sess)
 	if err != nil {
 		return "", nil, err
 	}
@@ -137,15 +137,18 @@ func (rdb RethinkDB) UpdateMany(gun string, updates []MetaUpdate) error {
 // the given GUN and role, an error is returned.
 func (rdb RethinkDB) GetCurrent(gun, role string) (created *time.Time, data []byte, err error) {
 	file := RDBTUFFile{}
-	res, err := gorethink.DB(rdb.dbName).Table(file.TableName()).Get(
-		RDBTUFFile{
-			Gun:  gun,
-			Role: role,
-		}).Run(rdb.sess)
+	res, err := gorethink.DB(rdb.dbName).Table(file.TableName()).GetAllByIndex(
+		rdbGunRoleIdx,
+		gun,
+		role,
+	).OrderBy(gorethink.Desc("version")).Run(rdb.sess)
 	if err != nil {
 		return nil, nil, err
 	}
 	defer res.Close()
+	if res.IsNil() {
+		return nil, nil, ErrNotFound{}
+	}
 	err = res.One(&file)
 	return &file.CreatedAt, file.Data, err
 }
@@ -155,16 +158,19 @@ func (rdb RethinkDB) GetCurrent(gun, role string) (created *time.Time, data []by
 // not found, it returns storage.ErrNotFound
 func (rdb RethinkDB) GetChecksum(gun, role, checksum string) (created *time.Time, data []byte, err error) {
 	var file RDBTUFFile
-	res, err := gorethink.DB(rdb.dbName).Table(file.TableName()).Get(
-		RDBTUFFile{
-			Gun:    gun,
-			Role:   role,
-			Sha256: checksum,
-		}).Run(rdb.sess)
+	res, err := gorethink.DB(rdb.dbName).Table(file.TableName()).GetAllByIndex(
+		rdbGunRoleSha256Idx,
+		gun,
+		role,
+		checksum,
+	).Run(rdb.sess)
 	if err != nil {
 		return nil, nil, err
 	}
 	defer res.Close()
+	if res.IsNil() {
+		return nil, nil, ErrNotFound{}
+	}
 	err = res.One(&file)
 	return &file.CreatedAt, file.Data, err
 }
@@ -172,8 +178,10 @@ func (rdb RethinkDB) GetChecksum(gun, role, checksum string) (created *time.Time
 // Delete removes all metadata for a given GUN.  It does not return an
 // error if no metadata exists for the given GUN.
 func (rdb RethinkDB) Delete(gun string) error {
-	files := RDBTUFFile{Gun: gun}
-	_, err := gorethink.DB(rdb.dbName).Table(files.TableName()).Get(files).Delete().RunWrite(rdb.sess)
+	_, err := gorethink.DB(rdb.dbName).Table(RDBTUFFile{}.TableName()).GetAllByIndex(
+		"gun",
+		gun,
+	).Delete().RunWrite(rdb.sess)
 	if err != nil {
 		return fmt.Errorf("unable to delete %s from database: %s", gun, err.Error())
 	}

server/storage/rethinkdb_models.go

@@ -4,18 +4,30 @@ import (
 	"github.com/docker/notary/storage/rethinkdb"
 )
 
+// These consts are the index names we've defined for RethinkDB
+const (
+	rdbSha256Idx        = "sha256"
+	rdbGunRoleIdx       = "gun_role"
+	rdbGunRoleSha256Idx = "gun_role_sha256"
+)
+
 var (
 	tufFiles = rethinkdb.Table{
 		Name:       RDBTUFFile{}.TableName(),
 		PrimaryKey: []string{"gun", "role", "version"},
 		SecondaryIndexes: map[string][]string{
-			"sha256": nil,
+			rdbSha256Idx:        nil,
+			"gun":               nil,
+			rdbGunRoleIdx:       {"gun", "role"},
+			rdbGunRoleSha256Idx: {"gun", "role", "sha256"},
 		},
 	}
 
 	keys = rethinkdb.Table{
-		Name:             RDBKey{}.TableName(),
-		PrimaryKey:       "id",
-		SecondaryIndexes: nil,
+		Name:       RDBKey{}.TableName(),
+		PrimaryKey: "id",
+		SecondaryIndexes: map[string][]string{
+			rdbGunRoleIdx: {"gun", "role"},
+		},
 	}
 )

signer/keydbstore/rethink_keydbstore.go

@@ -14,17 +14,17 @@ import (
 	jose "github.com/dvsekhvalnov/jose2go"
 )
 
-// KeyRethinkDBStore persists and manages private keys on a RethinkDB database
-type KeyRethinkDBStore struct {
+// RethinkDBKeyStore persists and manages private keys on a RethinkDB database
+type RethinkDBKeyStore struct {
 	sync.Mutex
-	session          *gorethink.Session
+	sess             *gorethink.Session
 	defaultPassAlias string
 	retriever        passphrase.Retriever
 	cachedKeys       map[string]data.PrivateKey
 }
 
-// RethinkPrivateKey represents a PrivateKey in the rethink database
-type RethinkPrivateKey struct {
+// RDBPrivateKey represents a PrivateKey in the rethink database
+type RDBPrivateKey struct {
 	rethinkdb.Timing
 	KeyID           string `gorethink:"key_id"`
 	EncryptionAlg   string `gorethink:"encryption_alg"`
@@ -35,22 +35,17 @@ type RethinkPrivateKey struct {
 	Private         string `gorethink:"private"`
 }
 
-// TableName sets a specific table name for our RethinkPrivateKey
-func (g RethinkPrivateKey) TableName() string {
+// TableName sets a specific table name for our RDBPrivateKey
+func (g RDBPrivateKey) TableName() string {
 	return "private_keys"
 }
 
-// DatabaseName sets a specific table name for our RethinkPrivateKey
-func (g RethinkPrivateKey) DatabaseName() string {
-	return "notarysigner"
-}
-
-// NewKeyRethinkDBStore returns a new KeyRethinkDBStore backed by a RethinkDB database
-func NewKeyRethinkDBStore(passphraseRetriever passphrase.Retriever, defaultPassAlias string, rethinkSession *gorethink.Session) *KeyRethinkDBStore {
+// NewRethinkDBKeyStore returns a new RethinkDBKeyStore backed by a RethinkDB database
+func NewRethinkDBKeyStore(passphraseRetriever passphrase.Retriever, defaultPassAlias string, rethinkSession *gorethink.Session) *RethinkDBKeyStore {
 	cachedKeys := make(map[string]data.PrivateKey)
 
-	return &KeyRethinkDBStore{
-		session:          rethinkSession,
+	return &RethinkDBKeyStore{
+		sess:             rethinkSession,
 		defaultPassAlias: defaultPassAlias,
 		retriever:        passphraseRetriever,
 		cachedKeys:       cachedKeys,
@@ -58,15 +53,19 @@ func NewKeyRethinkDBStore(passphraseRetriever passphrase.Retriever, defaultPassA
 }
 
 // Name returns a user friendly name for the storage location
-func (s *KeyRethinkDBStore) Name() string {
+func (rdb *RethinkDBKeyStore) Name() string {
 	return "RethinkDB"
 }
 
+func (rdb RethinkDBKeyStore) dbName() string {
+	return "notarysigner"
+}
+
 // AddKey stores the contents of a private key. Both role and gun are ignored,
 // we always use Key IDs as name, and don't support aliases
-func (s *KeyRethinkDBStore) AddKey(keyInfo trustmanager.KeyInfo, privKey data.PrivateKey) error {
+func (rdb *RethinkDBKeyStore) AddKey(keyInfo trustmanager.KeyInfo, privKey data.PrivateKey) error {
 
-	passphrase, _, err := s.retriever(privKey.ID(), s.defaultPassAlias, false, 1)
+	passphrase, _, err := rdb.retriever(privKey.ID(), rdb.defaultPassAlias, false, 1)
 	if err != nil {
 		return err
 	}
@@ -77,7 +76,7 @@ func (s *KeyRethinkDBStore) AddKey(keyInfo trustmanager.KeyInfo, privKey data.Pr
 	}
 
 	now := time.Now()
-	rethinkPrivKey := RethinkPrivateKey{
+	rethinkPrivKey := RDBPrivateKey{
 		Timing: rethinkdb.Timing{
 			CreatedAt: now,
 			UpdatedAt: now,
@@ -85,37 +84,37 @@ func (s *KeyRethinkDBStore) AddKey(keyInfo trustmanager.KeyInfo, privKey data.Pr
 		KeyID:           privKey.ID(),
 		EncryptionAlg:   EncryptionAlg,
 		KeywrapAlg:      KeywrapAlg,
-		PassphraseAlias: s.defaultPassAlias,
+		PassphraseAlias: rdb.defaultPassAlias,
 		Algorithm:       privKey.Algorithm(),
 		Public:          string(privKey.Public()),
 		Private:         encryptedKey}
 
 	// Add encrypted private key to the database
-	_, err = gorethink.DB(rethinkPrivKey.DatabaseName()).Table(rethinkPrivKey.TableName()).Insert(rethinkPrivKey).RunWrite(s.session)
+	_, err = gorethink.DB(rdb.dbName()).Table(rethinkPrivKey.TableName()).Insert(rethinkPrivKey).RunWrite(rdb.sess)
 	if err != nil {
 		return fmt.Errorf("failed to add private key to database: %s", privKey.ID())
 	}
 
 	// Add the private key to our cache
-	s.Lock()
-	defer s.Unlock()
-	s.cachedKeys[privKey.ID()] = privKey
+	rdb.Lock()
+	defer rdb.Unlock()
+	rdb.cachedKeys[privKey.ID()] = privKey
 
 	return nil
 }
 
 // GetKey returns the PrivateKey given a KeyID
-func (s *KeyRethinkDBStore) GetKey(name string) (data.PrivateKey, string, error) {
-	s.Lock()
-	defer s.Unlock()
-	cachedKeyEntry, ok := s.cachedKeys[name]
+func (rdb *RethinkDBKeyStore) GetKey(name string) (data.PrivateKey, string, error) {
+	rdb.Lock()
+	defer rdb.Unlock()
+	cachedKeyEntry, ok := rdb.cachedKeys[name]
 	if ok {
 		return cachedKeyEntry, "", nil
 	}
 
 	// Retrieve the RethinkDB private key from the database
-	dbPrivateKey := RethinkPrivateKey{}
-	res, err := gorethink.DB(dbPrivateKey.DatabaseName()).Table(dbPrivateKey.TableName()).Get(RethinkPrivateKey{KeyID: name}).Run(s.session)
+	dbPrivateKey := RDBPrivateKey{}
+	res, err := gorethink.DB(rdb.dbName()).Table(dbPrivateKey.TableName()).Get(RDBPrivateKey{KeyID: name}).Run(rdb.sess)
 	if err != nil {
 		return nil, "", trustmanager.ErrKeyNotFound{}
 	}
@@ -127,7 +126,7 @@ func (s *KeyRethinkDBStore) GetKey(name string) (data.PrivateKey, string, error)
 	}
 
 	// Get the passphrase to use for this key
-	passphrase, _, err := s.retriever(dbPrivateKey.KeyID, dbPrivateKey.PassphraseAlias, false, 1)
+	passphrase, _, err := rdb.retriever(dbPrivateKey.KeyID, dbPrivateKey.PassphraseAlias, false, 1)
 	if err != nil {
 		return nil, "", err
 	}
@@ -146,31 +145,31 @@ func (s *KeyRethinkDBStore) GetKey(name string) (data.PrivateKey, string, error)
 	}
 
 	// Add the key to cache
-	s.cachedKeys[privKey.ID()] = privKey
+	rdb.cachedKeys[privKey.ID()] = privKey
 
 	return privKey, "", nil
 }
 
 // GetKeyInfo always returns empty and an error. This method is here to satisfy the KeyStore interface
-func (s *KeyRethinkDBStore) GetKeyInfo(name string) (trustmanager.KeyInfo, error) {
-	return trustmanager.KeyInfo{}, fmt.Errorf("GetKeyInfo currently not supported for KeyRethinkDBStore, as it does not track roles or GUNs")
+func (rdb RethinkDBKeyStore) GetKeyInfo(name string) (trustmanager.KeyInfo, error) {
+	return trustmanager.KeyInfo{}, fmt.Errorf("GetKeyInfo currently not supported for RethinkDBKeyStore, as it does not track roles or GUNs")
 }
 
 // ListKeys always returns nil. This method is here to satisfy the KeyStore interface
-func (s *KeyRethinkDBStore) ListKeys() map[string]trustmanager.KeyInfo {
+func (rdb RethinkDBKeyStore) ListKeys() map[string]trustmanager.KeyInfo {
 	return nil
 }
 
 // RemoveKey removes the key from the table
-func (s *KeyRethinkDBStore) RemoveKey(keyID string) error {
-	s.Lock()
-	defer s.Unlock()
+func (rdb RethinkDBKeyStore) RemoveKey(keyID string) error {
+	rdb.Lock()
+	defer rdb.Unlock()
 
-	delete(s.cachedKeys, keyID)
+	delete(rdb.cachedKeys, keyID)
 
 	// Delete the key from the database
-	dbPrivateKey := RethinkPrivateKey{KeyID: keyID}
-	_, err := gorethink.DB(dbPrivateKey.DatabaseName()).Table(dbPrivateKey.TableName()).Get(dbPrivateKey).Delete().RunWrite(s.session)
+	dbPrivateKey := RDBPrivateKey{KeyID: keyID}
+	_, err := gorethink.DB(rdb.dbName()).Table(dbPrivateKey.TableName()).Get(dbPrivateKey).Delete().RunWrite(rdb.sess)
 	if err != nil {
 		return fmt.Errorf("unable to delete private key from database: %s", err.Error())
 	}
@@ -179,10 +178,10 @@ func (s *KeyRethinkDBStore) RemoveKey(keyID string) error {
 }
 
 // RotateKeyPassphrase rotates the key-encryption-key
-func (s *KeyRethinkDBStore) RotateKeyPassphrase(name, newPassphraseAlias string) error {
+func (rdb RethinkDBKeyStore) RotateKeyPassphrase(name, newPassphraseAlias string) error {
 	// Retrieve the RethinkDB private key from the database
-	dbPrivateKey := RethinkPrivateKey{KeyID: name}
-	res, err := gorethink.DB(dbPrivateKey.DatabaseName()).Table(dbPrivateKey.TableName()).Get(dbPrivateKey).Run(s.session)
+	dbPrivateKey := RDBPrivateKey{KeyID: name}
+	res, err := gorethink.DB(rdb.dbName()).Table(dbPrivateKey.TableName()).Get(dbPrivateKey).Run(rdb.sess)
 	if err != nil {
 		return trustmanager.ErrKeyNotFound{}
 	}
@@ -194,7 +193,7 @@ func (s *KeyRethinkDBStore) RotateKeyPassphrase(name, newPassphraseAlias string)
 	}
 
 	// Get the current passphrase to use for this key
-	passphrase, _, err := s.retriever(dbPrivateKey.KeyID, dbPrivateKey.PassphraseAlias, false, 1)
+	passphrase, _, err := rdb.retriever(dbPrivateKey.KeyID, dbPrivateKey.PassphraseAlias, false, 1)
 	if err != nil {
 		return err
 	}
@@ -206,7 +205,7 @@ func (s *KeyRethinkDBStore) RotateKeyPassphrase(name, newPassphraseAlias string)
 	}
 
 	// Get the new passphrase to use for this key
-	newPassphrase, _, err := s.retriever(dbPrivateKey.KeyID, newPassphraseAlias, false, 1)
+	newPassphrase, _, err := rdb.retriever(dbPrivateKey.KeyID, newPassphraseAlias, false, 1)
 	if err != nil {
 		return err
 	}
@@ -220,7 +219,7 @@ func (s *KeyRethinkDBStore) RotateKeyPassphrase(name, newPassphraseAlias string)
 	// Update the database object
 	dbPrivateKey.Private = newEncryptedKey
 	dbPrivateKey.PassphraseAlias = newPassphraseAlias
-	if _, err := gorethink.DB(dbPrivateKey.DatabaseName()).Table(dbPrivateKey.TableName()).Get(RethinkPrivateKey{KeyID: name}).Update(dbPrivateKey).RunWrite(s.session); err != nil {
+	if _, err := gorethink.DB(rdb.dbName()).Table(dbPrivateKey.TableName()).Get(RDBPrivateKey{KeyID: name}).Update(dbPrivateKey).RunWrite(rdb.sess); err != nil {
 		return err
 	}
 
@@ -228,15 +227,22 @@ func (s *KeyRethinkDBStore) RotateKeyPassphrase(name, newPassphraseAlias string)
 }
 
 // ExportKey is currently unimplemented and will always return an error
-func (s *KeyRethinkDBStore) ExportKey(keyID string) ([]byte, error) {
-	return nil, errors.New("Exporting from a KeyRethinkDBStore is not supported.")
+func (rdb RethinkDBKeyStore) ExportKey(keyID string) ([]byte, error) {
+	return nil, errors.New("Exporting from a RethinkDBKeyStore is not supported.")
+}
+
+// Bootstrap sets up the database and tables
+func (rdb RethinkDBKeyStore) Bootstrap() error {
+	return rethinkdb.SetupDB(rdb.sess, rdb.dbName(), []rethinkdb.Table{
+		privateKeys,
+	})
 }
 
 // HealthCheck verifies that DB exists and is query-able
-func (s *KeyRethinkDBStore) HealthCheck() error {
+func (rdb RethinkDBKeyStore) CheckHealth() error {
 	var tableOk bool
-	dbPrivateKey := RethinkPrivateKey{}
-	res, err := gorethink.DB(dbPrivateKey.DatabaseName()).TableList().Contains(dbPrivateKey.TableName()).Run(s.session)
+	dbPrivateKey := RDBPrivateKey{}
+	res, err := gorethink.DB(rdb.dbName()).TableList().Contains(dbPrivateKey.TableName()).Run(rdb.sess)
 	if err != nil {
 		return err
 	}