diff --git a/Makefile b/Makefile
index 6516d5d4a4..745789ef39 100644
--- a/Makefile
+++ b/Makefile
@@ -15,6 +15,7 @@ GO_LDFLAGS=-ldflags "-w $(CTIMEVAR)"
 GO_LDFLAGS_STATIC=-ldflags "-w $(CTIMEVAR) -extldflags -static"
 GOOSES = darwin freebsd linux
 GOARCHS = amd64
+NOTARY_BUILDFLAGS="pkcs11"
 GO_EXC = go
 NOTARYDIR := /go/src/github.com/docker/notary
 
@@ -49,15 +50,15 @@ version/version.go:
 
 ${PREFIX}/bin/notary-server: NOTARY_VERSION $(shell find . -type f -name '*.go')
 	@echo "+ $@"
-	@godep go build -o $@ ${GO_LDFLAGS} ./cmd/notary-server
+	@godep go build -tags ${NOTARY_BUILDFLAGS} -o $@ ${GO_LDFLAGS} ./cmd/notary-server
 
 ${PREFIX}/bin/notary: NOTARY_VERSION $(shell find . -type f -name '*.go')
 	@echo "+ $@"
-	@godep go build -o $@ ${GO_LDFLAGS} ./cmd/notary
+	@godep go build -tags ${NOTARY_BUILDFLAGS} -o $@ ${GO_LDFLAGS} ./cmd/notary
 
 ${PREFIX}/bin/notary-signer: NOTARY_VERSION $(shell find . -type f -name '*.go')
 	@echo "+ $@"
-	@godep go build -o $@ ${GO_LDFLAGS} ./cmd/notary-signer
+	@godep go build -tags ${NOTARY_BUILDFLAGS} -o $@ ${GO_LDFLAGS} ./cmd/notary-signer
 
 vet: go_version
 	@echo "+ $@"
@@ -73,16 +74,16 @@ lint:
 
 build: go_version
 	@echo "+ $@"
-	@go build -v ${GO_LDFLAGS} ./...
+	@go build -tags ${NOTARY_BUILDFLAGS} -v ${GO_LDFLAGS} ./...
 
 test: OPTS =
 test: go_version
 	@echo "+ $@ $(OPTS)"
-	go test $(OPTS) ./...
+	go test -tags ${NOTARY_BUILDFLAGS} $(OPTS) ./...
 
 test-full: vet lint
 	@echo "+ $@"
-	go test -v ./...
+	go test -tags ${NOTARY_BUILDFLAGS} -v ./...
 
 protos:
 	@protoc --go_out=plugins=grpc:. proto/*.proto
diff --git a/client/repo.go b/client/repo.go
index 4e7f2a406c..35820d09a1 100644
--- a/client/repo.go
+++ b/client/repo.go
@@ -3,12 +3,14 @@
 package client
 
 import (
+	"fmt"
 	"net/http"
 	"path/filepath"
 
 	"github.com/docker/notary/cryptoservice"
 	"github.com/docker/notary/keystoremanager"
 	"github.com/docker/notary/pkg/passphrase"
+	"github.com/docker/notary/trustmanager"
 	"github.com/docker/notary/tuf/store"
 )
 
@@ -16,9 +18,14 @@ import (
 // It takes the base directory under where all the trust files will be stored
 // (usually ~/.docker/trust/).
 func NewNotaryRepository(baseDir, gun, baseURL string, rt http.RoundTripper,
-	passphraseRetriever passphrase.Retriever) (*NotaryRepository, error) {
+	retriever passphrase.Retriever) (*NotaryRepository, error) {
+	keysPath := filepath.Join(baseDir, keystoremanager.PrivDir)
+	fileKeyStore, err := trustmanager.NewKeyFileStore(keysPath, retriever)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create private key store in directory: %s", keysPath)
+	}
 
-	keyStoreManager, err := keystoremanager.NewKeyStoreManager(baseDir, passphraseRetriever)
+	keyStoreManager, err := keystoremanager.NewKeyStoreManager(baseDir, fileKeyStore)
 	if err != nil {
 		return nil, err
 	}
diff --git a/client/repo_pkcs11.go b/client/repo_pkcs11.go
index ab59f39585..45c91cf0e2 100644
--- a/client/repo_pkcs11.go
+++ b/client/repo_pkcs11.go
@@ -3,6 +3,7 @@
 package client
 
 import (
+	"fmt"
 	"net/http"
 	"path/filepath"
 
@@ -10,6 +11,7 @@ import (
 	"github.com/docker/notary/keystoremanager"
 	"github.com/docker/notary/pkg/passphrase"
 	"github.com/docker/notary/signer/api"
+	"github.com/docker/notary/trustmanager"
 	"github.com/docker/notary/tuf/store"
 )
 
@@ -17,14 +19,16 @@ import (
 // It takes the base directory under where all the trust files will be stored
 // (usually ~/.docker/trust/).
 func NewNotaryRepository(baseDir, gun, baseURL string, rt http.RoundTripper,
-	passphraseRetriever passphrase.Retriever) (*NotaryRepository, error) {
+	retriever passphrase.Retriever) (*NotaryRepository, error) {
 
-	keyStoreManager, err := keystoremanager.NewKeyStoreManager(baseDir, passphraseRetriever)
+	keysPath := filepath.Join(baseDir, keystoremanager.PrivDir)
+	fileKeyStore, err := trustmanager.NewKeyFileStore(keysPath, retriever)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create private key store in directory: %s", keysPath)
 	}
 
-	yubiKeyStore := api.NewYubiKeyStore()
+	keyStoreManager, err := keystoremanager.NewKeyStoreManager(baseDir, fileKeyStore)
+	yubiKeyStore := api.NewYubiKeyStore(retriever)
 	cryptoService := cryptoservice.NewCryptoService(gun, yubiKeyStore, keyStoreManager.KeyStore)
 
 	nRepo := &NotaryRepository{