diff --git a/Dockerfile b/Dockerfile index 9830ddaf7b..1da7f852ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,10 +8,14 @@ RUN apt-get update && apt-get install -y \ patch \ tar \ xz-utils \ + python \ + python-pip \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* -RUN go get golang.org/x/tools/cmd/cover +RUN useradd -ms /bin/bash notary \ + && pip install codecov \ + && go get golang.org/x/tools/cmd/cover github.com/golang/lint/golint # Configure the container for OSX cross compilation ENV OSX_SDK MacOSX10.11.sdk @@ -27,8 +31,7 @@ ENV PATH /osxcross/target/bin:$PATH ENV NOTARYDIR /go/src/github.com/docker/notary COPY . ${NOTARYDIR} - -ENV GOPATH ${NOTARYDIR}/Godeps/_workspace:$GOPATH +RUN chmod -R a+rw /go WORKDIR ${NOTARYDIR} diff --git a/Makefile b/Makefile index 743991b29b..c7f8c76117 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,8 @@ NOTARYDIR := /go/src/github.com/docker/notary GO_VERSION := $(shell go version | grep "1\.[6-9]\(\.[0-9]+\)*") # check to make sure we have the right version -ifeq ($(strip $(GO_VERSION)),) + +ifeq ($(strip $(GO_VERSION))$(SKIPENVCHECK),) $(error Bad Go version - please install Go >= 1.6) endif @@ -40,8 +41,6 @@ COVERPROFILE?=$(COVERDIR)/cover.out COVERMODE=count PKGS ?= $(shell go list -tags "${NOTARY_BUILDTAGS}" ./... | grep -v /vendor/ | tr '\n' ' ') -GO_VERSION = $(shell go version | awk '{print $$3}') - .PHONY: clean all fmt vet lint build test binaries cross cover docker-images notary-dockerfile .DELETE_ON_ERROR: cover .DEFAULT: default @@ -137,8 +136,9 @@ test-full: vet lint @echo go test -tags "${NOTARY_BUILDTAGS}" $(TESTOPTS) -v $(PKGS) +integration: TESTDB = mysql integration: - buildscripts/integrationtest.sh development.yml + buildscripts/integrationtest.sh development.$(TESTDB).yml protos: @protoc --go_out=plugins=grpc:. proto/*.proto diff --git a/buildscripts/circle_parallelism.sh b/buildscripts/circle_parallelism.sh new file mode 100755 index 0000000000..3f4bcbbeff --- /dev/null +++ b/buildscripts/circle_parallelism.sh @@ -0,0 +1,14 @@ + #!/usr/bin/env bash + + case $CIRCLE_NODE_INDEX in + 0) docker run --rm -e NOTARY_BUILDTAGS=pkcs11 notary_client make vet lint fmt misspell + docker run --rm -e NOTARY_BUILDTAGS=pkcs11 --env-file buildscripts/env.list --user notary notary_client bash -c "make ci && codecov" + ;; + 1) docker run --rm -e NOTARY_BUILDTAGS=none notary_client make vet lint fmt misspell + docker run --rm -e NOTARY_BUILDTAGS=none --env-file buildscripts/env.list --user notary notary_client bash -c "make ci && codecov" + ;; + 2) SKIPENVCHECK=1 make TESTDB=mysql integration + ;; + 3) SKIPENVCHECK=1 make TESTDB=rethink integration + ;; + esac diff --git a/buildscripts/env.list b/buildscripts/env.list new file mode 100644 index 0000000000..ee8fcbe7a2 --- /dev/null +++ b/buildscripts/env.list @@ -0,0 +1,40 @@ +# These are codecov environment variables to pass through +CODECOV_TOKEN +CODECOV_ENV + +CI + +# These are the CircleCI environment variables to pass through for codecov +CIRCLECI +CIRCLE_BRANCH +CIRCLE_BUILD_NUM +CIRCLE_NODE_INDEX +CIRCLE_BUILD_NUM +CIRCLE_NODE_INDEX +CIRCLE_PR_NUMBER +CIRCLE_PROJECT_USERNAME +CIRCLE_PROJECT_REPONAME +CIRCLE_SHA1 + +# These are the Jenkins environment variables to pass through for codecov +JENKINS_URL +ghprbSourceBranch +GIT_BRANCH +ghprbActualCommit +GIT_COMMIT +ghprbPullId +BUILD_NUMBER +BUILD_URL +WORKSPACE + +# These are the Travis environment variables to pass through for codecov +# http://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables +# TRAVIS +# TRAVIS_BRANCH +# TRAVIS_JOB_NUMBER +# TRAVIS_PULL_REQUEST +# TRAVIS_JOB_ID +# TRAVIS_TAG +# TRAVIS_REPO_SLUG +# TRAVIS_COMMIT +# TRAVIS_BUILD_DIR diff --git a/buildscripts/integrationtest.sh b/buildscripts/integrationtest.sh index 216464e58c..6b6e8d8400 100755 --- a/buildscripts/integrationtest.sh +++ b/buildscripts/integrationtest.sh @@ -3,10 +3,11 @@ composeFile="$1" function cleanup { - docker-compose -f $composeFile stop + rm -f bin/notary + docker-compose -f $composeFile kill # if we're in CircleCI, we cannot remove any containers if [[ -z "${CIRCLECI}" ]]; then - docker-compose -f $composeFile rm -f + docker-compose -f $composeFile down -v --remove-orphans fi } diff --git a/buildscripts/testclient.sh b/buildscripts/testclient.sh index 10fae60af6..009277d3bc 100755 --- a/buildscripts/testclient.sh +++ b/buildscripts/testclient.sh @@ -2,6 +2,7 @@ set -e +make clean make client set +e diff --git a/circle.yml b/circle.yml index 1d2c993cb7..091ed3d301 100644 --- a/circle.yml +++ b/circle.yml @@ -1,87 +1,23 @@ -# Pony-up! machine: pre: - # Install gvm - - bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/1.0.22/binscripts/gvm-installer) # Upgrade docker - - sudo curl -L -o /usr/bin/docker 'https://s3-external-1.amazonaws.com/circle-downloads/docker-1.9.1-circleci' - - sudo chmod 0755 /usr/bin/docker - - post: - # Install many go versions - - gvm install go1.6.1 -B --name=stable + - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 # upgrade compose - sudo pip install --upgrade docker-compose services: - docker - environment: - # Convenient shortcuts to "common" locations - CHECKOUT: /home/ubuntu/$CIRCLE_PROJECT_REPONAME - BASE_DIR: src/github.com/docker/notary - # Trick circle brainflat "no absolute path" behavior - BASE_STABLE: ../../../$HOME/.gvm/pkgsets/stable/global/$BASE_DIR - # Workaround Circle parsing dumb bugs and/or YAML wonkyness - CIRCLE_PAIN: "mode: set" - # Put the coverage profile somewhere codecov's script can find it - COVERPROFILE: coverage.out - - hosts: - # Not used yet - fancy: 127.0.0.1 - dependencies: - pre: - # Copy the code to the gopath of all go versions - - > - gvm use stable && - mkdir -p "$(dirname $BASE_STABLE)" && - cp -R "$CHECKOUT" "$BASE_STABLE" - override: - # don't use circleci's default dependency installation step of `go get -d -u ./...` - # since we already vendor everything; additionally install linting and misspell tools - - > - gvm use stable && - go get github.com/golang/lint/golint && - go get -u github.com/client9/misspell/cmd/misspell + - docker build -t notary_client . test: - pre: - # Output the go versions we are going to test - - gvm use stable && go version - - # CLEAN - - gvm use stable && make clean: - pwd: $BASE_STABLE - - # FMT - - gvm use stable && make fmt: - pwd: $BASE_STABLE - - # VET - - gvm use stable && make vet: - pwd: $BASE_STABLE - - # LINT - - gvm use stable && make lint: - pwd: $BASE_STABLE - - # MISSPELL - - gvm use stable && make misspell: - pwd: $BASE_STABLE - override: - # Test stable, and report - # hacking this to be parallel - - case $CIRCLE_NODE_INDEX in 0) gvm use stable && NOTARY_BUILDTAGS=pkcs11 make ci ;; 1) gvm use stable && NOTARY_BUILDTAGS=none make ci ;; 2) gvm use stable && make integration ;; esac: + # circleci only supports manual parellism + - buildscripts/circle_parallelism.sh: parallel: true timeout: 600 - pwd: $BASE_STABLE - post: - # Report to codecov.io - - case $CIRCLE_NODE_INDEX in 0) bash <(curl -s https://codecov.io/bash) ;; 1) bash <(curl -s https://codecov.io/bash) ;; esac: - parallel: true - pwd: $BASE_STABLE + - docker-compose -f docker-compose.yml down -v + - docker-compose -f docker-compose.rethink.yml down -v diff --git a/development.yml b/development.mysql.yml similarity index 100% rename from development.yml rename to development.mysql.yml diff --git a/docker-compose.rethink.yml b/docker-compose.rethink.yml index 8b99b5a4a2..f47108eae5 100644 --- a/docker-compose.rethink.yml +++ b/docker-compose.rethink.yml @@ -52,7 +52,7 @@ services: rdb: aliases: - rdb-02.rdb - command: "--bind all --no-http-admin --server-name rdb_02 --canonical-address rdb-02.rdb --directory /var/data/rethinkdb --join rdb-01:29015 --driver-tls-ca /tls/ca.pem --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem --cluster-tls-key /tls/key.pem --cluster-tls-cert /tls/cert.pem --cluster-tls-ca /tls/ca.pem" + command: "--bind all --no-http-admin --server-name rdb_02 --canonical-address rdb-02.rdb --directory /var/data/rethinkdb --join rdb-01 --driver-tls-ca /tls/ca.pem --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem --cluster-tls-key /tls/key.pem --cluster-tls-cert /tls/cert.pem --cluster-tls-ca /tls/ca.pem" depends_on: - rdb-01 rdb-03: @@ -64,7 +64,7 @@ services: rdb: aliases: - rdb-03.rdb - command: "--bind all --no-http-admin --server-name rdb_03 --canonical-address rdb-03.rdb --directory /var/data/rethinkdb --join rdb-02:29015 --driver-tls-ca /tls/ca.pem --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem --cluster-tls-key /tls/key.pem --cluster-tls-cert /tls/cert.pem --cluster-tls-ca /tls/ca.pem" + command: "--bind all --no-http-admin --server-name rdb_03 --canonical-address rdb-03.rdb --directory /var/data/rethinkdb --join rdb-02 --driver-tls-ca /tls/ca.pem --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem --cluster-tls-key /tls/key.pem --cluster-tls-cert /tls/cert.pem --cluster-tls-ca /tls/ca.pem" depends_on: - rdb-01 - rdb-02 @@ -79,7 +79,7 @@ services: aliases: - rdb-proxy - rdb-proxy.rdp - command: "proxy --bind all --join rdb-03:29015 --driver-tls-ca /tls/ca.pem --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem --cluster-tls-key /tls/key.pem --cluster-tls-cert /tls/cert.pem --cluster-tls-ca /tls/ca.pem" + command: "proxy --bind all --join rdb-03 --driver-tls-ca /tls/ca.pem --driver-tls-key /tls/key.pem --driver-tls-cert /tls/cert.pem --cluster-tls-key /tls/key.pem --cluster-tls-cert /tls/cert.pem --cluster-tls-ca /tls/ca.pem" depends_on: - rdb-01 - rdb-02 diff --git a/docker-compose.yml b/docker-compose.yml index e94e420cd7..cc4923b8bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,8 +22,6 @@ mysql: - ./notarymysql/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d - notary_data:/var/lib/mysql image: mariadb:10.1.10 - ports: - - "3306:3306" environment: - TERM=dumb - MYSQL_ALLOW_EMPTY_PASSWORD="true"