From 0f6486ec7fdc92b1c77d4d4701641a52ce941792 Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Wed, 27 Jun 2018 15:38:45 +0100 Subject: [PATCH 1/9] Initial commit * Add renewal timer * Install libaugeas0 in python-augeas part build This part needs libaugeas0 to build. * Bump to 0.26.1 * Always act directly on upstream master I want to keep this always working, so move to master. We can reintroduce upstream stable releases when we are ready for general use. Closes: #5 That particular issue seems to no longer happen. Presumably something changed in upstream git or in PyPI. If it happens again, hopefully I'll have CI against upstream master up by then and I'll be able to pin it down. * Add empty Travis build * Add Travis automatic snap edge publication * Add integration test This uses upstream's test suite from their source tree to check the built snap to make sure it behaves as expected, before attempting upload to the store. * Point Augeas to its lens library Augeas defaults to looking in /usr/share/augeas/lenses, which in a snap isn't found at this path, but inside $SNAP. So set AUGEAS_LENS_LIB to where the lenses can be found within the snap. This fixes the Apache plugin that uses Augeas. --- snap/.snapcraft/travis_snapcraft.cfg | Bin 0 -> 2448 bytes snap/.travis.yml | 11 +++++ snap/snapcraft.yaml | 67 +++++++++++++++++++++++++++ snap/test.bash | 34 ++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 snap/.snapcraft/travis_snapcraft.cfg create mode 100644 snap/.travis.yml create mode 100644 snap/snapcraft.yaml create mode 100644 snap/test.bash diff --git a/snap/.snapcraft/travis_snapcraft.cfg b/snap/.snapcraft/travis_snapcraft.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d54f41155fe3ff5850721619d5e34d6b0ad85afc GIT binary patch literal 2448 zcmV;B32*ioqW&}!9P1blSSTry(okTVjYkP?5Wp~YL%*7glNCXMj}n<x)aFiR0p~b(~Lr-2a%WaDxNW za?T9WiJ>=#5)Tv?W^tOlDqEo4dg!o4Bsp-DcwCs0&#uf66A*9Je9ZncL}uaGpw%iI z2^X^wfNaDms2C;|Gsk-%Hy$nN9+sqSkSmk}rD&I;?a9O_C~o%wWi5e@C%g!8Ff0Bo z#hU>DS&>7I+i?1)LQHx?n5)hVWPMPL^=f_lwmisSuxt7`qG5#~$3Uk0B?_#3Gu|$y zA?h1^nZ=AQf>X3UNbKbly&DRX$~<*(t=e6sVrUKEwRLsOy0h~*>U?9KuOhThsG3Dk z(m}Qa?^ew_=*BVFBySco&{Qr>6mYKiP~W6aPn*$ibOLO*aW4Rb! zH1`)DqVjauEA#DjT0+#QNEN%!10i!VGh9)($2gq%!e+Ak$#MTRbA)JsJNS6 z!dQ7w;L61&A=I{~I3fsbl^4(<(EB6*R}rx{NPv66sFYV&2?zi-rY*Iti%UxnS%Wla zJWzamc&fCV0LSWNi6rm!32AazC_g@@tnoeO4;Uyi4**X&ni0 z!@|b)O0QMjb!?CV>7Br$X{<0f!fZw1WEQU%QKP9wy#PQmV=aWlb)XckcsZ!MKg9b_ zrgzTjPBNzx@IepXqUjwLL9f2INb(l@i+)obL=^nrbGLnA>xBv%{yO~dN9p)ukkLz= zt#h^pA;p(uCihtoFaj{S8U}H*{sV}>za$3iW4Zi>M`Q(C%v4r+GK-usI-d&3K+OSs z)!{?S*`>5QWVHV-KoT=LC!ycSAQ_cM)QtU3jwm=6fO`d;YD7q8Yo#EacGsgGwYlGh z|4FDD2m>FB3d;ptjq{g5Pxkw0=Wy+vR>`f17L@CBG5kdq~8Lo*>RcGMRt7#Q237y zD|mr7V-~-TBv-?Lty9i9Ro10M5=P{L4)D^%l?#m4m?<*+2W-ExQ3~Gq81sd66B^Dt z7o4{k%oP}2OasXZvGlIn8f`qzH% zs0gJIA@_`Tpsl+U$!twrGg*qk z`oNJOxdC$PCyDFz;hynHJkR#j>265mFUgsaXWAzgEn42N1AQ0o;Qm!b{gb5CInGAY zj8Tm5S-vo>ss@(q0%BZ9B_~6e-wh|(Bol-32pwUY)0hG=#(fh^>$L&btbk7t_wW0t zc6B^FgrN8AFj+}N!SYWDMTs`?m;W+Gc5*BYE&4JK2?2~Rakjy113&GLm2|t{jvC6G zqYB_{0A#v5@7_GbjBQ)KGhzTRFsUpa{NyhG^krSR%Z8edP}_AschHegr+w_A0=)GQ zQNgX?psx`(HhfK(J93l`&f49!HYE|;3Y+wq43WIJ(=8Ec_C8^b81a*9LeT< zTfIiq%Eh?|-ie{5ozKc>!@N%t@_cLih>6U2Hk-wCg}KlGwU8O^jP*nuOLdlal@;Z4 z6=o9ooLrc)uv4ilB-_x_6>W3-#uYK>cHXg|iNMl$f+5mKPOlpWGxC6SyAn6vl>M`8 zyb_WU4C}T%bZ|kh7Pu5lRqN{YICM|?O=)+WNe~&XMfRH4ORFNO0<^CoGnGh)A;2P% z=-@5~;f(g3ml!7N8#Kw2@GgJ7gbOI4#JIvLS0F@FIL|R|yvdl9XN=z3U%IrfbGRN& zSm$m$BnSb)pOEN?vSf%%Lxd7FDuWH_s8{^fvgglT+Nn&k^B)$Tf+5LzOG8*J(e{1> zcPX;+Uaai526@f8zA2bdclu)#_jDQ-Hv<;=+OTi9IpJiYEQ^|%zWprvV@sXeCgs*K zX$>pCL(*~A0CH{2Z$9kWyI{1ZB`}q<>kHnd0=5U=lHhe0HZ+Mvhb$zYAkqPcWYc4@ zO#sdX4!Xus6raVi#} z?7SbP&}-?}8_+?B;brcRR7$-svQ8Qe*pC!LJJ--7{M?z?id&iI+N95Ygq4rGaL}jI zzg@1O@CVI+QQD@oEnQE21@d=Aocw!K-9AL3qb_btf0BilGrk8ud_6$QmROJ|WCXx` zU+(RdGK|dkFgNsT(#n`8tvrkF?N>av0Dv`cl(ejja|i3_H1`oa_5J!En2hP6+YL!M zIBhzz;jj3PDtKgr+I^_Kc8W!2*+E_V)T*OBqww_Wid151=!vll3u)HxK(#&)f&YYYsC>SyoUOOF(0Z{V(qivF0#(W6oYjMvR6$S|`s9Jl zS%Ztzb~ouF4Q>Ow7Q3#JZiI&!FXfNmx(1kv!w OnZUwb{V>Dqu1Ux#JH%H2 literal 0 HcmV?d00001 diff --git a/snap/.travis.yml b/snap/.travis.yml new file mode 100644 index 000000000..603609b6d --- /dev/null +++ b/snap/.travis.yml @@ -0,0 +1,11 @@ +language: bash +script: docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && cd $(pwd) && snapcraft" && sudo apt-get -y install snapd && sudo snap install --dangerous --classic *.snap && sudo bash test.bash +sudo: required +services: +- docker +deploy: + 'on': + branch: master + provider: script + script: openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d && docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" + skip_cleanup: true diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 000000000..46425f182 --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,67 @@ +name: certbot +version: script +version-script: cd parts/certbot/src && git describe|sed s/^v// +summary: Automatically configure HTTPS using Let's Encrypt +description: | + The objective of Certbot, Let's Encrypt, and the ACME (Automated + Certificate Management Environment) protocol is to make it possible + to set up an HTTPS server and have it automatically obtain a + browser-trusted certificate, without any human intervention. This is + accomplished by running a certificate management agent on the web + server. + + This agent is used to: + - Automatically prove to the Let's Encrypt CA that you control the website + - Obtain a browser-trusted certificate and set it up on your web server + - Keep track of when your certificate is going to expire, and renew it + - Help you revoke the certificate if that ever becomes necessary. +confinement: classic +grade: devel + +apps: + certbot: + command: certbot + environment: + PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" + AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + renew: + command: certbot -q renew + daemon: oneshot + environment: + PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" + AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + passthrough: + # Run approximately twice a day with randomization + timer: 00:00~24:00/2 + +parts: + python-augeas: + plugin: python + source: git://github.com/basak/python-augeas + source-branch: snap + python-version: python2 + build-packages: [libaugeas0] + certbot: + plugin: python + source: git://github.com/certbot/certbot + source-branch: master + python-version: python2 + after: [python-augeas] + certbot-nginx: + plugin: python + source: git://github.com/certbot/certbot + source-branch: master + source-subdir: certbot-nginx + build-packages: [libaugeas-dev] + stage-packages: [libaugeas0] + after: [certbot] + python-version: python2 + certbot-apache: + plugin: python + source: git://github.com/certbot/certbot + source-branch: master + source-subdir: certbot-apache + build-packages: [libaugeas-dev] + stage-packages: [libaugeas0] + after: [certbot] + python-version: python2 diff --git a/snap/test.bash b/snap/test.bash new file mode 100644 index 000000000..e723b67bf --- /dev/null +++ b/snap/test.bash @@ -0,0 +1,34 @@ +#!/bin/bash + +set -ex + +apt-get -y install lsb-release net-tools wget python nginx + +wget https://github.com/docker/compose/releases/download/1.15.0-rc1/docker-compose-Linux-x86_64 -O /usr/local/bin/docker-compose +chmod +x /usr/local/bin/docker-compose + +cat << EOF > /usr/local/bin/coverage +#!/bin/bash -xe + +if [ "\$1" != "run" ]; then + exit 0; +fi + +"\${@:7}" +EOF +chmod +x /usr/local/bin/coverage + +certbot_version=$(certbot --version 2>&1 | grep "^certbot" | cut -d " " -f 2) + +cd parts/certbot/src + +tests/boulder-fetch.sh +until curl http://localhost:4000/directory 2>/dev/null; do + echo waiting for boulder + sleep 1 +done +# Not needed under Travis Trusty? +#sed -i "s/'1.3.6.1.5.5.7.1.24'/-e '1.3.6.1.5.5.7.1.24' -e 'status_request'/g" tests/certbot-boulder-integration.sh +tests/boulder-integration.sh + +echo "Success!" From d63be466a85ed4633a24844b43ce78857c524053 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 3 May 2019 10:15:48 +0200 Subject: [PATCH 2/9] Various optimizations part 1 * Configure for python3 * Update tests * Use appropriate virtualenv * Install nginx for the integration tests * Try use LD_LIBRARY_PATH to find augeas shared library in snap when python-augeas is invoked * Update travis to use build-in setup capabilities * Update .travis.yml * Add acme build * Update tests * Try more recent dist * Update command * Clean tests * Add back augeas * Add env * Revert to last working snapcraft config * Add a gitignore * Reintegrate acme. Declare augeas in certbot parts * Use release version of certbot * Try new approach * Fix config * Directly install version of python-augeas from pypi * Restart from basic * Clone only once certbot repository. Use pinned versions of dependencies from certbot-auto. * Try relatively to source * Use snapcraft env variables * Strip hashes * Fix path * Redefine path * Continue to prepare the runtime * Fix command line * Update .travis.yml * Add back certbot-apache * Update snapcraft.yaml * Build snap against the latest release of certbot --- snap/.gitignore | 8 ++++++++ snap/.travis.yml | 24 +++++++++++++++++++--- snap/get_latest_version.py | 21 +++++++++++++++++++ snap/snapcraft.yaml | 42 ++++++++++++++++++-------------------- snap/test.bash | 34 ------------------------------ snap/test.sh | 10 +++++++++ 6 files changed, 80 insertions(+), 59 deletions(-) create mode 100644 snap/.gitignore create mode 100644 snap/get_latest_version.py delete mode 100644 snap/test.bash create mode 100644 snap/test.sh diff --git a/snap/.gitignore b/snap/.gitignore new file mode 100644 index 000000000..2a339aeaa --- /dev/null +++ b/snap/.gitignore @@ -0,0 +1,8 @@ +.snapcraft +parts +prime +stage +*.snap +certbot +venv +.idea diff --git a/snap/.travis.yml b/snap/.travis.yml index 603609b6d..1abe677cd 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -1,11 +1,29 @@ +dist: xenial language: bash -script: docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && cd $(pwd) && snapcraft" && sudo apt-get -y install snapd && sudo snap install --dangerous --classic *.snap && sudo bash test.bash +install: + - EXPORT CERTBOT_VERSION=$(python3 get_latest_version.py) + - git clone https://github.com/certbot/certbot.git certbot --branch ${CERTBOT_VERSION} + - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt +script: + - docker run -v $(pwd):$(pwd) -t -e CERTBOT_VERSION=${CERTBOT_VERSION} snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" + - sudo snap install --dangerous --classic *.snap + - sudo bash test.sh sudo: required +addons: + apt: + packages: + - snapd + - nginx-light + - python3-venv + - python3-requests + - python3-packaging services: -- docker + - docker deploy: 'on': branch: master provider: script - script: openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d && docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" + script: | + openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d + docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" skip_cleanup: true diff --git a/snap/get_latest_version.py b/snap/get_latest_version.py new file mode 100644 index 000000000..b9a57e2aa --- /dev/null +++ b/snap/get_latest_version.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import sys + +import requests +from packaging import version + + +def main(): + result = requests.get('https://api.github.com/repos/certbot/certbot/tags') + result.raise_for_status() + + tags = [version.parse(entry['name'].replace('v', '')) for entry in result.json()] + tags.sort() + + latest_tag = 'v{0}'.format(tags[-1]) + + sys.stdout.write(latest_tag) + + +if __name__ == '__main__': + main() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 46425f182..0a3772ad6 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -2,6 +2,7 @@ name: certbot version: script version-script: cd parts/certbot/src && git describe|sed s/^v// summary: Automatically configure HTTPS using Let's Encrypt +base: core18 description: | The objective of Certbot, Let's Encrypt, and the ACME (Automated Certificate Management Environment) protocol is to make it possible @@ -23,45 +24,42 @@ apps: command: certbot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + AUGEAS_LENS_LIB: "$SNAP/usr/share/augeas/lenses/dist" + LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" renew: command: certbot -q renew daemon: oneshot environment: PATH: "$SNAP/bin:$SNAP/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" AUGEAS_LENS_LIB: $SNAP/usr/share/augeas/lenses/dist + LD_LIBRARY_PATH: "$SNAP/usr/lib/x86_64-linux-gnu/:$LD_LIBRARY_PATH" passthrough: # Run approximately twice a day with randomization timer: 00:00~24:00/2 parts: - python-augeas: + acme: plugin: python - source: git://github.com/basak/python-augeas - source-branch: snap - python-version: python2 - build-packages: [libaugeas0] + source: certbot + source-subdir: acme + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 certbot: plugin: python - source: git://github.com/certbot/certbot - source-branch: master - python-version: python2 - after: [python-augeas] - certbot-nginx: - plugin: python - source: git://github.com/certbot/certbot - source-branch: master - source-subdir: certbot-nginx - build-packages: [libaugeas-dev] - stage-packages: [libaugeas0] - after: [certbot] - python-version: python2 + source: certbot + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 + python-packages: + - acme + after: [acme] certbot-apache: plugin: python - source: git://github.com/certbot/certbot - source-branch: master + source: certbot source-subdir: certbot-apache + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 + python-packages: + - certbot build-packages: [libaugeas-dev] stage-packages: [libaugeas0] after: [certbot] - python-version: python2 diff --git a/snap/test.bash b/snap/test.bash deleted file mode 100644 index e723b67bf..000000000 --- a/snap/test.bash +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -ex - -apt-get -y install lsb-release net-tools wget python nginx - -wget https://github.com/docker/compose/releases/download/1.15.0-rc1/docker-compose-Linux-x86_64 -O /usr/local/bin/docker-compose -chmod +x /usr/local/bin/docker-compose - -cat << EOF > /usr/local/bin/coverage -#!/bin/bash -xe - -if [ "\$1" != "run" ]; then - exit 0; -fi - -"\${@:7}" -EOF -chmod +x /usr/local/bin/coverage - -certbot_version=$(certbot --version 2>&1 | grep "^certbot" | cut -d " " -f 2) - -cd parts/certbot/src - -tests/boulder-fetch.sh -until curl http://localhost:4000/directory 2>/dev/null; do - echo waiting for boulder - sleep 1 -done -# Not needed under Travis Trusty? -#sed -i "s/'1.3.6.1.5.5.7.1.24'/-e '1.3.6.1.5.5.7.1.24' -e 'status_request'/g" tests/certbot-boulder-integration.sh -tests/boulder-integration.sh - -echo "Success!" diff --git a/snap/test.sh b/snap/test.sh new file mode 100644 index 000000000..cf2418448 --- /dev/null +++ b/snap/test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +cd certbot + +python3 -m venv venv +. venv/bin/activate +pip install -e certbot-ci + +pytest certbot-ci/certbot_integration_tests/certbot_tests --numprocesses 4 --acme-server=pebble From e0b72d9a62db7260ff5114422f839dd80e40c5ae Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Tue, 20 Aug 2019 14:47:23 +0100 Subject: [PATCH 3/9] Travis improvements * Add Travis notifications * Adjust automatic snap deployment configuration Travis now has a documented[1] "snap" provider and the previous experimental mechanism seems to have stopped working, presumably because it was deprecated in favour of this new mechanism. [1] https://docs.travis-ci.com/user/deployment/snaps/ --- snap/.travis.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/snap/.travis.yml b/snap/.travis.yml index 1abe677cd..5e5b17037 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -22,8 +22,11 @@ services: deploy: 'on': branch: master - provider: script - script: | - openssl aes-256-cbc -K $encrypted_edc9d3b1405a_key -iv $encrypted_edc9d3b1405a_iv -in .snapcraft/travis_snapcraft.cfg -out .snapcraft/snapcraft.cfg -d - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "cd $(pwd) && snapcraft push *.snap --release edge" + provider: snap + snap: certbot_*.snap + channel: edge skip_cleanup: true +notifications: + email: + recipients: [robie.basak@canonical.com] + on_failure: change From 2c622944ddaa8c4641df33bb2386839a8d74e9ef Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Wed, 21 Aug 2019 23:45:04 +0200 Subject: [PATCH 4/9] Various optimizations part 2 * Revert logic of building against a tag * Fix schema, add nginx * Update snapcraft.yaml * Update snapcraft.yaml * Update snapcraft.yaml * Update test.sh * Update test.sh * Update test.sh * Update test.sh * Config tests * Add an apache test * Relaunch CI * Clean config * Install venv * Decompose steps * Update test.sh * Use virtual environment * Update python-augeas * Add fork python-augeas * Update .travis.yml * Exclusion rule * Try with after --- snap/.travis.yml | 23 +++++++---------------- snap/get_latest_version.py | 21 --------------------- snap/snapcraft.yaml | 19 +++++++++++++------ snap/test.sh | 11 ++++++----- 4 files changed, 26 insertions(+), 48 deletions(-) delete mode 100644 snap/get_latest_version.py diff --git a/snap/.travis.yml b/snap/.travis.yml index 5e5b17037..3c934b7bd 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -1,24 +1,15 @@ -dist: xenial -language: bash +language: python +before_install: + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light snapd python3-venv + - sudo -E /etc/init.d/nginx stop install: - - EXPORT CERTBOT_VERSION=$(python3 get_latest_version.py) - - git clone https://github.com/certbot/certbot.git certbot --branch ${CERTBOT_VERSION} + - git clone https://github.com/certbot/certbot.git certbot --branch master - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt +before_script: + - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" script: - - docker run -v $(pwd):$(pwd) -t -e CERTBOT_VERSION=${CERTBOT_VERSION} snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" - sudo snap install --dangerous --classic *.snap - sudo bash test.sh -sudo: required -addons: - apt: - packages: - - snapd - - nginx-light - - python3-venv - - python3-requests - - python3-packaging -services: - - docker deploy: 'on': branch: master diff --git a/snap/get_latest_version.py b/snap/get_latest_version.py deleted file mode 100644 index b9a57e2aa..000000000 --- a/snap/get_latest_version.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python3 -import sys - -import requests -from packaging import version - - -def main(): - result = requests.get('https://api.github.com/repos/certbot/certbot/tags') - result.raise_for_status() - - tags = [version.parse(entry['name'].replace('v', '')) for entry in result.json()] - tags.sort() - - latest_tag = 'v{0}'.format(tags[-1]) - - sys.stdout.write(latest_tag) - - -if __name__ == '__main__': - main() diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 0a3772ad6..b7bfab9ca 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -2,7 +2,6 @@ name: certbot version: script version-script: cd parts/certbot/src && git describe|sed s/^v// summary: Automatically configure HTTPS using Let's Encrypt -base: core18 description: | The objective of Certbot, Let's Encrypt, and the ACME (Automated Certificate Management Environment) protocol is to make it possible @@ -38,6 +37,12 @@ apps: timer: 00:00~24:00/2 parts: + python-augeas: + plugin: python + source: git://github.com/basak/python-augeas + source-branch: snap + python-version: python3 + build-packages: [libaugeas-dev] acme: plugin: python source: certbot @@ -49,8 +54,6 @@ parts: source: certbot constraints: $SNAPCRAFT_PART_SRC/constraints.txt python-version: python3 - python-packages: - - acme after: [acme] certbot-apache: plugin: python @@ -58,8 +61,12 @@ parts: source-subdir: certbot-apache constraints: $SNAPCRAFT_PART_SRC/constraints.txt python-version: python3 - python-packages: - - certbot - build-packages: [libaugeas-dev] + after: [python-augeas, certbot] stage-packages: [libaugeas0] + certbot-nginx: + plugin: python + source: certbot + source-subdir: certbot-nginx + constraints: $SNAPCRAFT_PART_SRC/constraints.txt + python-version: python3 after: [certbot] diff --git a/snap/test.sh b/snap/test.sh index cf2418448..e89743d47 100644 --- a/snap/test.sh +++ b/snap/test.sh @@ -1,10 +1,11 @@ #!/bin/bash set -ex -cd certbot - python3 -m venv venv -. venv/bin/activate -pip install -e certbot-ci +venv/bin/python -m pip install -e certbot/certbot-ci +venv/bin/python -m pytest certbot/certbot-ci/certbot_integration_tests -n 4 -pytest certbot-ci/certbot_integration_tests/certbot_tests --numprocesses 4 --acme-server=pebble +# DO NOT RUN `apache-conf-test` LOCALLY, IT WILL BREAK YOUR APACHE CONFIGURATION! +if [ -n "$TRAVIS" ]; then + venv/bin/python certbot/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules +fi From b69f5588f4c05d184a5d1e983ffa7898affa0e4d Mon Sep 17 00:00:00 2001 From: Robie Basak Date: Tue, 3 Sep 2019 15:41:11 +0100 Subject: [PATCH 5/9] Continued improvements * Remove legacy Store upload credentials These have not been needed since 5d7969a. * Work around dev part dependency failure case Get pip to install certbot from its VCS repository directly during the build of the nginx and apache plugin parts. This works around issue #12 when it affects the interaction between the apache or nginx plugin and certbot itself. It does not work around the case where the same problem occurs in the interaction between certbot and acme. This looks harder to work around because pip's VCS URL handling doesn't appear to include a facility to install from a subdirectory of a git repository and this is where the acme source is located. * Switch to using lxd for the snapcraft run The Docker image is 16.04 only. Before we can switch to 18.04, we need to remove Docker, which means using the snapcraft snap using Travis' new snap support, and then using the lxd functionality in snapcraft so that it is enabled to build in the appropriate environment. * Switch build to use core 18 Fixes: #14 * Move constraints into a list This seems to be a requirement either either newer snapcraft, snapcraft in lxd or in the move to core18 (it isn't clear to me which). This fixes the error: Failed to load plugin: properties failed to load for certbot: The 'constraints' property does not match the required schema: '$SNAPCRAFT_PART_SRC/constraints.txt' is not of type 'array' * version-script -> snapcraftctl set-version The use of version-script seems to break with either newer snapcraft, snapcraft with lxd or core18 (it's not clear to me which). The breakage is related to "parts/certbot/src" not being found. This can be fixed with $SNAPCRAFT_PART_SRC, but this doesn't seem to be defined at "version-script time". However https://snapcraft.io/docs/deprecation-notices/dn10 deprecates the use of version-script, and if we convert to the recommended new way, then we use override-pull instead and $SNAPCRAFT_PART_SRC is defined there, so this conveniently fixes both problems at once. * Do not explicitly install snapd Since this is now handled by the Travis addon, we do not need to do it explicitly. --- snap/.snapcraft/travis_snapcraft.cfg | Bin 2448 -> 0 bytes snap/.travis.yml | 14 ++++++++++++-- snap/snapcraft.yaml | 17 +++++++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) delete mode 100644 snap/.snapcraft/travis_snapcraft.cfg diff --git a/snap/.snapcraft/travis_snapcraft.cfg b/snap/.snapcraft/travis_snapcraft.cfg deleted file mode 100644 index d54f41155fe3ff5850721619d5e34d6b0ad85afc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2448 zcmV;B32*ioqW&}!9P1blSSTry(okTVjYkP?5Wp~YL%*7glNCXMj}n<x)aFiR0p~b(~Lr-2a%WaDxNW za?T9WiJ>=#5)Tv?W^tOlDqEo4dg!o4Bsp-DcwCs0&#uf66A*9Je9ZncL}uaGpw%iI z2^X^wfNaDms2C;|Gsk-%Hy$nN9+sqSkSmk}rD&I;?a9O_C~o%wWi5e@C%g!8Ff0Bo z#hU>DS&>7I+i?1)LQHx?n5)hVWPMPL^=f_lwmisSuxt7`qG5#~$3Uk0B?_#3Gu|$y zA?h1^nZ=AQf>X3UNbKbly&DRX$~<*(t=e6sVrUKEwRLsOy0h~*>U?9KuOhThsG3Dk z(m}Qa?^ew_=*BVFBySco&{Qr>6mYKiP~W6aPn*$ibOLO*aW4Rb! zH1`)DqVjauEA#DjT0+#QNEN%!10i!VGh9)($2gq%!e+Ak$#MTRbA)JsJNS6 z!dQ7w;L61&A=I{~I3fsbl^4(<(EB6*R}rx{NPv66sFYV&2?zi-rY*Iti%UxnS%Wla zJWzamc&fCV0LSWNi6rm!32AazC_g@@tnoeO4;Uyi4**X&ni0 z!@|b)O0QMjb!?CV>7Br$X{<0f!fZw1WEQU%QKP9wy#PQmV=aWlb)XckcsZ!MKg9b_ zrgzTjPBNzx@IepXqUjwLL9f2INb(l@i+)obL=^nrbGLnA>xBv%{yO~dN9p)ukkLz= zt#h^pA;p(uCihtoFaj{S8U}H*{sV}>za$3iW4Zi>M`Q(C%v4r+GK-usI-d&3K+OSs z)!{?S*`>5QWVHV-KoT=LC!ycSAQ_cM)QtU3jwm=6fO`d;YD7q8Yo#EacGsgGwYlGh z|4FDD2m>FB3d;ptjq{g5Pxkw0=Wy+vR>`f17L@CBG5kdq~8Lo*>RcGMRt7#Q237y zD|mr7V-~-TBv-?Lty9i9Ro10M5=P{L4)D^%l?#m4m?<*+2W-ExQ3~Gq81sd66B^Dt z7o4{k%oP}2OasXZvGlIn8f`qzH% zs0gJIA@_`Tpsl+U$!twrGg*qk z`oNJOxdC$PCyDFz;hynHJkR#j>265mFUgsaXWAzgEn42N1AQ0o;Qm!b{gb5CInGAY zj8Tm5S-vo>ss@(q0%BZ9B_~6e-wh|(Bol-32pwUY)0hG=#(fh^>$L&btbk7t_wW0t zc6B^FgrN8AFj+}N!SYWDMTs`?m;W+Gc5*BYE&4JK2?2~Rakjy113&GLm2|t{jvC6G zqYB_{0A#v5@7_GbjBQ)KGhzTRFsUpa{NyhG^krSR%Z8edP}_AschHegr+w_A0=)GQ zQNgX?psx`(HhfK(J93l`&f49!HYE|;3Y+wq43WIJ(=8Ec_C8^b81a*9LeT< zTfIiq%Eh?|-ie{5ozKc>!@N%t@_cLih>6U2Hk-wCg}KlGwU8O^jP*nuOLdlal@;Z4 z6=o9ooLrc)uv4ilB-_x_6>W3-#uYK>cHXg|iNMl$f+5mKPOlpWGxC6SyAn6vl>M`8 zyb_WU4C}T%bZ|kh7Pu5lRqN{YICM|?O=)+WNe~&XMfRH4ORFNO0<^CoGnGh)A;2P% z=-@5~;f(g3ml!7N8#Kw2@GgJ7gbOI4#JIvLS0F@FIL|R|yvdl9XN=z3U%IrfbGRN& zSm$m$BnSb)pOEN?vSf%%Lxd7FDuWH_s8{^fvgglT+Nn&k^B)$Tf+5LzOG8*J(e{1> zcPX;+Uaai526@f8zA2bdclu)#_jDQ-Hv<;=+OTi9IpJiYEQ^|%zWprvV@sXeCgs*K zX$>pCL(*~A0CH{2Z$9kWyI{1ZB`}q<>kHnd0=5U=lHhe0HZ+Mvhb$zYAkqPcWYc4@ zO#sdX4!Xus6raVi#} z?7SbP&}-?}8_+?B;brcRR7$-svQ8Qe*pC!LJJ--7{M?z?id&iI+N95Ygq4rGaL}jI zzg@1O@CVI+QQD@oEnQE21@d=Aocw!K-9AL3qb_btf0BilGrk8ud_6$QmROJ|WCXx` zU+(RdGK|dkFgNsT(#n`8tvrkF?N>av0Dv`cl(ejja|i3_H1`oa_5J!En2hP6+YL!M zIBhzz;jj3PDtKgr+I^_Kc8W!2*+E_V)T*OBqww_Wid151=!vll3u)HxK(#&)f&YYYsC>SyoUOOF(0Z{V(qivF0#(W6oYjMvR6$S|`s9Jl zS%Ztzb~ouF4Q>Ow7Q3#JZiI&!FXfNmx(1kv!w OnZUwb{V>Dqu1Ux#JH%H2 diff --git a/snap/.travis.yml b/snap/.travis.yml index 3c934b7bd..418407558 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -1,12 +1,15 @@ language: python before_install: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light snapd python3-venv + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light python3-venv - sudo -E /etc/init.d/nginx stop install: + - sudo /snap/bin/lxd.migrate -yes + - sudo /snap/bin/lxd waitready + - sudo /snap/bin/lxd init --auto - git clone https://github.com/certbot/certbot.git certbot --branch master - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt before_script: - - docker run -v $(pwd):$(pwd) -t snapcore/snapcraft sh -c "apt-get update -qq && apt-get install -qq git && cd $(pwd) && snapcraft" + - sudo snapcraft --use-lxd script: - sudo snap install --dangerous --classic *.snap - sudo bash test.sh @@ -21,3 +24,10 @@ notifications: email: recipients: [robie.basak@canonical.com] on_failure: change +addons: + snaps: + - name: snapcraft + channel: stable + classic: true + - name: lxd + channel: stable diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b7bfab9ca..38d32ed0b 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,4 @@ name: certbot -version: script -version-script: cd parts/certbot/src && git describe|sed s/^v// summary: Automatically configure HTTPS using Let's Encrypt description: | The objective of Certbot, Let's Encrypt, and the ACME (Automated @@ -17,6 +15,8 @@ description: | - Help you revoke the certificate if that ever becomes necessary. confinement: classic grade: devel +base: core18 +adopt-info: certbot apps: certbot: @@ -47,26 +47,31 @@ parts: plugin: python source: certbot source-subdir: acme - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: plugin: python source: certbot - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [acme] + override-pull: | + snapcraftctl pull + snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//` certbot-apache: plugin: python source: certbot source-subdir: certbot-apache - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [python-augeas, certbot] stage-packages: [libaugeas0] + python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 certbot-nginx: plugin: python source: certbot source-subdir: certbot-nginx - constraints: $SNAPCRAFT_PART_SRC/constraints.txt + constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [certbot] + python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 From 193b44a0fa86355941e4f188f1492a713d05bdf6 Mon Sep 17 00:00:00 2001 From: Sergio Schvezov Date: Thu, 9 Jan 2020 12:41:37 -0300 Subject: [PATCH 6/9] Snap plugin * snap: move snapcraft.yaml to snap directory Signed-off-by: Sergio Schvezov * snap: use a local plugin to get around the delivered plugin Add a plugin to the project which behaves as expected until a version of snapcraft satisfies the project needs. Additional snapcraft.yaml changes were made to accommodate for the snap to build. Signed-off-by: Sergio Schvezov * snap: compile pycache in the last step for the last part Signed-off-by: Sergio Schvezov --- snap/snap/plugins/x_python.py | 499 +++++++++++++++++++++++++++++++++ snap/{ => snap}/snapcraft.yaml | 26 +- 2 files changed, 517 insertions(+), 8 deletions(-) create mode 100644 snap/snap/plugins/x_python.py rename snap/{ => snap}/snapcraft.yaml (79%) diff --git a/snap/snap/plugins/x_python.py b/snap/snap/plugins/x_python.py new file mode 100644 index 000000000..5c8c3b86a --- /dev/null +++ b/snap/snap/plugins/x_python.py @@ -0,0 +1,499 @@ +# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- +# +# This is an almost verbatim copy of what is in the snapcraft +# tree, can be completely removed once the functionality in +# snapcraft is in place. (LP: #1841861) +# +# Copyright (C) 2016-2020 Canonical Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""The python plugin can be used for python 2 or 3 based parts. + +It can be used for python projects where you would want to do: + + - import python modules with a requirements.txt + - build a python project that has a setup.py + - install packages straight from pip + +This plugin uses the common plugin keywords as well as those for "sources". +For more information check the 'plugins' topic for the former and the +'sources' topic for the latter. + +Additionally, this plugin uses the following plugin-specific keywords: + + - requirements: + (list of strings) + List of paths to requirements files. + - constraints: + (list of strings) + List of paths to constraint files. + - process-dependency-links: + (bool; default: false) + Enable the processing of dependency links in pip, which allow one + project to provide places to look for another project + - python-packages: + (list) + A list of dependencies to get from PyPI + - python-version: + (string; default: python3) + The python version to use. Valid options are: python2 and python3 + +If the plugin finds a python interpreter with a basename that matches +`python-version` in the directory on the following fixed path: +`/usr/bin/` then this interpreter would +be preferred instead and no interpreter would be brought in through +`stage-packages` mechanisms. +""" + +import collections +import contextlib +import os +import re +from shutil import which +from textwrap import dedent +from typing import List, Optional, Set + +import requests + +import snapcraft +from snapcraft.common import isurl +from snapcraft.internal import errors, mangling +from snapcraft.internal.errors import SnapcraftPluginCommandError +from snapcraft.plugins import _python + + +class UnsupportedPythonVersionError(snapcraft.internal.errors.SnapcraftError): + + fmt = "Unsupported python version: {python_version!r}" + + +class SnapcraftPluginPythonFileMissing(snapcraft.internal.errors.SnapcraftError): + + fmt = ( + "Failed to find the referred {plugin_property} file at the given " + "path: {plugin_property_value!r}.\n" + "Check the property and ensure the file exists." + ) + + def __init__(self, *, plugin_property, plugin_property_value): + super().__init__( + plugin_property=plugin_property, plugin_property_value=plugin_property_value + ) + + +class PythonPlugin(snapcraft.BasePlugin): + @classmethod + def schema(cls): + schema = super().schema() + schema["properties"]["requirements"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["constraints"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["python-packages"] = { + "type": "array", + "minitems": 1, + "uniqueItems": True, + "items": {"type": "string"}, + "default": [], + } + schema["properties"]["process-dependency-links"] = { + "type": "boolean", + "default": False, + } + schema["properties"]["python-version"] = { + "type": "string", + "default": "python3", + "enum": ["python2", "python3"], + } + schema["anyOf"] = [{"required": ["source"]}, {"required": ["python-packages"]}] + + return schema + + @classmethod + def get_pull_properties(cls): + # Inform Snapcraft of the properties associated with pulling. If these + # change in the YAML Snapcraft will consider the pull step dirty. + return [ + "requirements", + "constraints", + "python-packages", + "process-dependency-links", + "python-version", + ] + + @property + def plugin_stage_packages(self): + if self.options.python_version == "python2": + python_base = "python" + elif self.options.python_version == "python3": + python_base = "python3" + + if self.project.info.get_build_base() in ("core", "core16", "core18"): + stage_packages = [python_base] + else: + stage_packages = [] + + if self.project.info.get_build_base() == "core18" and python_base == "python3": + stage_packages.append("{}-distutils".format(python_base)) + + return stage_packages + + # ignore mypy error: Read-only property cannot override read-write property + @property # type: ignore + def stage_packages(self): + try: + _python.get_python_command( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ) + except _python.errors.MissingPythonCommandError: + return super().stage_packages + self.plugin_stage_packages + else: + return super().stage_packages + + @property + def _pip(self): + if not self.__pip: + self.__pip = _python.Pip( + python_major_version=self._python_major_version, + part_dir=self.partdir, + install_dir=self.installdir, + stage_dir=self.project.stage_dir, + ) + return self.__pip + + def __init__(self, name, options, project): + super().__init__(name, options, project) + + self._setup_base_tools(project.info.get_build_base()) + + self._manifest = collections.OrderedDict() + + # Pip requires only the major version of python rather than the command + # name like our option requires. + match = re.match(r"python(?P\d).*", self.options.python_version) + if not match: + raise UnsupportedPythonVersionError( + python_version=self.options.python_version + ) + + self._python_major_version = match.group("major_version") + self.__pip = None + + def _setup_base_tools(self, base): + # NOTE: stage-packages are lazily loaded. + if base in ("core", "core16", "core18"): + if self.options.python_version == "python3": + self.build_packages.extend( + [ + "python3-dev", + "python3-pip", + "python3-pkg-resources", + "python3-setuptools", + ] + ) + elif self.options.python_version == "python2": + self.build_packages.extend( + [ + "python-dev", + "python-pip", + "python-pkg-resources", + "python-setuptools", + ] + ) + else: + raise errors.PluginBaseError(part_name=self.name, base=base) + + def pull(self): + super().pull() + + self._pip.setup() + + with simple_env_bzr(os.path.join(self.installdir, "bin")): + # Download this project, using its setup.py if present. This will + # also download any python-packages requested. + self._download_project() + + def clean_pull(self): + super().clean_pull() + self._pip.clean_packages() + + def build(self): + super().build() + + with simple_env_bzr(os.path.join(self.installdir, "bin")): + # Install the packages that have already been downloaded + installed_pipy_packages = self._install_project() + + requirements = self._get_list_of_packages_from_property( + self.options.requirements + ) + if requirements: + self._manifest["requirements-contents"] = requirements + + constraints = self._get_list_of_packages_from_property(self.options.constraints) + if constraints: + self._manifest["constraints-contents"] = constraints + + self._manifest["python-packages"] = [ + "{}={}".format(name, installed_pipy_packages[name]) + for name in installed_pipy_packages + ] + + try: + _python.generate_sitecustomize( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ) + except _python.errors.MissingUserSitePackagesError as site_error: + print("Part {!r} generated to site-packages: {!s}.".format( + self.name, site_error)) + + def _find_file(self, *, filename: str) -> Optional[str]: + # source-subdir defaults to '' + for basepath in [self.builddir, self.sourcedir]: + if basepath == self.sourcedir: + # This is overwritten in the base plugin + # TODO add consistency + source_subdir = self.options.source_subdir + else: + source_subdir = "" + filepath = os.path.join(basepath, source_subdir, filename) + if os.path.exists(filepath): + return filepath + + return None + + def _get_setup_py_dir(self): + setup_py_dir = None + setup_py_path = self._find_file(filename="setup.py") + if setup_py_path: + setup_py_dir = os.path.dirname(setup_py_path) + + return setup_py_dir + + def _get_list_of_packages_from_property(self, property_list: Set[str]) -> List[str]: + """Return a sorted list of all packages found in property.""" + package_list = list() # type: List[str] + for entry in property_list: + contents = self._get_file_contents(entry) + package_list.extend(contents.splitlines()) + return package_list + + def _get_normalized_property_set( + self, property_name, property_list: List[str] + ) -> Set[str]: + """Return a normalized set from a requirements or constraints list.""" + normalized = set() # type: Set[str] + for entry in property_list: + if isurl(entry): + normalized.add(entry) + else: + entry_file = self._find_file(filename=entry) + if not entry_file: + raise SnapcraftPluginPythonFileMissing( + plugin_property=property_name, plugin_property_value=entry + ) + normalized.add(entry_file) + + return normalized + + def _install_wheels(self, wheels): + installed = self._pip.list() + wheel_names = [os.path.basename(w).split("-")[0] for w in wheels] + + # we want to avoid installing what is already provided in + # stage-packages + need_install = [k for k in wheel_names if k not in installed] + self._pip.install( + need_install, + upgrade=True, + install_deps=False, + process_dependency_links=self.options.process_dependency_links, + ) + + def _download_project(self): + constraints = self._get_normalized_property_set( + "constraints", self.options.constraints + ) + requirements = self._get_normalized_property_set( + "requirements", self.options.requirements + ) + + self._pip.download( + self.options.python_packages, + setup_py_dir=None, + constraints=constraints, + requirements=requirements, + process_dependency_links=self.options.process_dependency_links, + ) + + def _install_project(self): + setup_py_dir = self._get_setup_py_dir() + constraints = self._get_normalized_property_set( + "constraints", self.options.constraints + ) + requirements = self._get_normalized_property_set( + "requirements", self.options.requirements + ) + + # setup.py is handled in a different step as some projects may + # need to satisfy dependencies for setup.py to be parsed. + wheels = self._pip.wheel( + self.options.python_packages, + setup_py_dir=None, + constraints=constraints, + requirements=requirements, + process_dependency_links=self.options.process_dependency_links, + ) + + if wheels: + self._install_wheels(wheels) + + if setup_py_dir is not None: + self._pip.download( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + requirements=set(), + process_dependency_links=self.options.process_dependency_links, + ) + wheels = self._pip.wheel( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + requirements=set(), + process_dependency_links=self.options.process_dependency_links, + ) + + if wheels: + self._install_wheels(wheels) + + setup_py_path = os.path.join(setup_py_dir, "setup.py") + if os.path.exists(setup_py_path): + self._pip.install( + [], + setup_py_dir=setup_py_dir, + constraints=constraints, + process_dependency_links=self.options.process_dependency_links, + upgrade=True, + ) + # pbr and others don't work using `pip install .` + # LP: #1670852 + # There is also a chance that this setup.py is distutils based + # in which case we will rely on the `pip install .` ran before + # this. + with contextlib.suppress(SnapcraftPluginCommandError): + self._setup_tools_install(setup_py_path) + + return self._pip.list() + + def _setup_tools_install(self, setup_file): + command = [ + _python.get_python_command( + self._python_major_version, + stage_dir=self.project.stage_dir, + install_dir=self.installdir, + ), + os.path.basename(setup_file), + "--no-user-cfg", + "install", + "--single-version-externally-managed", + "--user", + "--record", + "install.txt", + ] + self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file)) + + # Fix all shebangs to use the in-snap python. The stuff installed from + # pip has already been fixed, but anything done in this step has not. + mangling.rewrite_python_shebangs(self.installdir) + + def _get_file_contents(self, path): + if isurl(path): + return requests.get(path).text + else: + file_path = os.path.join(self.sourcedir, path) + with open(file_path) as _file: + return _file.read() + + def get_manifest(self): + return self._manifest + + def snap_fileset(self): + fileset = super().snap_fileset() + fileset.append("-bin/pip") + fileset.append("-bin/pip2") + fileset.append("-bin/pip3") + fileset.append("-bin/pip2.7") + fileset.append("-bin/pip3.*") + fileset.append("-bin/easy_install*") + fileset.append("-bin/wheel") + # Holds all the .pyc files. It is a major cause of inter part + # conflict. + fileset.append("-**/__pycache__") + fileset.append("-**/*.pyc") + # The RECORD files include hashes useful when uninstalling packages. + # In the snap they will cause conflicts when more than one part uses + # the python plugin. + fileset.append("-lib/python*/site-packages/*/RECORD") + return fileset + + +@contextlib.contextmanager +def simple_env_bzr(bin_dir): + """Create an appropriate environment to run bzr. + + The python plugin sets up PYTHONUSERBASE and PYTHONHOME which + conflicts with bzr when using python3 as those two environment + variables will make bzr look for modules in the wrong location. + """ + os.makedirs(bin_dir, exist_ok=True) + bzr_bin = os.path.join(bin_dir, "bzr") + real_bzr_bin = which("bzr") + if real_bzr_bin: + exec_line = 'exec {} "$@"'.format(real_bzr_bin) + else: + exec_line = "echo bzr needs to be in PATH; exit 1" + with open(bzr_bin, "w") as f: + f.write( + dedent( + """#!/bin/sh + unset PYTHONUSERBASE + unset PYTHONHOME + {} + """.format( + exec_line + ) + ) + ) + os.chmod(bzr_bin, 0o777) + try: + yield + finally: + os.remove(bzr_bin) + if not os.listdir(bin_dir): + os.rmdir(bin_dir) diff --git a/snap/snapcraft.yaml b/snap/snap/snapcraft.yaml similarity index 79% rename from snap/snapcraft.yaml rename to snap/snap/snapcraft.yaml index 38d32ed0b..0e4a12204 100644 --- a/snap/snapcraft.yaml +++ b/snap/snap/snapcraft.yaml @@ -38,19 +38,19 @@ apps: parts: python-augeas: - plugin: python + plugin: x-python source: git://github.com/basak/python-augeas source-branch: snap python-version: python3 build-packages: [libaugeas-dev] acme: - plugin: python + plugin: x-python source: certbot source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: - plugin: python + plugin: x-python source: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -58,20 +58,30 @@ parts: override-pull: | snapcraftctl pull snapcraftctl set-version `cd $SNAPCRAFT_PART_SRC && git describe|sed s/^v//` + # Workaround for lack of site-packages leading to empty sitecustomize.py + stage: + - -usr/lib/python3.6/sitecustomize.py certbot-apache: - plugin: python + plugin: x-python source: certbot source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [python-augeas, certbot] stage-packages: [libaugeas0] - python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 + stage: + # Prefer cffi + - -lib/python3.6/site-packages/augeas.py certbot-nginx: - plugin: python + plugin: x-python source: certbot source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 - after: [certbot] - python-packages: ['git+https://github.com/certbot/certbot.git'] # workaround for #12 + # This is the last step, compile pycache now as there should be no conflicts. + override-prime: | + snapcraftctl prime + ./usr/bin/python3 -m compileall -q . + # After certbot-apache to not rebuild duplicates (essentially sharing what was already staged, + # like zope) + after: [certbot-apache] From c12baf7d8c873a1492088f844fe885a16bf19cd0 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 9 Jan 2020 10:26:37 -0800 Subject: [PATCH 7/9] Fix path to apache-conf-test --- snap/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snap/test.sh b/snap/test.sh index e89743d47..e6aca7efd 100644 --- a/snap/test.sh +++ b/snap/test.sh @@ -7,5 +7,5 @@ venv/bin/python -m pytest certbot/certbot-ci/certbot_integration_tests -n 4 # DO NOT RUN `apache-conf-test` LOCALLY, IT WILL BREAK YOUR APACHE CONFIGURATION! if [ -n "$TRAVIS" ]; then - venv/bin/python certbot/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules + venv/bin/python certbot/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules fi From 2bcabe6626ad4aa0073269bcee9719c24da469cb Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Tue, 18 Feb 2020 13:58:18 +0100 Subject: [PATCH 8/9] Fix certbot part build in snap * Declare properly source-subdir to build the certbot part * Use snapcraft 3.10+, remove the custom python plugin --- snap/.travis.yml | 3 +- snap/snap/plugins/x_python.py | 499 ---------------------------------- snap/snap/snapcraft.yaml | 11 +- 3 files changed, 8 insertions(+), 505 deletions(-) delete mode 100644 snap/snap/plugins/x_python.py diff --git a/snap/.travis.yml b/snap/.travis.yml index 418407558..125893985 100644 --- a/snap/.travis.yml +++ b/snap/.travis.yml @@ -27,7 +27,8 @@ notifications: addons: snaps: - name: snapcraft - channel: stable + # TODO: move back to stable channel once snapcraft 3.10 has been pushed to it. + channel: candidate classic: true - name: lxd channel: stable diff --git a/snap/snap/plugins/x_python.py b/snap/snap/plugins/x_python.py deleted file mode 100644 index 5c8c3b86a..000000000 --- a/snap/snap/plugins/x_python.py +++ /dev/null @@ -1,499 +0,0 @@ -# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- -# -# This is an almost verbatim copy of what is in the snapcraft -# tree, can be completely removed once the functionality in -# snapcraft is in place. (LP: #1841861) -# -# Copyright (C) 2016-2020 Canonical Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""The python plugin can be used for python 2 or 3 based parts. - -It can be used for python projects where you would want to do: - - - import python modules with a requirements.txt - - build a python project that has a setup.py - - install packages straight from pip - -This plugin uses the common plugin keywords as well as those for "sources". -For more information check the 'plugins' topic for the former and the -'sources' topic for the latter. - -Additionally, this plugin uses the following plugin-specific keywords: - - - requirements: - (list of strings) - List of paths to requirements files. - - constraints: - (list of strings) - List of paths to constraint files. - - process-dependency-links: - (bool; default: false) - Enable the processing of dependency links in pip, which allow one - project to provide places to look for another project - - python-packages: - (list) - A list of dependencies to get from PyPI - - python-version: - (string; default: python3) - The python version to use. Valid options are: python2 and python3 - -If the plugin finds a python interpreter with a basename that matches -`python-version` in the directory on the following fixed path: -`/usr/bin/` then this interpreter would -be preferred instead and no interpreter would be brought in through -`stage-packages` mechanisms. -""" - -import collections -import contextlib -import os -import re -from shutil import which -from textwrap import dedent -from typing import List, Optional, Set - -import requests - -import snapcraft -from snapcraft.common import isurl -from snapcraft.internal import errors, mangling -from snapcraft.internal.errors import SnapcraftPluginCommandError -from snapcraft.plugins import _python - - -class UnsupportedPythonVersionError(snapcraft.internal.errors.SnapcraftError): - - fmt = "Unsupported python version: {python_version!r}" - - -class SnapcraftPluginPythonFileMissing(snapcraft.internal.errors.SnapcraftError): - - fmt = ( - "Failed to find the referred {plugin_property} file at the given " - "path: {plugin_property_value!r}.\n" - "Check the property and ensure the file exists." - ) - - def __init__(self, *, plugin_property, plugin_property_value): - super().__init__( - plugin_property=plugin_property, plugin_property_value=plugin_property_value - ) - - -class PythonPlugin(snapcraft.BasePlugin): - @classmethod - def schema(cls): - schema = super().schema() - schema["properties"]["requirements"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["constraints"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["python-packages"] = { - "type": "array", - "minitems": 1, - "uniqueItems": True, - "items": {"type": "string"}, - "default": [], - } - schema["properties"]["process-dependency-links"] = { - "type": "boolean", - "default": False, - } - schema["properties"]["python-version"] = { - "type": "string", - "default": "python3", - "enum": ["python2", "python3"], - } - schema["anyOf"] = [{"required": ["source"]}, {"required": ["python-packages"]}] - - return schema - - @classmethod - def get_pull_properties(cls): - # Inform Snapcraft of the properties associated with pulling. If these - # change in the YAML Snapcraft will consider the pull step dirty. - return [ - "requirements", - "constraints", - "python-packages", - "process-dependency-links", - "python-version", - ] - - @property - def plugin_stage_packages(self): - if self.options.python_version == "python2": - python_base = "python" - elif self.options.python_version == "python3": - python_base = "python3" - - if self.project.info.get_build_base() in ("core", "core16", "core18"): - stage_packages = [python_base] - else: - stage_packages = [] - - if self.project.info.get_build_base() == "core18" and python_base == "python3": - stage_packages.append("{}-distutils".format(python_base)) - - return stage_packages - - # ignore mypy error: Read-only property cannot override read-write property - @property # type: ignore - def stage_packages(self): - try: - _python.get_python_command( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ) - except _python.errors.MissingPythonCommandError: - return super().stage_packages + self.plugin_stage_packages - else: - return super().stage_packages - - @property - def _pip(self): - if not self.__pip: - self.__pip = _python.Pip( - python_major_version=self._python_major_version, - part_dir=self.partdir, - install_dir=self.installdir, - stage_dir=self.project.stage_dir, - ) - return self.__pip - - def __init__(self, name, options, project): - super().__init__(name, options, project) - - self._setup_base_tools(project.info.get_build_base()) - - self._manifest = collections.OrderedDict() - - # Pip requires only the major version of python rather than the command - # name like our option requires. - match = re.match(r"python(?P\d).*", self.options.python_version) - if not match: - raise UnsupportedPythonVersionError( - python_version=self.options.python_version - ) - - self._python_major_version = match.group("major_version") - self.__pip = None - - def _setup_base_tools(self, base): - # NOTE: stage-packages are lazily loaded. - if base in ("core", "core16", "core18"): - if self.options.python_version == "python3": - self.build_packages.extend( - [ - "python3-dev", - "python3-pip", - "python3-pkg-resources", - "python3-setuptools", - ] - ) - elif self.options.python_version == "python2": - self.build_packages.extend( - [ - "python-dev", - "python-pip", - "python-pkg-resources", - "python-setuptools", - ] - ) - else: - raise errors.PluginBaseError(part_name=self.name, base=base) - - def pull(self): - super().pull() - - self._pip.setup() - - with simple_env_bzr(os.path.join(self.installdir, "bin")): - # Download this project, using its setup.py if present. This will - # also download any python-packages requested. - self._download_project() - - def clean_pull(self): - super().clean_pull() - self._pip.clean_packages() - - def build(self): - super().build() - - with simple_env_bzr(os.path.join(self.installdir, "bin")): - # Install the packages that have already been downloaded - installed_pipy_packages = self._install_project() - - requirements = self._get_list_of_packages_from_property( - self.options.requirements - ) - if requirements: - self._manifest["requirements-contents"] = requirements - - constraints = self._get_list_of_packages_from_property(self.options.constraints) - if constraints: - self._manifest["constraints-contents"] = constraints - - self._manifest["python-packages"] = [ - "{}={}".format(name, installed_pipy_packages[name]) - for name in installed_pipy_packages - ] - - try: - _python.generate_sitecustomize( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ) - except _python.errors.MissingUserSitePackagesError as site_error: - print("Part {!r} generated to site-packages: {!s}.".format( - self.name, site_error)) - - def _find_file(self, *, filename: str) -> Optional[str]: - # source-subdir defaults to '' - for basepath in [self.builddir, self.sourcedir]: - if basepath == self.sourcedir: - # This is overwritten in the base plugin - # TODO add consistency - source_subdir = self.options.source_subdir - else: - source_subdir = "" - filepath = os.path.join(basepath, source_subdir, filename) - if os.path.exists(filepath): - return filepath - - return None - - def _get_setup_py_dir(self): - setup_py_dir = None - setup_py_path = self._find_file(filename="setup.py") - if setup_py_path: - setup_py_dir = os.path.dirname(setup_py_path) - - return setup_py_dir - - def _get_list_of_packages_from_property(self, property_list: Set[str]) -> List[str]: - """Return a sorted list of all packages found in property.""" - package_list = list() # type: List[str] - for entry in property_list: - contents = self._get_file_contents(entry) - package_list.extend(contents.splitlines()) - return package_list - - def _get_normalized_property_set( - self, property_name, property_list: List[str] - ) -> Set[str]: - """Return a normalized set from a requirements or constraints list.""" - normalized = set() # type: Set[str] - for entry in property_list: - if isurl(entry): - normalized.add(entry) - else: - entry_file = self._find_file(filename=entry) - if not entry_file: - raise SnapcraftPluginPythonFileMissing( - plugin_property=property_name, plugin_property_value=entry - ) - normalized.add(entry_file) - - return normalized - - def _install_wheels(self, wheels): - installed = self._pip.list() - wheel_names = [os.path.basename(w).split("-")[0] for w in wheels] - - # we want to avoid installing what is already provided in - # stage-packages - need_install = [k for k in wheel_names if k not in installed] - self._pip.install( - need_install, - upgrade=True, - install_deps=False, - process_dependency_links=self.options.process_dependency_links, - ) - - def _download_project(self): - constraints = self._get_normalized_property_set( - "constraints", self.options.constraints - ) - requirements = self._get_normalized_property_set( - "requirements", self.options.requirements - ) - - self._pip.download( - self.options.python_packages, - setup_py_dir=None, - constraints=constraints, - requirements=requirements, - process_dependency_links=self.options.process_dependency_links, - ) - - def _install_project(self): - setup_py_dir = self._get_setup_py_dir() - constraints = self._get_normalized_property_set( - "constraints", self.options.constraints - ) - requirements = self._get_normalized_property_set( - "requirements", self.options.requirements - ) - - # setup.py is handled in a different step as some projects may - # need to satisfy dependencies for setup.py to be parsed. - wheels = self._pip.wheel( - self.options.python_packages, - setup_py_dir=None, - constraints=constraints, - requirements=requirements, - process_dependency_links=self.options.process_dependency_links, - ) - - if wheels: - self._install_wheels(wheels) - - if setup_py_dir is not None: - self._pip.download( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - requirements=set(), - process_dependency_links=self.options.process_dependency_links, - ) - wheels = self._pip.wheel( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - requirements=set(), - process_dependency_links=self.options.process_dependency_links, - ) - - if wheels: - self._install_wheels(wheels) - - setup_py_path = os.path.join(setup_py_dir, "setup.py") - if os.path.exists(setup_py_path): - self._pip.install( - [], - setup_py_dir=setup_py_dir, - constraints=constraints, - process_dependency_links=self.options.process_dependency_links, - upgrade=True, - ) - # pbr and others don't work using `pip install .` - # LP: #1670852 - # There is also a chance that this setup.py is distutils based - # in which case we will rely on the `pip install .` ran before - # this. - with contextlib.suppress(SnapcraftPluginCommandError): - self._setup_tools_install(setup_py_path) - - return self._pip.list() - - def _setup_tools_install(self, setup_file): - command = [ - _python.get_python_command( - self._python_major_version, - stage_dir=self.project.stage_dir, - install_dir=self.installdir, - ), - os.path.basename(setup_file), - "--no-user-cfg", - "install", - "--single-version-externally-managed", - "--user", - "--record", - "install.txt", - ] - self.run(command, env=self._pip.env(), cwd=os.path.dirname(setup_file)) - - # Fix all shebangs to use the in-snap python. The stuff installed from - # pip has already been fixed, but anything done in this step has not. - mangling.rewrite_python_shebangs(self.installdir) - - def _get_file_contents(self, path): - if isurl(path): - return requests.get(path).text - else: - file_path = os.path.join(self.sourcedir, path) - with open(file_path) as _file: - return _file.read() - - def get_manifest(self): - return self._manifest - - def snap_fileset(self): - fileset = super().snap_fileset() - fileset.append("-bin/pip") - fileset.append("-bin/pip2") - fileset.append("-bin/pip3") - fileset.append("-bin/pip2.7") - fileset.append("-bin/pip3.*") - fileset.append("-bin/easy_install*") - fileset.append("-bin/wheel") - # Holds all the .pyc files. It is a major cause of inter part - # conflict. - fileset.append("-**/__pycache__") - fileset.append("-**/*.pyc") - # The RECORD files include hashes useful when uninstalling packages. - # In the snap they will cause conflicts when more than one part uses - # the python plugin. - fileset.append("-lib/python*/site-packages/*/RECORD") - return fileset - - -@contextlib.contextmanager -def simple_env_bzr(bin_dir): - """Create an appropriate environment to run bzr. - - The python plugin sets up PYTHONUSERBASE and PYTHONHOME which - conflicts with bzr when using python3 as those two environment - variables will make bzr look for modules in the wrong location. - """ - os.makedirs(bin_dir, exist_ok=True) - bzr_bin = os.path.join(bin_dir, "bzr") - real_bzr_bin = which("bzr") - if real_bzr_bin: - exec_line = 'exec {} "$@"'.format(real_bzr_bin) - else: - exec_line = "echo bzr needs to be in PATH; exit 1" - with open(bzr_bin, "w") as f: - f.write( - dedent( - """#!/bin/sh - unset PYTHONUSERBASE - unset PYTHONHOME - {} - """.format( - exec_line - ) - ) - ) - os.chmod(bzr_bin, 0o777) - try: - yield - finally: - os.remove(bzr_bin) - if not os.listdir(bin_dir): - os.rmdir(bin_dir) diff --git a/snap/snap/snapcraft.yaml b/snap/snap/snapcraft.yaml index 0e4a12204..53643865a 100644 --- a/snap/snap/snapcraft.yaml +++ b/snap/snap/snapcraft.yaml @@ -38,20 +38,21 @@ apps: parts: python-augeas: - plugin: x-python + plugin: python source: git://github.com/basak/python-augeas source-branch: snap python-version: python3 build-packages: [libaugeas-dev] acme: - plugin: x-python + plugin: python source: certbot source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: - plugin: x-python + plugin: python source: certbot + source-subdir: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 after: [acme] @@ -62,7 +63,7 @@ parts: stage: - -usr/lib/python3.6/sitecustomize.py certbot-apache: - plugin: x-python + plugin: python source: certbot source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] @@ -73,7 +74,7 @@ parts: # Prefer cffi - -lib/python3.6/site-packages/augeas.py certbot-nginx: - plugin: x-python + plugin: python source: certbot source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] From 335894ab3b0665dcf177ba2121a83422b1053fde Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 9 Apr 2020 13:15:28 -0700 Subject: [PATCH 9/9] Merge snap code into the Certbot repo * merge .gitignore * Move snapcraft.yml up one level. * update source * move test.sh to tox.ini * use new tox.ini in .travis.yml * move snap build code * make script executable * remove unused python3-dev * don't use deprecated classic flag * go back to stable channel * add nginx in snap addons * add deploy steps * Add comments explaining external tox envs. * error if not in CI * don't use --depth * remove old .travis.yml * Add big comment about SNAP_TOKEN. * Set all_branches: true. * Add repo setting. * run travis on tags * Add more documenting comments to .travis.yml. --- .gitignore | 7 ++++ .travis.yml | 67 ++++++++++++++++++++++++++++++--- snap/.gitignore | 8 ---- snap/.travis.yml | 34 ----------------- snap/local/build_and_install.sh | 14 +++++++ snap/{snap => }/snapcraft.yaml | 8 ++-- snap/test.sh | 11 ------ tox.ini | 21 +++++++++-- 8 files changed, 105 insertions(+), 65 deletions(-) delete mode 100644 snap/.gitignore delete mode 100644 snap/.travis.yml create mode 100755 snap/local/build_and_install.sh rename snap/{snap => }/snapcraft.yaml (97%) delete mode 100644 snap/test.sh diff --git a/.gitignore b/.gitignore index 6505e716c..064e7fffe 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,10 @@ tests/letstest/venv3/ .certbot_test_workspace **/assets/pebble* **/assets/challtestsrv* + +# snap files +.snapcraft +parts +prime +stage +*.snap diff --git a/.travis.yml b/.travis.yml index 72cd3a408..3b22429c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,17 +11,23 @@ before_script: # Use Travis retry feature for farm tests since they are flaky - 'if [[ "$TOXENV" == "travis-test-farm"* ]]; then export TRAVIS_RETRY=travis_retry; fi' - export TOX_TESTENV_PASSENV=TRAVIS + - 'if [[ "$SNAP" == true ]]; then snap/local/build_and_install.sh; fi' # Only build pushes to the master branch, PRs, and branches beginning with -# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x`. This reduces -# the number of simultaneous Travis runs, which speeds turnaround time on -# review since there is a cap of on the number of simultaneous runs. +# `test-`, `travis-test-`, or of the form `digit(s).digit(s).x` or +# `vdigit(s).digit(s).digit(s)`. As documented at +# https://docs.travis-ci.com/user/customizing-the-build/#safelisting-or-blocklisting-branches, +# this includes tags so pushing tags of the form `vdigit(s).digit(s).digit(s)` +# will also trigger tests. This reduces the number of simultaneous Travis runs, +# which speeds turnaround time on review since there is a cap of on the number +# of simultaneous runs. branches: # When changing these branches, please ensure the documentation under # "Running tests in CI" is still correct. only: - master - - /^\d+\.\d+\.x$/ + - /^\d+\.\d+\.x$/ # this matches our point release branches + - /^v\d+\.\d+\.\d+$/ # this matches our release tags - /^(travis-)?test-.*$/ # Jobs for the main test suite are always executed (including on PRs) except for pushes on master. @@ -36,10 +42,16 @@ extended-test-suite: &extended-test-suite matrix: include: # Main test suite - - python: "2.7" + - stage: "Test" + python: "2.7" env: ACME_SERVER=pebble TOXENV=integration <<: *not-on-master + # As documented at + # https://docs.travis-ci.com/user/build-stages/#how-to-define-build-stages, + # the previous stage will be automatically applied to all subsequent jobs + # until a new stage is defined. + # This job is always executed, including on master - python: "3.8" env: TOXENV=py38-cover FYI="py38 tests + code coverage" @@ -220,6 +232,51 @@ matrix: packages: # don't install nginx and apache - libaugeas0 <<: *extended-test-suite + - stage: "Snap" + sudo: required + env: SNAP=true TOXENV=integration-external,apacheconftest-external-with-pebble + addons: + apt: + packages: + - nginx-light + snaps: + - name: snapcraft + channel: stable + confinement: classic + - name: lxd + channel: stable + git: + # By default, Travis clones the repo to a depth of 50 commits which can + # break the ability to use `git describe` to set the version of the + # snap. This setting removes the --depth flag from git commands solving + # this problem. See + # https://docs.travis-ci.com/user/customizing-the-build#git-clone-depth + # for more info. + depth: false + deploy: + # This section relies on credentials stored in a SNAP_TOKEN environment + # variable in Travis. See + # https://docs.travis-ci.com/user/deployment/snaps/ for more info. + # This credential has a maximum lifetime of 1 year and the current + # credential will expire on 4/22/2021. The value of SNAP_TOKEN will + # need to be updated to use a new credential before then to prevent + # automated deploys from breaking. Remembering to do this is also + # tracked by https://github.com/certbot/certbot/issues/7931. + 'on': + # Deploy on release tags or nightly runs from any branch. We only try + # to deploy from the certbot/certbot repo to prevent errors if forks + # of this repo try to run tests. + all_branches: true + condition: -n $TRAVIS_TAG || $TRAVIS_EVENT_TYPE = cron + repo: certbot/certbot + provider: snap + snap: certbot_*.snap + channel: edge + # skip_cleanup is needed to prevent Travis from deleting the snaps we + # just built and tested. See + # https://docs.travis-ci.com/user/deployment#uploading-files-and-skip_cleanup. + skip_cleanup: true + <<: *extended-test-suite # container-based infrastructure sudo: false diff --git a/snap/.gitignore b/snap/.gitignore deleted file mode 100644 index 2a339aeaa..000000000 --- a/snap/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.snapcraft -parts -prime -stage -*.snap -certbot -venv -.idea diff --git a/snap/.travis.yml b/snap/.travis.yml deleted file mode 100644 index 125893985..000000000 --- a/snap/.travis.yml +++ /dev/null @@ -1,34 +0,0 @@ -language: python -before_install: - - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install nginx-light python3-venv - - sudo -E /etc/init.d/nginx stop -install: - - sudo /snap/bin/lxd.migrate -yes - - sudo /snap/bin/lxd waitready - - sudo /snap/bin/lxd init --auto - - git clone https://github.com/certbot/certbot.git certbot --branch master - - certbot/tools/strip_hashes.py certbot/letsencrypt-auto-source/pieces/dependency-requirements.txt > certbot/constraints.txt -before_script: - - sudo snapcraft --use-lxd -script: - - sudo snap install --dangerous --classic *.snap - - sudo bash test.sh -deploy: - 'on': - branch: master - provider: snap - snap: certbot_*.snap - channel: edge - skip_cleanup: true -notifications: - email: - recipients: [robie.basak@canonical.com] - on_failure: change -addons: - snaps: - - name: snapcraft - # TODO: move back to stable channel once snapcraft 3.10 has been pushed to it. - channel: candidate - classic: true - - name: lxd - channel: stable diff --git a/snap/local/build_and_install.sh b/snap/local/build_and_install.sh new file mode 100755 index 000000000..4c9754d3e --- /dev/null +++ b/snap/local/build_and_install.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -ex + +if [[ -z "$TRAVIS" ]]; then + echo "This script makes global changes to the system it is run on so should only be run in CI." + exit 1 +fi + +sudo /snap/bin/lxd.migrate -yes +sudo /snap/bin/lxd waitready +sudo /snap/bin/lxd init --auto +tools/strip_hashes.py letsencrypt-auto-source/pieces/dependency-requirements.txt > constraints.txt +sudo snapcraft --use-lxd +sudo snap install --dangerous --classic *.snap diff --git a/snap/snap/snapcraft.yaml b/snap/snapcraft.yaml similarity index 97% rename from snap/snap/snapcraft.yaml rename to snap/snapcraft.yaml index 53643865a..2e4f68c00 100644 --- a/snap/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -45,13 +45,13 @@ parts: build-packages: [libaugeas-dev] acme: plugin: python - source: certbot + source: . source-subdir: acme constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 certbot: plugin: python - source: certbot + source: . source-subdir: certbot constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -64,7 +64,7 @@ parts: - -usr/lib/python3.6/sitecustomize.py certbot-apache: plugin: python - source: certbot + source: . source-subdir: certbot-apache constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 @@ -75,7 +75,7 @@ parts: - -lib/python3.6/site-packages/augeas.py certbot-nginx: plugin: python - source: certbot + source: . source-subdir: certbot-nginx constraints: [$SNAPCRAFT_PART_SRC/constraints.txt] python-version: python3 diff --git a/snap/test.sh b/snap/test.sh deleted file mode 100644 index e6aca7efd..000000000 --- a/snap/test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -ex - -python3 -m venv venv -venv/bin/python -m pip install -e certbot/certbot-ci -venv/bin/python -m pytest certbot/certbot-ci/certbot_integration_tests -n 4 - -# DO NOT RUN `apache-conf-test` LOCALLY, IT WILL BREAK YOUR APACHE CONFIGURATION! -if [ -n "$TRAVIS" ]; then - venv/bin/python certbot/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules -fi diff --git a/tox.ini b/tox.ini index 7f5b7bd5a..46e73e32f 100644 --- a/tox.ini +++ b/tox.ini @@ -138,15 +138,22 @@ commands = [testenv:apacheconftest] commands = - {[base]pip_install} acme certbot certbot-apache certbot-compatibility-test + {[base]pip_install} acme certbot certbot-apache {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test --debian-modules passenv = SERVER +[testenv:apacheconftest-external-with-pebble] +# Run apacheconftest with pebble and Certbot outside of tox's virtual +# environment. +commands = + {[base]pip_install} certbot-ci + {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules + [testenv:apacheconftest-with-pebble] commands = - {[base]pip_install} acme certbot certbot-apache certbot-ci certbot-compatibility-test - {toxinidir}/certbot-apache/tests/apache-conf-files/apache-conf-test-pebble.py --debian-modules + {[base]pip_install} acme certbot certbot-apache + {[testenv:apacheconftest-external-with-pebble]commands} [testenv:nginxroundtrip] commands = @@ -250,6 +257,14 @@ commands = --cov-config=certbot-ci/certbot_integration_tests/.coveragerc coverage report --include 'certbot/*' --show-missing --fail-under=62 +[testenv:integration-external] +# Run integration tests with Certbot outside of tox's virtual environment. +commands = + {[base]pip_install} certbot-ci + pytest certbot-ci/certbot_integration_tests \ + --acme-server={env:ACME_SERVER:pebble} +passenv = DOCKER_* + [testenv:integration-certbot-oldest] commands = {[base]pip_install} certbot