From e5247ced5e08bd797ce79c5a07a9fbd0d0cfe54c Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Wed, 23 Dec 2015 18:32:20 -0500 Subject: [PATCH 01/96] Include the python version being used in the user agent for ACME requests. --- acme/acme/client.py | 5 ++++- letsencrypt/client.py | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index c3e28ef47..cc90b7866 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -2,6 +2,7 @@ import datetime import heapq import logging +import platform import time import six @@ -483,6 +484,8 @@ class Client(object): # pylint: disable=too-many-instance-attributes 'Successful revocation must return HTTP OK status') +DEFAULT_USER_AGENT = 'acme-python Python/%s' % platform.python_version() + class ClientNetwork(object): """Client network.""" JSON_CONTENT_TYPE = 'application/json' @@ -490,7 +493,7 @@ class ClientNetwork(object): REPLAY_NONCE_HEADER = 'Replay-Nonce' def __init__(self, key, alg=jose.RS256, verify_ssl=True, - user_agent='acme-python'): + user_agent=DEFAULT_USER_AGENT): self.key = key self.alg = alg self.verify_ssl = verify_ssl diff --git a/letsencrypt/client.py b/letsencrypt/client.py index 080ee7991..b7c4ccab8 100644 --- a/letsencrypt/client.py +++ b/letsencrypt/client.py @@ -1,6 +1,7 @@ """Let's Encrypt client API.""" import logging import os +import platform from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives.asymmetric import rsa @@ -51,9 +52,13 @@ def _determine_user_agent(config): """ if config.user_agent is None: - ua = "LetsEncryptPythonClient/{0} ({1}) Authenticator/{2} Installer/{3}" + ua = ( + "LetsEncryptPythonClient/{0} ({1}) Authenticator/{2} Installer/{3}" + " Python/{4}" + ) ua = ua.format(letsencrypt.__version__, " ".join(le_util.get_os_info()), - config.authenticator, config.installer) + config.authenticator, config.installer, + platform.python_version()) else: ua = config.user_agent return ua From 5376c3074599bbbcf5f44b8fe9127a2f62490d80 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Tue, 5 Jan 2016 07:30:49 -0500 Subject: [PATCH 02/96] Revert changes to the acme directory Only send the Python version in the user agent for the letsencrypt tool. --- acme/acme/client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/acme/acme/client.py b/acme/acme/client.py index cc90b7866..c3e28ef47 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -2,7 +2,6 @@ import datetime import heapq import logging -import platform import time import six @@ -484,8 +483,6 @@ class Client(object): # pylint: disable=too-many-instance-attributes 'Successful revocation must return HTTP OK status') -DEFAULT_USER_AGENT = 'acme-python Python/%s' % platform.python_version() - class ClientNetwork(object): """Client network.""" JSON_CONTENT_TYPE = 'application/json' @@ -493,7 +490,7 @@ class ClientNetwork(object): REPLAY_NONCE_HEADER = 'Replay-Nonce' def __init__(self, key, alg=jose.RS256, verify_ssl=True, - user_agent=DEFAULT_USER_AGENT): + user_agent='acme-python'): self.key = key self.alg = alg self.verify_ssl = verify_ssl From 1d7fd33f4d71ff7509a997b85be3f39b9573e65c Mon Sep 17 00:00:00 2001 From: Breland Miley Date: Sun, 31 Jan 2016 18:35:35 -0800 Subject: [PATCH 03/96] Initial commit --- .gitignore | 62 +++++++++++++++++ LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 263 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..1dbc687de --- /dev/null +++ b/.gitignore @@ -0,0 +1,62 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +#Ipython Notebook +.ipynb_checkpoints diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 36fdf99a1d5d30eb7902316f32f1d0f60c536c07 Mon Sep 17 00:00:00 2001 From: Miley Date: Sun, 31 Jan 2016 22:33:17 -0800 Subject: [PATCH 04/96] Initial commit, not safe to use --- LICENSE.txt | 1 + MANIFEST.in | 4 ++ README.md | 35 +++++++++++ letsencrypt_route53/__init__.py | 1 + letsencrypt_route53/authenticator.py | 93 ++++++++++++++++++++++++++++ sample-aws-policy.json | 36 +++++++++++ setup.cfg | 2 + setup.py | 62 +++++++++++++++++++ 8 files changed, 234 insertions(+) create mode 100644 LICENSE.txt create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 letsencrypt_route53/__init__.py create mode 100644 letsencrypt_route53/authenticator.py create mode 100644 sample-aws-policy.json create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 000000000..7ff097c3c --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1 @@ +See LICENSE diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..4faaf21cd --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include LICENSE.txt +include README.md +recursive-include docs * +recursive-include letsencrypt_route53/tests/testdata * diff --git a/README.md b/README.md new file mode 100644 index 000000000..18725f1f7 --- /dev/null +++ b/README.md @@ -0,0 +1,35 @@ +## Route53 plugin for Let's Encrypt client + + +### Before you start + +It's expected that the root hosted zone for the domain in question already exists in your account. + +### Setup + +1. Install the letsencrypt client [https://letsencrypt.readthedocs.org/en/latest/using.html#installation](https://letsencrypt.readthedocs.org/en/latest/using.html#installation) + + ``` + pip install letsencrypt + ``` + +1. Install the letsencrypt-s3front plugin + + ``` + pip install letsencrypt-s3front + ``` + +### How to use it + +To generate a certificate and install it in a CloudFront distribution: +``` +AWS_ACCESS_KEY_ID="your_key" \ +AWS_SECRET_ACCESS_KEY="your_secret" \ +letsencrypt --agree-tos -a letsencrypt-route53:auth \ +-d the_domain +``` + +Follow the screen prompts and you should end up with the certificate in your +distribution. It may take a couple minutes to update. + +To automate the renewal process without prompts (for example, with a monthly cron), you can add the letsencrypt parameters --renew-by-default --text diff --git a/letsencrypt_route53/__init__.py b/letsencrypt_route53/__init__.py new file mode 100644 index 000000000..6f519023b --- /dev/null +++ b/letsencrypt_route53/__init__.py @@ -0,0 +1 @@ +"""Let's Encrypt Route53 plugin.""" diff --git a/letsencrypt_route53/authenticator.py b/letsencrypt_route53/authenticator.py new file mode 100644 index 000000000..280a5bb55 --- /dev/null +++ b/letsencrypt_route53/authenticator.py @@ -0,0 +1,93 @@ +"""Route53 Let's Encrypt authenticator plugin.""" +import os +import logging +import re +import subprocess + +import zope.component +import zope.interface + +import boto3 + +from acme import challenges + +from letsencrypt import errors +from letsencrypt import interfaces +from letsencrypt.plugins import common + + +logger = logging.getLogger(__name__) + +class Authenticator(common.Plugin): + zope.interface.implements(interfaces.IAuthenticator) + zope.interface.classProvides(interfaces.IPluginFactory) + + description = "Route53 Authenticator" + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self._httpd = None + + def prepare(self): # pylint: disable=missing-docstring,no-self-use + pass # pragma: no cover + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return ("") + + def get_chall_pref(self, domain): + # pylint: disable=missing-docstring,no-self-use,unused-argument + return [challenges.DNS] + + def perform(self, achalls): # pylint: disable=missing-docstring + responses = [] + for achall in achalls: + responses.append(self._perform_single(achall)) + return responses + + def _perform_single(self, achall): + # provision the TXT record, using the domain name given. Assumes the hosted zone exits, else fails the challenge + response, validation = achall.response_and_validation() + r53 = boto3.client('route53') + logger.info("Doing validation for " + achall) + listResponse = r53.list_hosted_zones_by_name(DNSName=achall.chall.path[1:]) + matches = listResponse.HostedZones; + if matches.size != 0: + logger.error("Route53 returned " + mathces.size + " matching hosted zones. Expected exactly one. Auth canceled.") + return None + else: + r53.change_resource_record_sets(HostedZoneId=matches[0].Id, + ChangeBatch={ + 'Comment': 'Let\'s Entcrypt Change', + 'Changes': [ + { + 'Action': 'UPSERT', + 'ResourceRecordSet': { + 'Name': achall.chall.path[1:], + 'Type': 'TXT', + 'TTL': 300, + 'ResourceRecords': [ + { + 'Value': validation + }, + ] + } + }, + ] + }) + + if response.simple_verify( + achall.chall, achall.domain, + achall.account_key.public_key(), self.config.http01_port): + return response + else: + logger.error( + "Self-verify of challenge failed, authorization abandoned!") + return None + + def cleanup(self, achalls): + # pylint: disable=missing-docstring,no-self-use,unused-argument + #TODO:Cleanup record  + r53 = boto3.client('route53') + #for achall in achalls: + # r53.delete_object(Bucket=self.conf('s3-bucket'), Key=achall.chall.path[1:]) + return None diff --git a/sample-aws-policy.json b/sample-aws-policy.json new file mode 100644 index 000000000..1af66c85d --- /dev/null +++ b/sample-aws-policy.json @@ -0,0 +1,36 @@ +{ + "Version": "2012-10-17", + "Id": "letsencrypt-route53 sample policy", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:UploadServerCertificate", + "iam:UpdateServerCertificate", + "iam:DeleteServerCertificate" + ], + "Resource": [ + "*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "route53:List*", + "route53:Get*", + ], + "Resource": [ + "*" + ] + }, + { + "Effect" : "Allow", + "Action" : [ + "route53:ChangeResourceRecordSets" + ], + "Resource" : [ + "arn:aws:route53:::hostedzone/YOURHOSTEDZONEID" + ] + } + ] +} diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..b88034e41 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..5b2e1da06 --- /dev/null +++ b/setup.py @@ -0,0 +1,62 @@ +import sys + +from distutils.core import setup +from setuptools import find_packages + +version = '0.1.3' + +install_requires = [ + 'acme>=0.1.1', + 'letsencrypt>=0.1.1', + 'PyOpenSSL', + 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? + 'setuptools', # pkg_resources + 'zope.interface', + 'boto3' +] + +if sys.version_info < (2, 7): + install_requires.append('mock<1.1.0') +else: + install_requires.append('mock') + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='letsencrypt-route53', + version=version, + description="Route53 plugin for Let's Encrypt client", + url='https://github.com/mindstorms6/letsencrypt-route53', + author="Breland Miley", + author_email='breland@bdawg.org', + license='Apache2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + keywords = ['letsencrypt', 'route53', 'aws'], + entry_points={ + 'letsencrypt.plugins': [ + 'auth = letsencrypt_s3front.authenticator:Authenticator' + ], + }, +) From bb982024f89d015dbc46e5f71027408a79791dd7 Mon Sep 17 00:00:00 2001 From: Miley Date: Sun, 31 Jan 2016 22:46:28 -0800 Subject: [PATCH 05/96] Fix entry point --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5b2e1da06..02d9e01d5 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ setup( keywords = ['letsencrypt', 'route53', 'aws'], entry_points={ 'letsencrypt.plugins': [ - 'auth = letsencrypt_s3front.authenticator:Authenticator' + 'auth = letsencrypt_route53.authenticator:Authenticator' ], }, ) From c8e911e00b68decb116cbd246490d0bd463237fa Mon Sep 17 00:00:00 2001 From: mindstorms6 Date: Tue, 2 Feb 2016 07:45:51 +0000 Subject: [PATCH 06/96] Updates for authenticator, still WIP --- README.md | 2 +- letsencrypt_route53/authenticator.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 18725f1f7..e39616a52 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ It's expected that the root hosted zone for the domain in question already exist 1. Install the letsencrypt-s3front plugin ``` - pip install letsencrypt-s3front + pip install letsencrypt-route53 ``` ### How to use it diff --git a/letsencrypt_route53/authenticator.py b/letsencrypt_route53/authenticator.py index 280a5bb55..95d24b0ab 100644 --- a/letsencrypt_route53/authenticator.py +++ b/letsencrypt_route53/authenticator.py @@ -36,7 +36,7 @@ class Authenticator(common.Plugin): def get_chall_pref(self, domain): # pylint: disable=missing-docstring,no-self-use,unused-argument - return [challenges.DNS] + return [challenges.DNS01] def perform(self, achalls): # pylint: disable=missing-docstring responses = [] @@ -48,8 +48,8 @@ class Authenticator(common.Plugin): # provision the TXT record, using the domain name given. Assumes the hosted zone exits, else fails the challenge response, validation = achall.response_and_validation() r53 = boto3.client('route53') - logger.info("Doing validation for " + achall) - listResponse = r53.list_hosted_zones_by_name(DNSName=achall.chall.path[1:]) + logger.info("Doing validation for " + response.domain) + listResponse = r53.list_hosted_zones_by_name(DNSName=response.domain) matches = listResponse.HostedZones; if matches.size != 0: logger.error("Route53 returned " + mathces.size + " matching hosted zones. Expected exactly one. Auth canceled.") @@ -62,7 +62,7 @@ class Authenticator(common.Plugin): { 'Action': 'UPSERT', 'ResourceRecordSet': { - 'Name': achall.chall.path[1:], + 'Name': achall.validation_domain_name(), 'Type': 'TXT', 'TTL': 300, 'ResourceRecords': [ From 80b491d11d40895cefd628b20cbd77fcc390d437 Mon Sep 17 00:00:00 2001 From: Miley Date: Mon, 8 Feb 2016 17:24:34 -0800 Subject: [PATCH 07/96] Readme update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e39616a52..6fc901580 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It's expected that the root hosted zone for the domain in question already exist pip install letsencrypt ``` -1. Install the letsencrypt-s3front plugin +1. Install the letsencrypt-route53 plugin ``` pip install letsencrypt-route53 From c4364f82fbbdd0dd1e15f6c5c56ac4e36b688212 Mon Sep 17 00:00:00 2001 From: Hugo Peixoto Date: Mon, 3 Oct 2016 20:08:08 +0100 Subject: [PATCH 08/96] Change package names --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 02d9e01d5..95580ea15 100644 --- a/setup.py +++ b/setup.py @@ -26,10 +26,10 @@ docs_extras = [ ] setup( - name='letsencrypt-route53', + name='hpeixoto-letsencrypt-route53', version=version, description="Route53 plugin for Let's Encrypt client", - url='https://github.com/mindstorms6/letsencrypt-route53', + url='https://github.com/lifeonmarspt/letsencrypt-route53', author="Breland Miley", author_email='breland@bdawg.org', license='Apache2.0', From 1a5f09f4cfd2b5a520ec1944d986469d5e5838a8 Mon Sep 17 00:00:00 2001 From: Hugo Peixoto Date: Mon, 3 Oct 2016 19:53:53 +0100 Subject: [PATCH 09/96] First working iteration --- letsencrypt_route53/authenticator.py | 119 +++++++++++++++++---------- setup.py | 4 +- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/letsencrypt_route53/authenticator.py b/letsencrypt_route53/authenticator.py index 95d24b0ab..95ade5916 100644 --- a/letsencrypt_route53/authenticator.py +++ b/letsencrypt_route53/authenticator.py @@ -1,23 +1,21 @@ """Route53 Let's Encrypt authenticator plugin.""" -import os import logging -import re -import subprocess +import time -import zope.component import zope.interface import boto3 from acme import challenges -from letsencrypt import errors from letsencrypt import interfaces from letsencrypt.plugins import common logger = logging.getLogger(__name__) +TTL = 30 + class Authenticator(common.Plugin): zope.interface.implements(interfaces.IAuthenticator) zope.interface.classProvides(interfaces.IPluginFactory) @@ -44,50 +42,85 @@ class Authenticator(common.Plugin): responses.append(self._perform_single(achall)) return responses + def _find_zone(self, r53, domain): + return max( + ( + zone for zone in r53.list_hosted_zones()["HostedZones"] + if (domain+".").endswith("."+zone["Name"]) + ), + key=lambda zone: len(zone["Name"]), + ) + + def _perform_single(self, achall): # provision the TXT record, using the domain name given. Assumes the hosted zone exits, else fails the challenge - response, validation = achall.response_and_validation() r53 = boto3.client('route53') - logger.info("Doing validation for " + response.domain) - listResponse = r53.list_hosted_zones_by_name(DNSName=response.domain) - matches = listResponse.HostedZones; - if matches.size != 0: - logger.error("Route53 returned " + mathces.size + " matching hosted zones. Expected exactly one. Auth canceled.") - return None - else: - r53.change_resource_record_sets(HostedZoneId=matches[0].Id, - ChangeBatch={ - 'Comment': 'Let\'s Entcrypt Change', - 'Changes': [ - { - 'Action': 'UPSERT', - 'ResourceRecordSet': { - 'Name': achall.validation_domain_name(), - 'Type': 'TXT', - 'TTL': 300, - 'ResourceRecords': [ - { - 'Value': validation - }, - ] - } - }, - ] - }) + logger.info("Doing validation for " + achall.domain) + listResponse = r53.list_hosted_zones_by_name(DNSName=achall.domain) - if response.simple_verify( - achall.chall, achall.domain, - achall.account_key.public_key(), self.config.http01_port): - return response - else: - logger.error( - "Self-verify of challenge failed, authorization abandoned!") + try: + zone = self._find_zone(r53, achall.domain) + except ValueError as e: + logger.error("Unable to find matching Route53 zone for domain " + achall.domain) return None + response, validation = achall.response_and_validation() + self._excute_r53_action(r53, achall, zone, validation, 'UPSERT', wait_for_change=True) + + for _ in xrange(TTL*2): + if response.simple_verify( + achall.chall, + achall.domain, + achall.account_key.public_key(), + ): + break + logger.info("Waiting for DNS propagation...") + time.sleep(1) + else: + logger.error("Unable to verify domain " + achall.domain) + return None + + return response + def cleanup(self, achalls): - # pylint: disable=missing-docstring,no-self-use,unused-argument - #TODO:Cleanup record  + # pylint: disable=missing-docstring r53 = boto3.client('route53') - #for achall in achalls: - # r53.delete_object(Bucket=self.conf('s3-bucket'), Key=achall.chall.path[1:]) + for achall in achalls: + try: + zone = self._find_zone(r53, achall.domain) + except ValueError: + logger.warn("Unable to find zone for " + achall.domain + ". Skipping cleanup.") + continue + + _, validation = achall.response_and_validation() + self._excute_r53_action(r53, achall, zone, validation, 'DELETE') return None + + + def _excute_r53_action(self, r53, achall, zone, validation, action, wait_for_change=False): + response = r53.change_resource_record_sets( + HostedZoneId=zone["Id"], + ChangeBatch={ + 'Comment': 'Let\'s Encrypt ' + action, + 'Changes': [ + { + 'Action': action, + 'ResourceRecordSet': { + 'Name': achall.validation_domain_name(achall.domain), + 'Type': 'TXT', + 'TTL': TTL, + 'ResourceRecords': [ + { + 'Value': '"' + validation + '"', + }, + ], + }, + }, + ], + }, + ) + + if wait_for_change: + while r53.get_change(Id=response["ChangeInfo"]["Id"])["ChangeInfo"]["Status"] == "PENDING": + logger.info("Waiting for " + action + " to propagate...") + time.sleep(1) diff --git a/setup.py b/setup.py index 95580ea15..172791aba 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,8 @@ from setuptools import find_packages version = '0.1.3' install_requires = [ - 'acme>=0.1.1', - 'letsencrypt>=0.1.1', + 'acme>=0.9.0.dev0', + 'letsencrypt>=0.9.0.dev0', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? 'setuptools', # pkg_resources From ebd2007e82c4dbf2b7f2e3d9905d8a8555f4de44 Mon Sep 17 00:00:00 2001 From: Hugo Peixoto Date: Mon, 3 Oct 2016 20:06:38 +0100 Subject: [PATCH 10/96] Add instructions and rationale --- README.md | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 6fc901580..e33b4e777 100644 --- a/README.md +++ b/README.md @@ -7,29 +7,33 @@ It's expected that the root hosted zone for the domain in question already exist ### Setup -1. Install the letsencrypt client [https://letsencrypt.readthedocs.org/en/latest/using.html#installation](https://letsencrypt.readthedocs.org/en/latest/using.html#installation) +1. Create a virtual environment - ``` - pip install letsencrypt - ``` +2. Make sure you have libssl-dev (or your regional equivalent) installed. -1. Install the letsencrypt-route53 plugin +3. Install by adding these to your requirements.txt file: - ``` - pip install letsencrypt-route53 - ``` +``` +--no-binary pycparser +-e git+https://github.com/certbot/certbot.git#egg=certbot +-e git+https://github.com/certbot/certbot.git#egg=acme&subdirectory=acme +hpeixoto-letsencrypt-route53 +``` + +We need DNS01 support in certbot, which is only available in master for now. +Additionally, pycparser suffers from +https://github.com/eliben/pycparser/issues/148, which is why we need to +recompile it, which depends on `libssl-dev`. ### How to use it -To generate a certificate and install it in a CloudFront distribution: -``` -AWS_ACCESS_KEY_ID="your_key" \ -AWS_SECRET_ACCESS_KEY="your_secret" \ -letsencrypt --agree-tos -a letsencrypt-route53:auth \ --d the_domain -``` +Make sure you have access to AWS's Route53 service, either through IAM roles or +via `.aws/credentials`. -Follow the screen prompts and you should end up with the certificate in your -distribution. It may take a couple minutes to update. - -To automate the renewal process without prompts (for example, with a monthly cron), you can add the letsencrypt parameters --renew-by-default --text +To generate a certificate: +``` +letsencrypt certonly \ + -n --agree-tos --email DEVOPS@COMPANY.COM \ + -a hpeixoto-letsencrypt-route53:auth \ + -d MY.DOMAIN.NAME +``` From 108903dd26f73f60a1e118a9523089d85070fc2d Mon Sep 17 00:00:00 2001 From: Hugo Peixoto Date: Mon, 3 Oct 2016 20:07:37 +0100 Subject: [PATCH 11/96] Bump version to 0.1.4 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 172791aba..20c18cde9 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.1.3' +version = '0.1.4' install_requires = [ 'acme>=0.9.0.dev0', From 4538766c480dd2d54bcb990c0b9f7e78f89ea152 Mon Sep 17 00:00:00 2001 From: Paulo Koch Date: Tue, 4 Oct 2016 14:20:12 +0100 Subject: [PATCH 12/96] Make it work as certbot-route53 --- MANIFEST.in | 2 +- README.md | 14 ++++++------ .../__init__.py | 0 .../authenticator.py | 5 ++--- sample-aws-policy.json | 2 +- setup.py | 22 +++++++++---------- 6 files changed, 22 insertions(+), 23 deletions(-) rename {letsencrypt_route53 => certbot_route53}/__init__.py (100%) rename {letsencrypt_route53 => certbot_route53}/authenticator.py (96%) diff --git a/MANIFEST.in b/MANIFEST.in index 4faaf21cd..568ab3f2e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include LICENSE.txt include README.md recursive-include docs * -recursive-include letsencrypt_route53/tests/testdata * +recursive-include certbot_route53/tests/testdata * diff --git a/README.md b/README.md index e33b4e777..59af3615c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,9 @@ It's expected that the root hosted zone for the domain in question already exist 1. Create a virtual environment 2. Make sure you have libssl-dev (or your regional equivalent) installed. +`pycparser` suffers from +https://github.com/eliben/pycparser/issues/148, which is why we need to +recompile it, which depends on `libssl-dev`. 3. Install by adding these to your requirements.txt file: @@ -17,13 +20,10 @@ It's expected that the root hosted zone for the domain in question already exist --no-binary pycparser -e git+https://github.com/certbot/certbot.git#egg=certbot -e git+https://github.com/certbot/certbot.git#egg=acme&subdirectory=acme -hpeixoto-letsencrypt-route53 +certbot-route53 ``` - We need DNS01 support in certbot, which is only available in master for now. -Additionally, pycparser suffers from -https://github.com/eliben/pycparser/issues/148, which is why we need to -recompile it, which depends on `libssl-dev`. + ### How to use it @@ -32,8 +32,8 @@ via `.aws/credentials`. To generate a certificate: ``` -letsencrypt certonly \ +certbot certonly \ -n --agree-tos --email DEVOPS@COMPANY.COM \ - -a hpeixoto-letsencrypt-route53:auth \ + -a certbot-route53:auth \ -d MY.DOMAIN.NAME ``` diff --git a/letsencrypt_route53/__init__.py b/certbot_route53/__init__.py similarity index 100% rename from letsencrypt_route53/__init__.py rename to certbot_route53/__init__.py diff --git a/letsencrypt_route53/authenticator.py b/certbot_route53/authenticator.py similarity index 96% rename from letsencrypt_route53/authenticator.py rename to certbot_route53/authenticator.py index 95ade5916..2b941a7e5 100644 --- a/letsencrypt_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -8,8 +8,8 @@ import boto3 from acme import challenges -from letsencrypt import interfaces -from letsencrypt.plugins import common +from certbot import interfaces +from certbot.plugins import common logger = logging.getLogger(__name__) @@ -56,7 +56,6 @@ class Authenticator(common.Plugin): # provision the TXT record, using the domain name given. Assumes the hosted zone exits, else fails the challenge r53 = boto3.client('route53') logger.info("Doing validation for " + achall.domain) - listResponse = r53.list_hosted_zones_by_name(DNSName=achall.domain) try: zone = self._find_zone(r53, achall.domain) diff --git a/sample-aws-policy.json b/sample-aws-policy.json index 1af66c85d..59c8fb7c7 100644 --- a/sample-aws-policy.json +++ b/sample-aws-policy.json @@ -1,6 +1,6 @@ { "Version": "2012-10-17", - "Id": "letsencrypt-route53 sample policy", + "Id": "certbot-route53 sample policy", "Statement": [ { "Effect": "Allow", diff --git a/setup.py b/setup.py index 20c18cde9..2aa0497f0 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,13 @@ version = '0.1.4' install_requires = [ 'acme>=0.9.0.dev0', - 'letsencrypt>=0.9.0.dev0', + 'certbot>=0.9.0.dev0', 'PyOpenSSL', 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? 'setuptools', # pkg_resources 'zope.interface', - 'boto3' + 'boto3', + 'dnspython', ] if sys.version_info < (2, 7): @@ -26,12 +27,12 @@ docs_extras = [ ] setup( - name='hpeixoto-letsencrypt-route53', + name='certbot-route53', version=version, - description="Route53 plugin for Let's Encrypt client", - url='https://github.com/lifeonmarspt/letsencrypt-route53', - author="Breland Miley", - author_email='breland@bdawg.org', + description="Route53 plugin for certbot", + url='https://github.com/lifeonmarspt/certbot-route53', + author="Hugo Peixoto", + author_email='hugo@lifeonmars.pt', license='Apache2.0', classifiers=[ 'Development Status :: 3 - Alpha', @@ -41,7 +42,6 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', @@ -53,10 +53,10 @@ setup( packages=find_packages(), include_package_data=True, install_requires=install_requires, - keywords = ['letsencrypt', 'route53', 'aws'], + keywords=['certbot', 'route53', 'aws'], entry_points={ - 'letsencrypt.plugins': [ - 'auth = letsencrypt_route53.authenticator:Authenticator' + 'certbot.plugins': [ + 'auth = certbot_route53.authenticator:Authenticator' ], }, ) From ebe5d0c4f28e9fc102a18265e28b7b9b9efbacc9 Mon Sep 17 00:00:00 2001 From: Waylon Flinn Date: Fri, 4 Nov 2016 20:26:34 -0500 Subject: [PATCH 13/96] add support for root domain --- certbot_route53/authenticator.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index 2b941a7e5..732080eeb 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -46,7 +46,7 @@ class Authenticator(common.Plugin): return max( ( zone for zone in r53.list_hosted_zones()["HostedZones"] - if (domain+".").endswith("."+zone["Name"]) + if (domain+".").endswith("."+zone["Name"]) or (domain+".") == (zone["Name"]) ), key=lambda zone: len(zone["Name"]), ) diff --git a/setup.py b/setup.py index 2aa0497f0..4b9b754a8 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import sys from distutils.core import setup from setuptools import find_packages -version = '0.1.4' +version = '0.1.5' install_requires = [ 'acme>=0.9.0.dev0', From 17c4c7f68efa54f63f7c448d372e98b14939f0ca Mon Sep 17 00:00:00 2001 From: Chow Loong Jin Date: Mon, 27 Feb 2017 15:27:32 +0800 Subject: [PATCH 14/96] Use zope decorators This makes it compatible with python3. --- certbot_route53/authenticator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index 2b941a7e5..915c0092d 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -16,9 +16,9 @@ logger = logging.getLogger(__name__) TTL = 30 +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): - zope.interface.implements(interfaces.IAuthenticator) - zope.interface.classProvides(interfaces.IPluginFactory) description = "Route53 Authenticator" From e9531dc80b75421b73f643b60c0780b6ee6fc92e Mon Sep 17 00:00:00 2001 From: Chow Loong Jin Date: Mon, 27 Feb 2017 16:57:33 +0800 Subject: [PATCH 15/96] Replace xrange with range --- certbot_route53/authenticator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index 915c0092d..7dbe8d80b 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -66,7 +66,7 @@ class Authenticator(common.Plugin): response, validation = achall.response_and_validation() self._excute_r53_action(r53, achall, zone, validation, 'UPSERT', wait_for_change=True) - for _ in xrange(TTL*2): + for _ in range(TTL*2): if response.simple_verify( achall.chall, achall.domain, From 1143ab7446ff33d5f1270235ef50165d6ea671c4 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Sat, 11 Mar 2017 10:30:39 -0800 Subject: [PATCH 16/96] Documentation and efficiency changes. These are from certbot/certbot#4174 Add more documentation, and help for NoCredentialsError. Allow multiple DNS records to be provisioned at once and waited for together. Fix doc strings to use "Certbot" instead of "Let's Encrypt." Set TTL to 0. Create a single boto3 session rather than one per API call. Use pagination in Route53 API in case there are many domains. Add a maximum wait time for update to propagate (10 minutes). --- certbot_route53/__init__.py | 2 +- certbot_route53/authenticator.py | 186 +++++++++++++++++-------------- 2 files changed, 103 insertions(+), 85 deletions(-) diff --git a/certbot_route53/__init__.py b/certbot_route53/__init__.py index 6f519023b..c91c79c22 100644 --- a/certbot_route53/__init__.py +++ b/certbot_route53/__init__.py @@ -1 +1 @@ -"""Let's Encrypt Route53 plugin.""" +"""Certbot Route53 plugin.""" diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index d383044c0..c27d28204 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -1,10 +1,12 @@ -"""Route53 Let's Encrypt authenticator plugin.""" +"""Certbot Route53 authenticator plugin.""" import logging import time +import datetime import zope.interface import boto3 +from botocore.exceptions import NoCredentialsError from acme import challenges @@ -14,112 +16,128 @@ from certbot.plugins import common logger = logging.getLogger(__name__) -TTL = 30 +INSTRUCTIONS = ( + "To use, create an IAM user and attach the AmazonRoute53FullAccess policy, then store " + "the access key ID and secret key in ~/.aws/credentials or in " + "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as described at " + "https://boto3.readthedocs.io/en/latest/guide/configuration.html") @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) class Authenticator(common.Plugin): + """Route53 Authenticator - description = "Route53 Authenticator" + This authenticator solves a DNS01 challenge by uploading the answer to AWS + Route53. + """ + + description = ("Authenticate domain names using the DNS challenge type, " + "by automatically updating TXT records using AWS Route53. Works only " + "if you use AWS Route53 to host DNS for your domains. " + + INSTRUCTIONS) def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - self._httpd = None + session = boto3.Session() + self.route53_client = session.client("route53") + # A list of (dns name, TXT value) tuples, for cleanup. + self.txt_records = [] def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover def more_info(self): # pylint: disable=missing-docstring,no-self-use - return ("") + return "Solve a DNS01 challenge using AWS Route53" def get_chall_pref(self, domain): # pylint: disable=missing-docstring,no-self-use,unused-argument return [challenges.DNS01] def perform(self, achalls): # pylint: disable=missing-docstring - responses = [] - for achall in achalls: - responses.append(self._perform_single(achall)) - return responses - - def _find_zone(self, r53, domain): - return max( - ( - zone for zone in r53.list_hosted_zones()["HostedZones"] - if (domain+".").endswith("."+zone["Name"]) or (domain+".") == (zone["Name"]) - ), - key=lambda zone: len(zone["Name"]), - ) - - - def _perform_single(self, achall): - # provision the TXT record, using the domain name given. Assumes the hosted zone exits, else fails the challenge - r53 = boto3.client('route53') - logger.info("Doing validation for " + achall.domain) - try: - zone = self._find_zone(r53, achall.domain) - except ValueError as e: - logger.error("Unable to find matching Route53 zone for domain " + achall.domain) - return None + change_ids = [self._create_single(achall) for achall in achalls] + for change_id in change_ids: + self._wait_for_change(change_id) + return [achall.response(achall.account_key) for achall in achalls] + except NoCredentialsError: + raise Exception("No AWS Route53 credentials found. " + INSTRUCTIONS) - response, validation = achall.response_and_validation() - self._excute_r53_action(r53, achall, zone, validation, 'UPSERT', wait_for_change=True) + def cleanup(self, achalls): # pylint: disable=missing-docstring + for name, value in self.txt_records: + self._delete_txt_record(name, value) - for _ in range(TTL*2): - if response.simple_verify( - achall.chall, - achall.domain, - achall.account_key.public_key(), - ): - break - logger.info("Waiting for DNS propagation...") - time.sleep(1) - else: - logger.error("Unable to verify domain " + achall.domain) - return None + def _create_single(self, achall): + """Create a TXT record, return a change_id""" + name, value = (achall.validation_domain_name(achall.domain), + achall.validation(achall.account_key)) + change_id = self._create_txt_record(name, value) + self.txt_records.append((name, value)) + return change_id - return response + def _find_zone_id_for_domain(self, domain): + paginator = self.route53_client.get_paginator("list_hosted_zones") + zones = [] + for page in paginator.paginate(): + for zone in page["HostedZones"]: + if ( + domain.endswith(zone["Name"]) or + (domain + ".").endswith(zone["Name"]) + ) and not zone["Config"]["PrivateZone"]: + zones.append((zone["Name"], zone["Id"])) - def cleanup(self, achalls): - # pylint: disable=missing-docstring - r53 = boto3.client('route53') - for achall in achalls: - try: - zone = self._find_zone(r53, achall.domain) - except ValueError: - logger.warn("Unable to find zone for " + achall.domain + ". Skipping cleanup.") - continue - - _, validation = achall.response_and_validation() - self._excute_r53_action(r53, achall, zone, validation, 'DELETE') - return None - - - def _excute_r53_action(self, r53, achall, zone, validation, action, wait_for_change=False): - response = r53.change_resource_record_sets( - HostedZoneId=zone["Id"], - ChangeBatch={ - 'Comment': 'Let\'s Encrypt ' + action, - 'Changes': [ - { - 'Action': action, - 'ResourceRecordSet': { - 'Name': achall.validation_domain_name(achall.domain), - 'Type': 'TXT', - 'TTL': TTL, - 'ResourceRecords': [ - { - 'Value': '"' + validation + '"', - }, - ], - }, - }, - ], - }, + if not zones: + raise ValueError( + "Unable to find a Route53 hosted zone for {}".format(domain) ) - if wait_for_change: - while r53.get_change(Id=response["ChangeInfo"]["Id"])["ChangeInfo"]["Status"] == "PENDING": - logger.info("Waiting for " + action + " to propagate...") - time.sleep(1) + # Order the zones that are suffixes for our desired to domain by + # length, this puts them in an order like: + # ["foo.bar.baz.com", "bar.baz.com", "baz.com", "com"] + # And then we choose the first one, which will be the most specific. + zones.sort(key=lambda z: len(z[0]), reverse=True) + return zones[0][1] + + def _change_txt_record(self, action, zone_id, domain, value): + response = self.route53_client.change_resource_record_sets( + HostedZoneId=zone_id, + ChangeBatch={ + "Comment": "certbot-route53 certificate validation " + action, + "Changes": [ + { + "Action": action, + "ResourceRecordSet": { + "Name": domain, + "Type": "TXT", + "TTL": 0, + "ResourceRecords": [ + # For some reason TXT records need to be + # manually quoted. + {"Value": '"{}"'.format(value)} + ], + } + } + ] + } + ) + return response["ChangeInfo"]["Id"] + + def _create_txt_record(self, host, value): + zone_id = self._find_zone_id_for_domain(host) + change_id = self._change_txt_record("UPSERT", zone_id, host, value) + return change_id + + def _delete_txt_record(self, host, value): + zone_id = self._find_zone_id_for_domain(host) + change_id = self._change_txt_record("DELETE", zone_id, host, value) + return change_id + + def _wait_for_change(self, change_id): + deadline = datetime.datetime.now() + datetime.timedelta(minutes=10) + while datetime.datetime.now() < deadline: + response = self.route53_client.get_change(Id=change_id) + if response["ChangeInfo"]["Status"] == "INSYNC": + return + time.sleep(5) + raise Exception( + "Timed out waiting for Route53 change. Current status: %s" % + response["ChangeInfo"]["Status"]) From b3a28869c89470943373e8b98417d7e45562ac7d Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 14 Mar 2017 17:50:51 -0700 Subject: [PATCH 17/96] Respond to review feedback. --- certbot_route53/authenticator.py | 53 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index c27d28204..c04299338 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -38,8 +38,6 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - session = boto3.Session() - self.route53_client = session.client("route53") # A list of (dns name, TXT value) tuples, for cleanup. self.txt_records = [] @@ -64,25 +62,34 @@ class Authenticator(common.Plugin): def cleanup(self, achalls): # pylint: disable=missing-docstring for name, value in self.txt_records: - self._delete_txt_record(name, value) + self._change_txt_record("DELETE", name, value) def _create_single(self, achall): """Create a TXT record, return a change_id""" - name, value = (achall.validation_domain_name(achall.domain), - achall.validation(achall.account_key)) - change_id = self._create_txt_record(name, value) + name = achall.validation_domain_name(achall.domain) + value = achall.validation(achall.account_key) + change_id = self._change_txt_record("UPSERT", name, value) self.txt_records.append((name, value)) return change_id def _find_zone_id_for_domain(self, domain): - paginator = self.route53_client.get_paginator("list_hosted_zones") + """Find the zone id responsible a given FQDN. + + That is, the id for the zone whose name is the longest parent of the + domain. + + domain should not have a trailing dot. + """ + client = boto3.client("route53") + paginator = client.get_paginator("list_hosted_zones") zones = [] + target_labels = domain.split(".") for page in paginator.paginate(): for zone in page["HostedZones"]: - if ( - domain.endswith(zone["Name"]) or - (domain + ".").endswith(zone["Name"]) - ) and not zone["Config"]["PrivateZone"]: + if zone["Config"]["PrivateZone"]: + continue + candidate_labels = zone["Name"].rstrip(".").split(".") + if candidate_labels == target_labels[-len(candidate_labels):]: zones.append((zone["Name"], zone["Id"])) if not zones: @@ -97,8 +104,10 @@ class Authenticator(common.Plugin): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] - def _change_txt_record(self, action, zone_id, domain, value): - response = self.route53_client.change_resource_record_sets( + def _change_txt_record(self, action, domain, value): + zone_id = self._find_zone_id_for_domain(domain) + client = boto3.client("route53") + response = client.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ "Comment": "certbot-route53 certificate validation " + action, @@ -108,7 +117,7 @@ class Authenticator(common.Plugin): "ResourceRecordSet": { "Name": domain, "Type": "TXT", - "TTL": 0, + "TTL": 10, "ResourceRecords": [ # For some reason TXT records need to be # manually quoted. @@ -121,20 +130,10 @@ class Authenticator(common.Plugin): ) return response["ChangeInfo"]["Id"] - def _create_txt_record(self, host, value): - zone_id = self._find_zone_id_for_domain(host) - change_id = self._change_txt_record("UPSERT", zone_id, host, value) - return change_id - - def _delete_txt_record(self, host, value): - zone_id = self._find_zone_id_for_domain(host) - change_id = self._change_txt_record("DELETE", zone_id, host, value) - return change_id - def _wait_for_change(self, change_id): - deadline = datetime.datetime.now() + datetime.timedelta(minutes=10) - while datetime.datetime.now() < deadline: - response = self.route53_client.get_change(Id=change_id) + for n in range(0, 120): + client = boto3.client("route53") + response = client.get_change(Id=change_id) if response["ChangeInfo"]["Status"] == "INSYNC": return time.sleep(5) From 4a3aa8dd11b0efda7d5a6d59d7c823d7b3eb1be1 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Tue, 14 Mar 2017 17:57:47 -0700 Subject: [PATCH 18/96] Remove documentation about creating IAM users. --- certbot_route53/authenticator.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index c04299338..fabf2dec4 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -17,10 +17,9 @@ from certbot.plugins import common logger = logging.getLogger(__name__) INSTRUCTIONS = ( - "To use, create an IAM user and attach the AmazonRoute53FullAccess policy, then store " - "the access key ID and secret key in ~/.aws/credentials or in " - "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, as described at " - "https://boto3.readthedocs.io/en/latest/guide/configuration.html") + "To use, configure credentials as described at " + "https://boto3.readthedocs.io/en/latest/guide/configuration.html#best-practices-for-configuring-credentials " + "and add the necessary permissions for Route53 access") @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) From cb720b06182812c692c3073d5c4c007b5909273c Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 15 Mar 2017 11:32:47 -0700 Subject: [PATCH 19/96] Address review feedback. --- certbot_route53/authenticator.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index fabf2dec4..82f42281e 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -6,7 +6,7 @@ import datetime import zope.interface import boto3 -from botocore.exceptions import NoCredentialsError +from botocore.exceptions import NoCredentialsError, ClientError from acme import challenges @@ -16,10 +16,12 @@ from certbot.plugins import common logger = logging.getLogger(__name__) +TTL = 10 + INSTRUCTIONS = ( - "To use, configure credentials as described at " + "To use certbot-route53, configure credentials as described at " "https://boto3.readthedocs.io/en/latest/guide/configuration.html#best-practices-for-configuring-credentials " - "and add the necessary permissions for Route53 access") + "and add the necessary permissions for Route53 access.") @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) @@ -37,8 +39,6 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) - # A list of (dns name, TXT value) tuples, for cleanup. - self.txt_records = [] def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover @@ -58,9 +58,13 @@ class Authenticator(common.Plugin): return [achall.response(achall.account_key) for achall in achalls] except NoCredentialsError: raise Exception("No AWS Route53 credentials found. " + INSTRUCTIONS) + except ClientError as e: + raise Exception(str(e) + "\n" + INSTRUCTIONS) def cleanup(self, achalls): # pylint: disable=missing-docstring - for name, value in self.txt_records: + for achall in achalls: + name = achall.validation_domain_name(achall.domain) + value = achall.validation(achall.account_key) self._change_txt_record("DELETE", name, value) def _create_single(self, achall): @@ -68,7 +72,6 @@ class Authenticator(common.Plugin): name = achall.validation_domain_name(achall.domain) value = achall.validation(achall.account_key) change_id = self._change_txt_record("UPSERT", name, value) - self.txt_records.append((name, value)) return change_id def _find_zone_id_for_domain(self, domain): @@ -76,13 +79,11 @@ class Authenticator(common.Plugin): That is, the id for the zone whose name is the longest parent of the domain. - - domain should not have a trailing dot. """ client = boto3.client("route53") paginator = client.get_paginator("list_hosted_zones") zones = [] - target_labels = domain.split(".") + target_labels = domain.rstrip(".").split(".") for page in paginator.paginate(): for zone in page["HostedZones"]: if zone["Config"]["PrivateZone"]: @@ -116,7 +117,7 @@ class Authenticator(common.Plugin): "ResourceRecordSet": { "Name": domain, "Type": "TXT", - "TTL": 10, + "TTL": TTL, "ResourceRecords": [ # For some reason TXT records need to be # manually quoted. @@ -130,8 +131,15 @@ class Authenticator(common.Plugin): return response["ChangeInfo"]["Id"] def _wait_for_change(self, change_id): + """Wait for TTL of any previous attempt to expire, then for INSYNC. + + Once Route53 returns INSYNC, challenge record is ready on all Route53 + DNS servers: + https://docs.aws.amazon.com/Route53/latest/APIReference/API_GetChange.html + """ + time.sleep(TTL) + client = boto3.client("route53") for n in range(0, 120): - client = boto3.client("route53") response = client.get_change(Id=change_id) if response["ChangeInfo"]["Status"] == "INSYNC": return From d67de61ad8c41bdf9d55768616bbb1a320972c77 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 15 Mar 2017 11:38:26 -0700 Subject: [PATCH 20/96] Move sleep(TTL) into perform. This means we only do it once, even when there are many hostnames. --- certbot_route53/authenticator.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index 82f42281e..fe87f96ea 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -53,6 +53,11 @@ class Authenticator(common.Plugin): def perform(self, achalls): # pylint: disable=missing-docstring try: change_ids = [self._create_single(achall) for achall in achalls] + # Sleep for at least the TTL, to ensure that any records cached by + # the ACME server after previous validation attempts are gone. In + # most cases we'll need to wait at least this long for the Route53 + # records to propagate, so this doesn't delay us much. + time.sleep(TTL) for change_id in change_ids: self._wait_for_change(change_id) return [achall.response(achall.account_key) for achall in achalls] @@ -131,13 +136,9 @@ class Authenticator(common.Plugin): return response["ChangeInfo"]["Id"] def _wait_for_change(self, change_id): - """Wait for TTL of any previous attempt to expire, then for INSYNC. - - Once Route53 returns INSYNC, challenge record is ready on all Route53 - DNS servers: + """Wait for a change to be propagated to all Route53 DNS servers. https://docs.aws.amazon.com/Route53/latest/APIReference/API_GetChange.html """ - time.sleep(TTL) client = boto3.client("route53") for n in range(0, 120): response = client.get_change(Id=change_id) From 64a7608956f1cd19924d12312cba6f8f6bcbb94d Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 15 Mar 2017 17:19:00 -0700 Subject: [PATCH 21/96] Add auto-ness to the UA --- certbot/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 0f6b35f09..e1d578087 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -52,9 +52,10 @@ def determine_user_agent(config): """ if config.user_agent is None: - ua = "CertbotACMEClient/{0} ({1}) Authenticator/{2} Installer/{3}" + auto = "" if cli.cli_command == "certbot" else "auto" + ua = "CertbotACMEClient{4}/{0} ({1}) Authenticator/{2} Installer/{3}" ua = ua.format(certbot.__version__, util.get_os_info_ua(), - config.authenticator, config.installer) + config.authenticator, config.installer, auto) else: ua = config.user_agent return ua From 6fa521bc5f82f19f187c870cf633b1a43a6a6001 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 15 Mar 2017 18:02:03 -0700 Subject: [PATCH 22/96] Also report if we're renewing --- certbot/cli.py | 1 + certbot/client.py | 8 +++++--- certbot/main.py | 13 +++++++++---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index b9172f21d..abcac6c94 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -526,6 +526,7 @@ class HelpfulArgumentParser(object): parsed_args = self.parser.parse_args(self.args) parsed_args.func = self.VERBS[self.verb] parsed_args.verb = self.verb + parsed_args.renewing = None # an important config property we don't know until later if self.detect_defaults: return parsed_args diff --git a/certbot/client.py b/certbot/client.py index 95de3ccb1..ab13da1f0 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -54,10 +54,11 @@ def determine_user_agent(config): if config.user_agent is None: auto = "no" if cli.cli_command == "certbot" else "yes" - ua = "CertbotACMEClient/{0} ({1}) Authenticator/{2} Installer/{3} Verb/{4} Auto/{5} Py/{6}" + ua = ("CertbotACMEClient/{0} ({1}) Authenticator/{2} Installer/{3} Verb/{4} " + "Renewing/{5} Auto/{6} Py/{7}") ua = ua.format(certbot.__version__, util.get_os_info_ua(), - config.authenticator, config.installer, config.verb, auto, - platform.python_version()) + config.authenticator, config.installer, config.verb, config.renewing, + auto, platform.python_version()) else: ua = config.user_agent return ua @@ -71,6 +72,7 @@ def sample_user_agent(): self.installer = "YYY" self.user_agent = None self.verb = "SUBCOMMAND" + self.renewing = "YESNOMAYBE" return determine_user_agent(DummyConfig()) diff --git a/certbot/main.py b/certbot/main.py index 1f247a7d6..43cc0b101 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -225,6 +225,10 @@ def _find_cert(config, domains, certname): RenewableCert instance or None. """ action, lineage = _find_lineage_for_domains_and_certname(config, domains, certname) + if action == "renew": + config.renewing = True + elif action == "newcert": + config.renewing = False if action == "reinstall": logger.info("Keeping the existing certificate") return (action != "reinstall"), lineage @@ -595,12 +599,12 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals except errors.PluginSelectionError as e: return e.message - # TODO: Handle errors from _init_le_client? - le_client = _init_le_client(config, authenticator, installer) - domains, certname = _find_domains_or_certname(config, installer) should_get_cert, lineage = _find_cert(config, domains, certname) + # TODO: Handle errors from _init_le_client? + le_client = _init_le_client(config, authenticator, installer) + new_lineage = lineage if should_get_cert: new_lineage = _get_and_save_cert(le_client, config, domains, @@ -673,9 +677,9 @@ def certonly(config, plugins): except errors.PluginSelectionError as e: logger.info("Could not choose appropriate plugin: %s", e) raise - le_client = _init_le_client(config, auth, installer) if config.csr: + le_client = _init_le_client(config, auth, installer) cert_path, fullchain_path = _csr_get_and_save_cert(config, le_client) _report_new_cert(config, cert_path, fullchain_path) _suggest_donation_if_appropriate(config) @@ -683,6 +687,7 @@ def certonly(config, plugins): domains, certname = _find_domains_or_certname(config, installer) should_get_cert, lineage = _find_cert(config, domains, certname) + le_client = _init_le_client(config, auth, installer) # run after _find_cert for config.renewing if possible if not should_get_cert: notify = zope.component.getUtility(interfaces.IDisplay).notification From f259a1754944dda291083d3c2155ecb932cc6fb7 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 16 Mar 2017 01:14:06 -0700 Subject: [PATCH 23/96] Lint --- certbot/cli.py | 2 +- certbot/main.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index abcac6c94..88b53796b 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -526,7 +526,7 @@ class HelpfulArgumentParser(object): parsed_args = self.parser.parse_args(self.args) parsed_args.func = self.VERBS[self.verb] parsed_args.verb = self.verb - parsed_args.renewing = None # an important config property we don't know until later + parsed_args.renewing = "unknown" # an important config property we don't know until later if self.detect_defaults: return parsed_args diff --git a/certbot/main.py b/certbot/main.py index 43cc0b101..d0dc0721a 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -687,7 +687,8 @@ def certonly(config, plugins): domains, certname = _find_domains_or_certname(config, installer) should_get_cert, lineage = _find_cert(config, domains, certname) - le_client = _init_le_client(config, auth, installer) # run after _find_cert for config.renewing if possible + # _init_le_client needs to run after _find_cert for config.renewing if possible + le_client = _init_le_client(config, auth, installer) if not should_get_cert: notify = zope.component.getUtility(interfaces.IDisplay).notification From 1542bce2613b9e5c6664a63b65c4b3b0331ba629 Mon Sep 17 00:00:00 2001 From: Paulo Koch Date: Thu, 16 Mar 2017 14:01:11 +0000 Subject: [PATCH 24/96] Fix the sample policy --- sample-aws-policy.json | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/sample-aws-policy.json b/sample-aws-policy.json index 59c8fb7c7..0b4dcae41 100644 --- a/sample-aws-policy.json +++ b/sample-aws-policy.json @@ -5,19 +5,8 @@ { "Effect": "Allow", "Action": [ - "iam:UploadServerCertificate", - "iam:UpdateServerCertificate", - "iam:DeleteServerCertificate" - ], - "Resource": [ - "*" - ] - }, - { - "Effect": "Allow", - "Action": [ - "route53:List*", - "route53:Get*", + "route53:ListHostedZones", + "route53:GetChange" ], "Resource": [ "*" From 3f7efbfa3c4dbcc1b2a191bf9cbdc56a1df0c424 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Mar 2017 13:26:42 -0700 Subject: [PATCH 25/96] Sleep after wait; stack trace --- certbot_route53/authenticator.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index fe87f96ea..c53d5366a 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -53,18 +53,17 @@ class Authenticator(common.Plugin): def perform(self, achalls): # pylint: disable=missing-docstring try: change_ids = [self._create_single(achall) for achall in achalls] + for change_id in change_ids: + self._wait_for_change(change_id) # Sleep for at least the TTL, to ensure that any records cached by # the ACME server after previous validation attempts are gone. In # most cases we'll need to wait at least this long for the Route53 # records to propagate, so this doesn't delay us much. time.sleep(TTL) - for change_id in change_ids: - self._wait_for_change(change_id) return [achall.response(achall.account_key) for achall in achalls] - except NoCredentialsError: - raise Exception("No AWS Route53 credentials found. " + INSTRUCTIONS) - except ClientError as e: - raise Exception(str(e) + "\n" + INSTRUCTIONS) + except (NoCredentialsError, ClientError) as e: + e.args = ("\n".join([str(e), INSTRUCTIONS]),) + raise def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: From 8850bd126ba811f2f498c9082e0da653e5031e24 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Fri, 17 Mar 2017 13:30:47 -0700 Subject: [PATCH 26/96] Final review feedback. --- certbot_route53/authenticator.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/certbot_route53/authenticator.py b/certbot_route53/authenticator.py index c53d5366a..855165455 100644 --- a/certbot_route53/authenticator.py +++ b/certbot_route53/authenticator.py @@ -39,6 +39,7 @@ class Authenticator(common.Plugin): def __init__(self, *args, **kwargs): super(Authenticator, self).__init__(*args, **kwargs) + self.r53 = boto3.client("route53") def prepare(self): # pylint: disable=missing-docstring,no-self-use pass # pragma: no cover @@ -52,7 +53,11 @@ class Authenticator(common.Plugin): def perform(self, achalls): # pylint: disable=missing-docstring try: - change_ids = [self._create_single(achall) for achall in achalls] + change_ids = [ + self._change_txt_record("UPSERT", achall) + for achall in achalls + ] + for change_id in change_ids: self._wait_for_change(change_id) # Sleep for at least the TTL, to ensure that any records cached by @@ -67,16 +72,7 @@ class Authenticator(common.Plugin): def cleanup(self, achalls): # pylint: disable=missing-docstring for achall in achalls: - name = achall.validation_domain_name(achall.domain) - value = achall.validation(achall.account_key) - self._change_txt_record("DELETE", name, value) - - def _create_single(self, achall): - """Create a TXT record, return a change_id""" - name = achall.validation_domain_name(achall.domain) - value = achall.validation(achall.account_key) - change_id = self._change_txt_record("UPSERT", name, value) - return change_id + self._change_txt_record("DELETE", achall) def _find_zone_id_for_domain(self, domain): """Find the zone id responsible a given FQDN. @@ -84,14 +80,14 @@ class Authenticator(common.Plugin): That is, the id for the zone whose name is the longest parent of the domain. """ - client = boto3.client("route53") - paginator = client.get_paginator("list_hosted_zones") + paginator = self.r53.get_paginator("list_hosted_zones") zones = [] target_labels = domain.rstrip(".").split(".") for page in paginator.paginate(): for zone in page["HostedZones"]: if zone["Config"]["PrivateZone"]: continue + candidate_labels = zone["Name"].rstrip(".").split(".") if candidate_labels == target_labels[-len(candidate_labels):]: zones.append((zone["Name"], zone["Id"])) @@ -108,10 +104,13 @@ class Authenticator(common.Plugin): zones.sort(key=lambda z: len(z[0]), reverse=True) return zones[0][1] - def _change_txt_record(self, action, domain, value): + def _change_txt_record(self, action, achall): + domain = achall.validation_domain_name(achall.domain) + value = achall.validation(achall.account_key) + zone_id = self._find_zone_id_for_domain(domain) - client = boto3.client("route53") - response = client.change_resource_record_sets( + + response = self.r53.change_resource_record_sets( HostedZoneId=zone_id, ChangeBatch={ "Comment": "certbot-route53 certificate validation " + action, From 4846217445af161cd14ed76e3eb6ca8f521080ad Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 17 Mar 2017 15:20:24 -0700 Subject: [PATCH 27/96] Make config.renewing always a string. --- certbot/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index d0dc0721a..bc61ef568 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -226,9 +226,9 @@ def _find_cert(config, domains, certname): """ action, lineage = _find_lineage_for_domains_and_certname(config, domains, certname) if action == "renew": - config.renewing = True + config.renewing = "Yes" elif action == "newcert": - config.renewing = False + config.renewing = "No" if action == "reinstall": logger.info("Keeping the existing certificate") return (action != "reinstall"), lineage From 1b65e17999b119369ff63fd9917d6e2fe2776b18 Mon Sep 17 00:00:00 2001 From: Paulo Koch Date: Fri, 17 Mar 2017 10:04:04 +0000 Subject: [PATCH 28/96] Tidy up installation --- LICENSE.txt | 1 - MANIFEST.in | 6 ++---- README.md | 27 ++++++++++++--------------- setup.cfg | 4 ++-- setup.py | 19 ++----------------- 5 files changed, 18 insertions(+), 39 deletions(-) delete mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 7ff097c3c..000000000 --- a/LICENSE.txt +++ /dev/null @@ -1 +0,0 @@ -See LICENSE diff --git a/MANIFEST.in b/MANIFEST.in index 568ab3f2e..9575a1c62 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,2 @@ -include LICENSE.txt -include README.md -recursive-include docs * -recursive-include certbot_route53/tests/testdata * +include LICENSE +include README diff --git a/README.md b/README.md index 59af3615c..7dab0b7f6 100644 --- a/README.md +++ b/README.md @@ -3,32 +3,29 @@ ### Before you start -It's expected that the root hosted zone for the domain in question already exists in your account. +It's expected that the root hosted zone for the domain in question already +exists in your account. ### Setup 1. Create a virtual environment -2. Make sure you have libssl-dev (or your regional equivalent) installed. -`pycparser` suffers from -https://github.com/eliben/pycparser/issues/148, which is why we need to -recompile it, which depends on `libssl-dev`. +2. Update its pip and setuptools (`VENV/bin/pip install -U setuptools pip`) +to avoid problems with cryptography's dependency on setuptools>=11.3. -3. Install by adding these to your requirements.txt file: - -``` ---no-binary pycparser --e git+https://github.com/certbot/certbot.git#egg=certbot --e git+https://github.com/certbot/certbot.git#egg=acme&subdirectory=acme -certbot-route53 -``` -We need DNS01 support in certbot, which is only available in master for now. +3. Make sure you have libssl-dev and libffi (or your regional equivalents) +installed. You might have to set compiler flags to pick things up (I have to +use `CPPFLAGS=-I/usr/local/opt/openssl/include +LDFLAGS=-L/usr/local/opt/openssl/lib` on my macOS to pick up brew's openssl, +for example). +4. Install this package. ### How to use it Make sure you have access to AWS's Route53 service, either through IAM roles or -via `.aws/credentials`. +via `.aws/credentials`. Check out +(sample-aws-policy.json)[sample-aws-policy.json]. To generate a certificate: ``` diff --git a/setup.cfg b/setup.cfg index b88034e41..3c6e79cf3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ -[metadata] -description-file = README.md +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index 4b9b754a8..49b1ea467 100644 --- a/setup.py +++ b/setup.py @@ -6,24 +6,10 @@ from setuptools import find_packages version = '0.1.5' install_requires = [ - 'acme>=0.9.0.dev0', - 'certbot>=0.9.0.dev0', - 'PyOpenSSL', - 'pyparsing>=1.5.5', # Python3 support; perhaps unnecessary? - 'setuptools', # pkg_resources + 'acme>=0.9.0', + 'certbot>=0.9.0', 'zope.interface', 'boto3', - 'dnspython', -] - -if sys.version_info < (2, 7): - install_requires.append('mock<1.1.0') -else: - install_requires.append('mock') - -docs_extras = [ - 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags - 'sphinx_rtd_theme', ] setup( @@ -51,7 +37,6 @@ setup( 'Topic :: Utilities', ], packages=find_packages(), - include_package_data=True, install_requires=install_requires, keywords=['certbot', 'route53', 'aws'], entry_points={ From 08932836f36d58ce175a926d0706171db21a036d Mon Sep 17 00:00:00 2001 From: Paulo Koch Date: Fri, 17 Mar 2017 10:06:42 +0000 Subject: [PATCH 29/96] Add my janky tester --- tester.pkoch-macos_sierra.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 tester.pkoch-macos_sierra.sh diff --git a/tester.pkoch-macos_sierra.sh b/tester.pkoch-macos_sierra.sh new file mode 100755 index 000000000..dbbaa2251 --- /dev/null +++ b/tester.pkoch-macos_sierra.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# I just wanted a place to dump the incantations I use for testing. +set -e + +brew install openssl libffi + +rm -rf scratch; mkdir scratch + +virtualenv scratch/venv -p /usr/local/bin/python2.7 +scratch/venv/bin/pip install -U pip setuptools + +CPPFLAGS=-I/usr/local/opt/openssl/include LDFLAGS=-L/usr/local/opt/openssl/lib scratch/venv/bin/pip install -e . + +scratch/venv/bin/certbot certonly -n --agree-tos --test-cert --email pkoch@lifeonmars.pt -a certbot-route53:auth -d pkoch.lifeonmars.pt --work-dir scratch --config-dir scratch --logs-dir scratch + +rm -rf scratch From ab0d5f830d83fbbae43c41039364b3fc5d06acc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paulo=20K=C3=B6ch?= Date: Wed, 5 Apr 2017 11:05:01 +0100 Subject: [PATCH 30/96] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 7dab0b7f6..cec9c295c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,5 @@ ## Route53 plugin for Let's Encrypt client - ### Before you start It's expected that the root hosted zone for the domain in question already @@ -25,7 +24,7 @@ for example). Make sure you have access to AWS's Route53 service, either through IAM roles or via `.aws/credentials`. Check out -(sample-aws-policy.json)[sample-aws-policy.json]. +[sample-aws-policy.json](sample-aws-policy.json) for the necessary permissions. To generate a certificate: ``` From a313eebc7fca0b1985b270e7ebf365034e2a2b0f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 5 Apr 2017 11:54:23 -0700 Subject: [PATCH 31/96] Conform to RFC 1945 --- certbot/client.py | 13 ++++++++----- certbot/main.py | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index ab13da1f0..db5590e00 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -53,12 +53,15 @@ def determine_user_agent(config): """ if config.user_agent is None: - auto = "no" if cli.cli_command == "certbot" else "yes" - ua = ("CertbotACMEClient/{0} ({1}) Authenticator/{2} Installer/{3} Verb/{4} " - "Renewing/{5} Auto/{6} Py/{7}") - ua = ua.format(certbot.__version__, util.get_os_info_ua(), + if cli.cli_command in ["certbot", "letsencrypt", "certbot-auto", "letsencrypt-auto"]: + auto = cli.cli_command + else: + auto = "other" + ua = ("CertbotACMEClient/{0} ({1}; {2}) Authenticator/{3} Installer/{4} " + "({5}; renewing:{6}) Py/{7}") + ua = ua.format(certbot.__version__, auto, util.get_os_info_ua(), config.authenticator, config.installer, config.verb, config.renewing, - auto, platform.python_version()) + platform.python_version()) else: ua = config.user_agent return ua diff --git a/certbot/main.py b/certbot/main.py index bc61ef568..6b6ff06b2 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -226,9 +226,9 @@ def _find_cert(config, domains, certname): """ action, lineage = _find_lineage_for_domains_and_certname(config, domains, certname) if action == "renew": - config.renewing = "Yes" + config.renewing = "yes" elif action == "newcert": - config.renewing = "No" + config.renewing = "no" if action == "reinstall": logger.info("Keeping the existing certificate") return (action != "reinstall"), lineage From 2f0ec5c38804619139ca69a7d87fc89d42e50b15 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 5 Apr 2017 12:40:26 -0700 Subject: [PATCH 32/96] Set renewing: correctly for the "renew" case. --- certbot/main.py | 1 + certbot/renewal.py | 1 + 2 files changed, 2 insertions(+) diff --git a/certbot/main.py b/certbot/main.py index ee178aa50..448ed21d5 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -696,6 +696,7 @@ def certonly(config, plugins): def renew(config, unused_plugins): """Renew previously-obtained certificates.""" try: + config.renewing = "yes" renewal.handle_renewal_request(config) finally: hooks.run_saved_post_hooks() diff --git a/certbot/renewal.py b/certbot/renewal.py index 6eb171763..7b651a92a 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -393,6 +393,7 @@ def handle_renewal_request(config): # elements from within the renewal configuration file). try: renewal_candidate = _reconstitute(lineage_config, renewal_file) + renewal_candidate.renewing = "yes" except Exception as e: # pylint: disable=broad-except logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) From 23a2ecb36e931fd62cfbde53c3e5d3f8cdb7d1ca Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 5 Apr 2017 15:28:14 -0700 Subject: [PATCH 33/96] Fixup --- certbot/renewal.py | 1 - 1 file changed, 1 deletion(-) diff --git a/certbot/renewal.py b/certbot/renewal.py index 7b651a92a..6eb171763 100644 --- a/certbot/renewal.py +++ b/certbot/renewal.py @@ -393,7 +393,6 @@ def handle_renewal_request(config): # elements from within the renewal configuration file). try: renewal_candidate = _reconstitute(lineage_config, renewal_file) - renewal_candidate.renewing = "yes" except Exception as e: # pylint: disable=broad-except logger.warning("Renewal configuration file %s produced an " "unexpected error: %s. Skipping.", renewal_file, e) From c35ca9775b8058b4b6acad11ffc1ca060fa07b03 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 6 Apr 2017 14:41:40 -0700 Subject: [PATCH 34/96] tweak comment --- certbot/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/cli.py b/certbot/cli.py index 0efec4352..14e3adf6d 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -532,7 +532,7 @@ class HelpfulArgumentParser(object): parsed_args = self.parser.parse_args(self.args) parsed_args.func = self.VERBS[self.verb] parsed_args.verb = self.verb - parsed_args.renewing = "unknown" # an important config property we don't know until later + parsed_args.renewing = "unknown" # used for the User Agent; not known until later if self.detect_defaults: return parsed_args From cb66ba95e1502741e8f97e21f53265093244c9b5 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 13 Apr 2017 20:00:31 -0700 Subject: [PATCH 35/96] Revert "Also report if we're renewing" This reverts commit 6fa521bc5f82f19f187c870cf633b1a43a6a6001. --- certbot/client.py | 5 ++--- certbot/main.py | 14 ++++---------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index f42911921..1c0b71e20 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -59,9 +59,9 @@ def determine_user_agent(config): else: auto = "other" ua = ("CertbotACMEClient/{0} ({1}; {2}) Authenticator/{3} Installer/{4} " - "({5}; renewing:{6}) Py/{7}") + "({5}) Py/{7}") ua = ua.format(certbot.__version__, auto, util.get_os_info_ua(), - config.authenticator, config.installer, config.verb, config.renewing, + config.authenticator, config.installer, config.verb, platform.python_version()) else: ua = config.user_agent @@ -76,7 +76,6 @@ def sample_user_agent(): self.installer = "YYY" self.user_agent = None self.verb = "SUBCOMMAND" - self.renewing = "YESNOMAYBE" return determine_user_agent(DummyConfig()) diff --git a/certbot/main.py b/certbot/main.py index 448ed21d5..e9bf3b9ed 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -215,10 +215,6 @@ def _find_cert(config, domains, certname): RenewableCert instance or None. """ action, lineage = _find_lineage_for_domains_and_certname(config, domains, certname) - if action == "renew": - config.renewing = "yes" - elif action == "newcert": - config.renewing = "no" if action == "reinstall": logger.info("Keeping the existing certificate") return (action != "reinstall"), lineage @@ -590,12 +586,12 @@ def run(config, plugins): # pylint: disable=too-many-branches,too-many-locals except errors.PluginSelectionError as e: return e.message - domains, certname = _find_domains_or_certname(config, installer) - should_get_cert, lineage = _find_cert(config, domains, certname) - # TODO: Handle errors from _init_le_client? le_client = _init_le_client(config, authenticator, installer) + domains, certname = _find_domains_or_certname(config, installer) + should_get_cert, lineage = _find_cert(config, domains, certname) + new_lineage = lineage if should_get_cert: new_lineage = _get_and_save_cert(le_client, config, domains, @@ -668,9 +664,9 @@ def certonly(config, plugins): except errors.PluginSelectionError as e: logger.info("Could not choose appropriate plugin: %s", e) raise + le_client = _init_le_client(config, auth, installer) if config.csr: - le_client = _init_le_client(config, auth, installer) cert_path, fullchain_path = _csr_get_and_save_cert(config, le_client) _report_new_cert(config, cert_path, fullchain_path) _suggest_donation_if_appropriate(config) @@ -678,8 +674,6 @@ def certonly(config, plugins): domains, certname = _find_domains_or_certname(config, installer) should_get_cert, lineage = _find_cert(config, domains, certname) - # _init_le_client needs to run after _find_cert for config.renewing if possible - le_client = _init_le_client(config, auth, installer) if not should_get_cert: notify = zope.component.getUtility(interfaces.IDisplay).notification From 5be0811bdcf707624ab020036ef8834952f7ca20 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Thu, 13 Apr 2017 20:07:03 -0700 Subject: [PATCH 36/96] Replace removed "renewing" with potential future substitutes :/ --- certbot/client.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/certbot/client.py b/certbot/client.py index 1c0b71e20..c450d19e7 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -58,10 +58,13 @@ def determine_user_agent(config): auto = cli.cli_command else: auto = "other" + flags = "dup" if config.duplicate else "" + flags += "force" if config.renew_by_default else "" + ua = ("CertbotACMEClient/{0} ({1}; {2}) Authenticator/{3} Installer/{4} " - "({5}) Py/{7}") + "({5}; flags: {6}) Py/{7}") ua = ua.format(certbot.__version__, auto, util.get_os_info_ua(), - config.authenticator, config.installer, config.verb, + config.authenticator, config.installer, config.verb, flags, platform.python_version()) else: ua = config.user_agent From 89af460792fcdfb23c7dc4f9fcdec1bfa07a2656 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 28 Apr 2017 15:03:50 -0700 Subject: [PATCH 37/96] Reuse dynamic install_requires. (#4554) * Revert "Make argparse dependency unconditional. (#2249)" This reverts commit 8f101034960ffc1e47879314585898efda234e60. * Update comment about environment markers --- acme/setup.py | 5 ++++- setup.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index a640ae6bb..1c887e5d3 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -8,7 +8,6 @@ version = '0.14.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ - 'argparse', # load_pem_private/public_key (>=0.6) # rsa_recover_prime_factors (>=0.8) 'cryptography>=0.8', @@ -28,6 +27,10 @@ install_requires = [ 'six', ] +# env markers cause problems with older pip and setuptools +if sys.version_info < (2, 7): + install_requires.append('argparse') + dev_extras = [ 'nose', 'tox', diff --git a/setup.py b/setup.py index 0e8d19a22..d2c092a05 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,6 @@ version = meta['version'] # https://github.com/pypa/pip/issues/988 for more info. install_requires = [ 'acme=={0}'.format(version), - 'argparse', # We technically need ConfigArgParse 0.10.0 for Python 2.6 support, but # saying so here causes a runtime error against our temporary fork of 0.9.3 # in which we added 2.6 support (see #2243), so we relax the requirement. @@ -56,6 +55,10 @@ install_requires = [ 'zope.interface', ] +# env markers cause problems with older pip and setuptools +if sys.version_info < (2, 7): + install_requires.append('argparse') + dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 'astroid==1.3.5', From 8fa12bef8e7c687400367cf45a022b43e8c92754 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 28 Apr 2017 16:06:45 -0700 Subject: [PATCH 38/96] Tell the world we're Python 3 compatible (#4568) * Mention python 3 support in setup.py * Build universal (py2 and py3 compatible) wheels * Mention Python 3.3+ support in docs * we work on python 3.6 too --- acme/setup.py | 1 + certbot-apache/setup.cfg | 2 ++ certbot-apache/setup.py | 5 +++++ certbot-compatibility-test/setup.cfg | 2 ++ certbot-compatibility-test/setup.py | 5 +++++ certbot-nginx/setup.cfg | 2 ++ certbot-nginx/setup.py | 5 +++++ docs/install.rst | 4 ++-- letshelp-certbot/setup.cfg | 2 ++ letshelp-certbot/setup.py | 5 +++++ setup.cfg | 3 +++ setup.py | 5 +++++ 12 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 certbot-apache/setup.cfg create mode 100644 certbot-compatibility-test/setup.cfg create mode 100644 certbot-nginx/setup.cfg create mode 100644 letshelp-certbot/setup.cfg diff --git a/acme/setup.py b/acme/setup.py index 1c887e5d3..cb12df51f 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -62,6 +62,7 @@ setup( 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-apache/setup.cfg b/certbot-apache/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-apache/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 9a473c584..19f0c74f8 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -42,6 +42,11 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/certbot-compatibility-test/setup.cfg b/certbot-compatibility-test/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-compatibility-test/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index aecae329f..dfd05bfd9 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -42,6 +42,11 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', ], diff --git a/certbot-nginx/setup.cfg b/certbot-nginx/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-nginx/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 786b5a1a1..63b6f16af 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -42,6 +42,11 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/docs/install.rst b/docs/install.rst index 6c56584be..a1e91c010 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -22,8 +22,8 @@ your system. System Requirements =================== -Certbot currently requires Python 2.6 or 2.7. By default, it requires root -access in order to write to ``/etc/letsencrypt``, +Certbot currently requires Python 2.6, 2.7, or 3.3+. By default, it requires +root access in order to write to ``/etc/letsencrypt``, ``/var/log/letsencrypt``, ``/var/lib/letsencrypt``; to bind to ports 80 and 443 (if you use the ``standalone`` plugin) and to read and modify webserver configurations (if you use the ``apache`` or ``nginx`` plugins). If none of diff --git a/letshelp-certbot/setup.cfg b/letshelp-certbot/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/letshelp-certbot/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/letshelp-certbot/setup.py b/letshelp-certbot/setup.py index b26ab41fe..3ce442b3e 100644 --- a/letshelp-certbot/setup.py +++ b/letshelp-certbot/setup.py @@ -33,6 +33,11 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', diff --git a/setup.cfg b/setup.cfg index 8d68bac30..3b4dbaf87 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,6 @@ +[bdist_wheel] +universal = 1 + [easy_install] zip_ok = false diff --git a/setup.py b/setup.py index d2c092a05..b0cf57810 100644 --- a/setup.py +++ b/setup.py @@ -97,6 +97,11 @@ setup( 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Security', 'Topic :: System :: Installation/Setup', From 0a4ee306a9de6a74d49964921204699e51809d7f Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 28 Apr 2017 17:56:10 -0700 Subject: [PATCH 39/96] Fix UA flag setting (and set more of them) --- certbot/cli.py | 4 +++- certbot/client.py | 45 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 14e3adf6d..82c9622f0 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1108,7 +1108,9 @@ def _create_subparsers(helpful): "plugin and use case, and to know when to deprecate support for past Python " "versions. If you wish to hide this information from the Let's " 'Encrypt server, set this to "". ' - '(default: {0})'.format(sample_user_agent())) + '(default: {0}). The flags encoded in the user agent are: ' + '--duplicate, --force-renew, --allow-subset-of-names, -n, and ' + 'whether any hooks are set.'.format(sample_user_agent())) helpful.add("certonly", "--csr", type=read_file, help="Path to a Certificate Signing Request (CSR) in DER or PEM format." diff --git a/certbot/client.py b/certbot/client.py index c450d19e7..ad7323e62 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -58,27 +58,50 @@ def determine_user_agent(config): auto = cli.cli_command else: auto = "other" - flags = "dup" if config.duplicate else "" - flags += "force" if config.renew_by_default else "" ua = ("CertbotACMEClient/{0} ({1}; {2}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") ua = ua.format(certbot.__version__, auto, util.get_os_info_ua(), - config.authenticator, config.installer, config.verb, flags, - platform.python_version()) + config.authenticator, config.installer, config.verb, + ua_flags(config), platform.python_version()) else: ua = config.user_agent return ua +def ua_flags(config): + "Turn some very important CLI flags into clues in the user agent." + if isinstance(config, DummyConfig): + return "FLAGS" + flags = [] + if config.duplicate: + flags.append("dup") + if config.renew_by_default: + flags.append("frn") + if config.allow_subset_of_names: + flags.append("asn") + if config.noninteractive_mode: + flags.append("n") + hook_names = ("pre", "post", "renew", "manual_auth", "manual_cleanup") + hooks = [getattr(config, h + "_hook") for h in hook_names] + if any(hooks): + flags.append("hook") + return " ".join(flags) + +class DummyConfig(object): + "Shim for computing a sample user agent." + def __init__(self): + self.authenticator = "XXX" + self.installer = "YYY" + self.user_agent = None + self.verb = "SUBCOMMAND" + + def __getattr__(self, name): + "Any config properties we might have are None." + return None + def sample_user_agent(): "Document what this Certbot's user agent string will be like." - class DummyConfig(object): - "Shim for computing a sample user agent." - def __init__(self): - self.authenticator = "XXX" - self.installer = "YYY" - self.user_agent = None - self.verb = "SUBCOMMAND" + return determine_user_agent(DummyConfig()) From f6c02728e4348a88fbca2acbacb2a8b55315b772 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 28 Apr 2017 18:37:58 -0700 Subject: [PATCH 40/96] Address review comments --- certbot/cli.py | 3 +-- certbot/client.py | 7 +------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index 82c9622f0..9e8778ceb 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -532,7 +532,6 @@ class HelpfulArgumentParser(object): parsed_args = self.parser.parse_args(self.args) parsed_args.func = self.VERBS[self.verb] parsed_args.verb = self.verb - parsed_args.renewing = "unknown" # used for the User Agent; not known until later if self.detect_defaults: return parsed_args @@ -1106,7 +1105,7 @@ def _create_subparsers(helpful): help="Set a custom user agent string for the client. User agent strings allow " "the CA to collect high level statistics about success rates by OS, " "plugin and use case, and to know when to deprecate support for past Python " - "versions. If you wish to hide this information from the Let's " + "versions and flags. If you wish to hide this information from the Let's " 'Encrypt server, set this to "". ' '(default: {0}). The flags encoded in the user agent are: ' '--duplicate, --force-renew, --allow-subset-of-names, -n, and ' diff --git a/certbot/client.py b/certbot/client.py index ad7323e62..dca6d8618 100644 --- a/certbot/client.py +++ b/certbot/client.py @@ -54,14 +54,9 @@ def determine_user_agent(config): """ if config.user_agent is None: - if cli.cli_command in ["certbot", "letsencrypt", "certbot-auto", "letsencrypt-auto"]: - auto = cli.cli_command - else: - auto = "other" - ua = ("CertbotACMEClient/{0} ({1}; {2}) Authenticator/{3} Installer/{4} " "({5}; flags: {6}) Py/{7}") - ua = ua.format(certbot.__version__, auto, util.get_os_info_ua(), + ua = ua.format(certbot.__version__, cli.cli_command, util.get_os_info_ua(), config.authenticator, config.installer, config.verb, ua_flags(config), platform.python_version()) else: From 72b6179e0eb2519b04d0c58dea1a1005d8ac5889 Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Fri, 28 Apr 2017 18:45:42 -0700 Subject: [PATCH 41/96] Remove vestigial thingy --- certbot/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/certbot/main.py b/certbot/main.py index e9bf3b9ed..256503d4e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -690,7 +690,6 @@ def certonly(config, plugins): def renew(config, unused_plugins): """Renew previously-obtained certificates.""" try: - config.renewing = "yes" renewal.handle_renewal_request(config) finally: hooks.run_saved_post_hooks() From 4ca702f6fbd5936f1b1e4ee794ebca09d5acd6fb Mon Sep 17 00:00:00 2001 From: Benjamin Qin Date: Mon, 1 May 2017 11:43:07 -0500 Subject: [PATCH 42/96] Update doc using.rst to correct a sample script (#4582) 'More information about renewal....' should not be part of the code block. --- docs/using.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/using.rst b/docs/using.rst index 1bcf1483b..dd0061f05 100644 --- a/docs/using.rst +++ b/docs/using.rst @@ -430,7 +430,7 @@ apply appropriate file permissions. esac done - More information about renewal hooks can be found by running +More information about renewal hooks can be found by running ``certbot --help renew``. If you're sure that this command executes successfully without human From 5ca8f7c5b943132b73b0becfeacb660361000172 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 1 May 2017 14:49:12 -0700 Subject: [PATCH 43/96] Add lockfile (#4449) * add lock_file * cleanup lock file * Add LockFile tests * add lock_dir * add lock_dir_until_exit * add set_up_core_dir and move lock_dir_until_exit * Move lock_and_call to certbot.test.util * Add lock to Apache * Add lock to the Nginx plugin * Improve permissions error message * sort plugins * add test_prepare_order * provide more actionable permissions error * Document and catch use of OSError * don't lock a directory twice * add conditional dependency on ordereddict * Add lock_test * expand sorted plugins comment * Add lock_test to lint * make make_lineage more conventional and flexible * enhance lock_test.py * add lock_test to tox * Readd success message * make py26 happy * add test_acquire_without_deletion --- acme/setup.py | 5 +- certbot-apache/certbot_apache/configurator.py | 8 + .../certbot_apache/tests/configurator_test.py | 17 ++ certbot-nginx/certbot_nginx/configurator.py | 8 + .../certbot_nginx/tests/configurator_test.py | 18 ++ certbot/errors.py | 4 + certbot/lock.py | 139 ++++++++++ certbot/log.py | 2 +- certbot/main.py | 8 +- certbot/plugins/disco.py | 12 +- certbot/plugins/disco_test.py | 49 ++-- certbot/tests/lock_test.py | 116 +++++++++ certbot/tests/main_test.py | 13 +- certbot/tests/renewal_test.py | 3 +- certbot/tests/util.py | 60 ++++- certbot/tests/util_test.py | 45 +++- certbot/util.py | 43 +++- setup.py | 5 +- tests/lock_test.py | 238 ++++++++++++++++++ tox.ini | 3 +- 20 files changed, 750 insertions(+), 46 deletions(-) create mode 100644 certbot/lock.py create mode 100644 certbot/tests/lock_test.py create mode 100644 tests/lock_test.py diff --git a/acme/setup.py b/acme/setup.py index cb12df51f..8d8d1a049 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -29,7 +29,10 @@ install_requires = [ # env markers cause problems with older pip and setuptools if sys.version_info < (2, 7): - install_requires.append('argparse') + install_requires.extend([ + 'argparse', + 'ordereddict', + ]) dev_extras = [ 'nose', diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 39d25619d..b82506dd2 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -197,6 +197,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): install_ssl_options_conf(self.mod_ssl_conf) + # Prevent two Apache plugins from modifying a config at once + try: + util.lock_dir_until_exit(self.conf("server-root")) + except (OSError, errors.LockError): + logger.debug("Encountered error:", exc_info=True) + raise errors.PluginError( + "Unable to lock %s", self.conf("server-root")) + def _check_aug_version(self): """ Checks that we have recent enough version of libaugeas. If augeas version is recent enough, it will support case insensitive diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 45e701bd5..3e12bf60a 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -95,6 +95,23 @@ class MultipleVhostsTest(util.ApacheTest): self.assertRaises( errors.NotSupportedError, self.config.prepare) + def test_prepare_locked(self): + server_root = self.config.conf("server-root") + self.config.config_test = mock.Mock() + os.remove(os.path.join(server_root, ".certbot.lock")) + certbot_util.lock_and_call(self._test_prepare_locked, server_root) + + @mock.patch("certbot_apache.parser.ApacheParser") + @mock.patch("certbot_apache.configurator.util.exe_exists") + def _test_prepare_locked(self, unused_parser, unused_exe_exists): + try: + self.config.prepare() + except errors.PluginError as err: + err_msg = str(err) + self.assertTrue("lock" in err_msg) + self.assertTrue(self.config.conf("server-root") in err_msg) + else: # pragma: no cover + self.fail("Exception wasn't raised!") def test_add_parser_arguments(self): # pylint: disable=no-self-use from certbot_apache.configurator import ApacheConfigurator diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index afa701a75..8608fb66a 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -162,6 +162,14 @@ class NginxConfigurator(common.Plugin): if self.version is None: self.version = self.get_version() + # Prevent two Nginx plugins from modifying a config at once + try: + util.lock_dir_until_exit(self.conf('server-root')) + except (OSError, errors.LockError): + logger.debug('Encountered error:', exc_info=True) + raise errors.PluginError( + 'Unable to lock %s', self.conf('server-root')) + # Entry point in main.py for installing cert def deploy_cert(self, domain, cert_path, key_path, chain_path=None, fullchain_path=None): diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index b9e70cd59..69f728b53 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -12,6 +12,7 @@ from acme import messages from certbot import achallenges from certbot import errors +from certbot.tests import util as certbot_test_util from certbot_nginx import obj from certbot_nginx import parser @@ -65,6 +66,23 @@ class NginxConfiguratorTest(util.NginxTest): self.config.prepare() self.assertEqual((1, 6, 2), self.config.version) + def test_prepare_locked(self): + server_root = self.config.conf("server-root") + self.config.config_test = mock.Mock() + os.remove(os.path.join(server_root, ".certbot.lock")) + certbot_test_util.lock_and_call(self._test_prepare_locked, server_root) + + @mock.patch("certbot_nginx.configurator.util.exe_exists") + def _test_prepare_locked(self, unused_exe_exists): + try: + self.config.prepare() + except errors.PluginError as err: + err_msg = str(err) + self.assertTrue("lock" in err_msg) + self.assertTrue(self.config.conf("server-root") in err_msg) + else: # pragma: no cover + self.fail("Exception wasn't raised!") + @mock.patch("certbot_nginx.configurator.socket.gethostbyaddr") def test_get_all_names(self, mock_gethostbyaddr): mock_gethostbyaddr.return_value = ('155.225.50.69.nephoscale.net', [], []) diff --git a/certbot/errors.py b/certbot/errors.py index 6d191404c..551add61c 100644 --- a/certbot/errors.py +++ b/certbot/errors.py @@ -33,6 +33,10 @@ class SignalExit(Error): """A Unix signal was received while in the ErrorHandler context manager.""" +class LockError(Error): + """File locking error.""" + + # Auth Handler Errors class AuthorizationError(Error): """Authorization error.""" diff --git a/certbot/lock.py b/certbot/lock.py new file mode 100644 index 000000000..5f59cc090 --- /dev/null +++ b/certbot/lock.py @@ -0,0 +1,139 @@ +"""Implements file locks for locking files and directories in UNIX.""" +import errno +import fcntl +import logging +import os + +from certbot import errors + +logger = logging.getLogger(__name__) + + +def lock_dir(dir_path): + """Place a lock file on the directory at dir_path. + + The lock file is placed in the root of dir_path with the name + .certbot.lock. + + :param str dir_path: path to directory + + :returns: the locked LockFile object + :rtype: LockFile + + :raises errors.LockError: if unable to acquire the lock + + """ + return LockFile(os.path.join(dir_path, '.certbot.lock')) + + +class LockFile(object): + """A UNIX lock file. + + This lock file is released when the locked file is closed or the + process exits. It cannot be used to provide synchronization between + threads. It is based on the lock_file package by Martin Horcicka. + + """ + def __init__(self, path): + """Initialize and acquire the lock file. + + :param str path: path to the file to lock + + :raises errors.LockError: if unable to acquire the lock + + """ + super(LockFile, self).__init__() + self._path = path + self._fd = None + + self.acquire() + + def acquire(self): + """Acquire the lock file. + + :raises errors.LockError: if lock is already held + :raises OSError: if unable to open or stat the lock file + + """ + while self._fd is None: + # Open the file + fd = os.open(self._path, os.O_CREAT | os.O_WRONLY, 0o600) + try: + self._try_lock(fd) + if self._lock_success(fd): + self._fd = fd + finally: + # Close the file if it is not the required one + if self._fd is None: + os.close(fd) + + def _try_lock(self, fd): + """Try to acquire the lock file without blocking. + + :param int fd: file descriptor of the opened file to lock + + """ + try: + fcntl.lockf(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError as err: + if err.errno in (errno.EACCES, errno.EAGAIN): + logger.debug( + "A lock on %s is held by another process.", self._path) + raise errors.LockError( + "Another instance of Certbot is already running.") + raise + + def _lock_success(self, fd): + """Did we successfully grab the lock? + + Because this class deletes the locked file when the lock is + released, it is possible another process removed and recreated + the file between us opening the file and acquiring the lock. + + :param int fd: file descriptor of the opened file to lock + + :returns: True if the lock was successfully acquired + :rtype: bool + + """ + try: + stat1 = os.stat(self._path) + except OSError as err: + if err.errno == errno.ENOENT: + return False + raise + + stat2 = os.fstat(fd) + # If our locked file descriptor and the file on disk refer to + # the same device and inode, they're the same file. + return stat1.st_dev == stat2.st_dev and stat1.st_ino == stat2.st_ino + + def __repr__(self): + repr_str = '{0}({1}) <'.format(self.__class__.__name__, self._path) + if self._fd is None: + repr_str += 'released>' + else: + repr_str += 'acquired>' + return repr_str + + def release(self): + """Remove, close, and release the lock file.""" + # It is important the lock file is removed before it's released, + # otherwise: + # + # process A: open lock file + # process B: release lock file + # process A: lock file + # process A: check device and inode + # process B: delete file + # process C: open and lock a different file at the same path + # + # Calling os.remove on a file that's in use doesn't work on + # Windows, but neither does locking with fcntl. + try: + os.remove(self._path) + finally: + try: + os.close(self._fd) + finally: + self._fd = None diff --git a/certbot/log.py b/certbot/log.py index 7660846a6..c7bc867f1 100644 --- a/certbot/log.py +++ b/certbot/log.py @@ -131,7 +131,7 @@ def setup_log_file_handler(config, logfile, fmt): """ # TODO: logs might contain sensitive data such as contents of the # private key! #525 - util.make_or_verify_core_dir( + util.set_up_core_dir( config.logs_dir, 0o700, os.geteuid(), config.strict_permissions) log_file_path = os.path.join(config.logs_dir, logfile) try: diff --git a/certbot/main.py b/certbot/main.py index 023c09aee..eafcf49dd 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -696,10 +696,10 @@ def renew(config, unused_plugins): def make_or_verify_needed_dirs(config): """Create or verify existence of config and work directories""" - util.make_or_verify_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) - util.make_or_verify_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, - os.geteuid(), config.strict_permissions) + util.set_up_core_dir(config.config_dir, constants.CONFIG_DIRS_MODE, + os.geteuid(), config.strict_permissions) + util.set_up_core_dir(config.work_dir, constants.CONFIG_DIRS_MODE, + os.geteuid(), config.strict_permissions) def set_displayer(config): diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index a17f8d7b3..6bf4bd369 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -12,6 +12,12 @@ from certbot import constants from certbot import errors from certbot import interfaces +try: + from collections import OrderedDict +except ImportError: # pragma: no cover + # OrderedDict was added in Python 2.7 + from ordereddict import OrderedDict # pylint: disable=import-error + logger = logging.getLogger(__name__) @@ -168,7 +174,11 @@ class PluginsRegistry(collections.Mapping): """Plugins registry.""" def __init__(self, plugins): - self._plugins = plugins + # plugins are sorted so the same order is used between runs. + # This prevents deadlock caused by plugins acquiring a lock + # and ensures at least one concurrent Certbot instance will run + # successfully. + self._plugins = OrderedDict(sorted(six.iteritems(plugins))) @classmethod def find_all(cls): diff --git a/certbot/plugins/disco_test.py b/certbot/plugins/disco_test.py index 6c3c39dca..220b902b3 100644 --- a/certbot/plugins/disco_test.py +++ b/certbot/plugins/disco_test.py @@ -1,4 +1,6 @@ """Tests for certbot.plugins.disco.""" +import functools +import string import unittest import mock @@ -182,12 +184,17 @@ class PluginEntryPointTest(unittest.TestCase): class PluginsRegistryTest(unittest.TestCase): """Tests for certbot.plugins.disco.PluginsRegistry.""" - def setUp(self): + @classmethod + def _create_new_registry(cls, plugins): from certbot.plugins.disco import PluginsRegistry - self.plugin_ep = mock.MagicMock(name="mock") + return PluginsRegistry(plugins) + + def setUp(self): + self.plugin_ep = mock.MagicMock() + self.plugin_ep.name = "mock" self.plugin_ep.__hash__.side_effect = TypeError - self.plugins = {"mock": self.plugin_ep} - self.reg = PluginsRegistry(self.plugins) + self.plugins = {self.plugin_ep.name: self.plugin_ep} + self.reg = self._create_new_registry(self.plugins) def test_find_all(self): from certbot.plugins.disco import PluginsRegistry @@ -207,9 +214,8 @@ class PluginsRegistryTest(unittest.TestCase): self.assertEqual(["mock"], list(self.reg)) def test_len(self): + self.assertEqual(0, len(self._create_new_registry({}))) self.assertEqual(1, len(self.reg)) - self.plugins.clear() - self.assertEqual(0, len(self.reg)) def test_init(self): self.plugin_ep.init.return_value = "baz" @@ -217,14 +223,11 @@ class PluginsRegistryTest(unittest.TestCase): self.plugin_ep.init.assert_called_once_with("bar") def test_filter(self): - self.plugins.update({ - "foo": "bar", - "bar": "foo", - "baz": "boo", - }) self.assertEqual( - {"foo": "bar", "baz": "boo"}, - self.reg.filter(lambda p_ep: str(p_ep).startswith("b"))) + self.plugins, + self.reg.filter(lambda p_ep: p_ep.name.startswith("m"))) + self.assertEqual( + {}, self.reg.filter(lambda p_ep: p_ep.name.startswith("b"))) def test_ifaces(self): self.plugin_ep.ifaces.return_value = True @@ -246,6 +249,17 @@ class PluginsRegistryTest(unittest.TestCase): self.assertEqual(["baz"], self.reg.prepare()) self.plugin_ep.prepare.assert_called_once_with() + def test_prepare_order(self): + order = [] + plugins = dict( + (c, mock.MagicMock(prepare=functools.partial(order.append, c))) + for c in string.ascii_letters) + reg = self._create_new_registry(plugins) + reg.prepare() + # order of prepare calls must be sorted to prevent deadlock + # caused by plugins acquiring locks during prepare + self.assertEqual(order, sorted(string.ascii_letters)) + def test_available(self): self.plugin_ep.available = True # pylint: disable=protected-access @@ -265,11 +279,12 @@ class PluginsRegistryTest(unittest.TestCase): repr(self.reg)) def test_str(self): + self.assertEqual("No plugins", str(self._create_new_registry({}))) self.plugin_ep.__str__ = lambda _: "Mock" - self.plugins["foo"] = "Mock" - self.assertEqual("Mock\n\nMock", str(self.reg)) - self.plugins.clear() - self.assertEqual("No plugins", str(self.reg)) + self.assertEqual("Mock", str(self.reg)) + plugins = {self.plugin_ep.name: self.plugin_ep, "foo": "Bar"} + reg = self._create_new_registry(plugins) + self.assertEqual("Bar\n\nMock", str(reg)) if __name__ == "__main__": diff --git a/certbot/tests/lock_test.py b/certbot/tests/lock_test.py new file mode 100644 index 000000000..e1a4f8c8a --- /dev/null +++ b/certbot/tests/lock_test.py @@ -0,0 +1,116 @@ +"""Tests for certbot.lock.""" +import functools +import multiprocessing +import os +import unittest + +import mock + +from certbot import errors +from certbot.tests import util as test_util + + +class LockDirTest(test_util.TempDirTestCase): + """Tests for certbot.lock.lock_dir.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.lock import lock_dir + return lock_dir(*args, **kwargs) + + def test_it(self): + assert_raises = functools.partial( + self.assertRaises, errors.LockError, self._call, self.tempdir) + lock_path = os.path.join(self.tempdir, '.certbot.lock') + test_util.lock_and_call(assert_raises, lock_path) + + +class LockFileTest(test_util.TempDirTestCase): + """Tests for certbot.lock.LockFile.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.lock import LockFile + return LockFile(*args, **kwargs) + + def setUp(self): + super(LockFileTest, self).setUp() + self.lock_path = os.path.join(self.tempdir, 'test.lock') + + def test_acquire_without_deletion(self): + # acquire the lock in another process but don't delete the file + child = multiprocessing.Process(target=self._call, + args=(self.lock_path,)) + child.start() + child.join() + self.assertEqual(child.exitcode, 0) + self.assertTrue(os.path.exists(self.lock_path)) + + # Test we're still able to properly acquire and release the lock + self.test_removed() + + def test_contention(self): + assert_raises = functools.partial( + self.assertRaises, errors.LockError, self._call, self.lock_path) + test_util.lock_and_call(assert_raises, self.lock_path) + + def test_locked_repr(self): + lock_file = self._call(self.lock_path) + locked_repr = repr(lock_file) + self._test_repr_common(lock_file, locked_repr) + self.assertTrue('acquired' in locked_repr) + + def test_released_repr(self): + lock_file = self._call(self.lock_path) + lock_file.release() + released_repr = repr(lock_file) + self._test_repr_common(lock_file, released_repr) + self.assertTrue('released' in released_repr) + + def _test_repr_common(self, lock_file, lock_repr): + self.assertTrue(lock_file.__class__.__name__ in lock_repr) + self.assertTrue(self.lock_path in lock_repr) + + def test_race(self): + should_delete = [True, False] + stat = os.stat + + def delete_and_stat(path): + """Wrap os.stat and maybe delete the file first.""" + if path == self.lock_path and should_delete.pop(0): + os.remove(path) + return stat(path) + + with mock.patch('certbot.lock.os.stat') as mock_stat: + mock_stat.side_effect = delete_and_stat + self._call(self.lock_path) + self.assertFalse(should_delete) + + def test_removed(self): + lock_file = self._call(self.lock_path) + lock_file.release() + self.assertFalse(os.path.exists(self.lock_path)) + + @mock.patch('certbot.lock.fcntl.lockf') + def test_unexpected_lockf_err(self, mock_lockf): + msg = 'hi there' + mock_lockf.side_effect = IOError(msg) + try: + self._call(self.lock_path) + except IOError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('IOError not raised') + + @mock.patch('certbot.lock.os.stat') + def test_unexpected_stat_err(self, mock_stat): + msg = 'hi there' + mock_stat.side_effect = OSError(msg) + try: + self._call(self.lock_path) + except OSError as err: + self.assertTrue(msg in str(err)) + else: # pragma: no cover + self.fail('OSError not raised') + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 562c4bb9d..23cff7edd 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -792,12 +792,12 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me print(lf.read()) def test_renew_verb(self): - test_util.make_lineage(self, 'sample-renewal.conf') + test_util.make_lineage(self.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run", "-tvv"] self._test_renewal_common(True, [], args=args, should_renew=True) def test_quiet_renew(self): - test_util.make_lineage(self, 'sample-renewal.conf') + test_util.make_lineage(self.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run"] _, _, stdout = self._test_renewal_common(True, [], args=args, should_renew=True) out = stdout.getvalue() @@ -809,13 +809,13 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me self.assertEqual("", out) def test_renew_hook_validation(self): - test_util.make_lineage(self, 'sample-renewal.conf') + test_util.make_lineage(self.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run", "--post-hook=no-such-command"] self._test_renewal_common(True, [], args=args, should_renew=False, error_expected=True) def test_renew_no_hook_validation(self): - test_util.make_lineage(self, 'sample-renewal.conf') + test_util.make_lineage(self.config_dir, 'sample-renewal.conf') args = ["renew", "--dry-run", "--post-hook=no-such-command", "--disable-hook-validation"] with mock.patch("certbot.hooks.post_hook"): @@ -825,7 +825,8 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me @mock.patch("certbot.cli.set_by_cli") def test_ancient_webroot_renewal_conf(self, mock_set_by_cli): mock_set_by_cli.return_value = False - rc_path = test_util.make_lineage(self, 'sample-renewal-ancient.conf') + rc_path = test_util.make_lineage( + self.config_dir, 'sample-renewal-ancient.conf') args = mock.MagicMock(account=None, config_dir=self.config_dir, logs_dir=self.logs_dir, work_dir=self.work_dir, email=None, webroot_path=None) @@ -846,7 +847,7 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me self._test_renewal_common(False, [], args=args, should_renew=False, error_expected=True) def test_renew_with_certname(self): - test_util.make_lineage(self, 'sample-renewal.conf') + test_util.make_lineage(self.config_dir, 'sample-renewal.conf') self._test_renewal_common(True, [], should_renew=True, args=['renew', '--dry-run', '--cert-name', 'sample-renewal']) diff --git a/certbot/tests/renewal_test.py b/certbot/tests/renewal_test.py index de3efe39c..869e6b104 100644 --- a/certbot/tests/renewal_test.py +++ b/certbot/tests/renewal_test.py @@ -21,7 +21,8 @@ class RenewalTest(util.TempDirTestCase): @mock.patch('certbot.cli.set_by_cli') def test_ancient_webroot_renewal_conf(self, mock_set_by_cli): mock_set_by_cli.return_value = False - rc_path = util.make_lineage(self, 'sample-renewal-ancient.conf') + rc_path = util.make_lineage( + self.config_dir, 'sample-renewal-ancient.conf') args = mock.MagicMock(account=None, config_dir=self.config_dir, logs_dir="logs", work_dir="work", email=None, webroot_path=None) diff --git a/certbot/tests/util.py b/certbot/tests/util.py index d58834335..76e3d5846 100644 --- a/certbot/tests/util.py +++ b/certbot/tests/util.py @@ -3,6 +3,7 @@ .. warning:: This module is not part of the public API. """ +import multiprocessing import os import pkg_resources import shutil @@ -13,12 +14,14 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization import mock import OpenSSL +from six.moves import reload_module # pylint: disable=import-error from acme import jose from certbot import constants from certbot import interfaces from certbot import storage +from certbot import util from certbot.display import util as display_util @@ -106,12 +109,13 @@ def skip_unless(condition, reason): # pragma: no cover return lambda cls: None -def make_lineage(self, testfile): +def make_lineage(config_dir, testfile): """Creates a lineage defined by testfile. This creates the archive, live, and renewal directories if necessary and creates a simple lineage. + :param str config_dir: path to the configuration directory :param str testfile: configuration file to base the lineage on :returns: path to the renewal conf file for the created lineage @@ -121,11 +125,11 @@ def make_lineage(self, testfile): lineage_name = testfile[:-len('.conf')] conf_dir = os.path.join( - self.config_dir, constants.RENEWAL_CONFIGS_DIR) + config_dir, constants.RENEWAL_CONFIGS_DIR) archive_dir = os.path.join( - self.config_dir, constants.ARCHIVE_DIR, lineage_name) + config_dir, constants.ARCHIVE_DIR, lineage_name) live_dir = os.path.join( - self.config_dir, constants.LIVE_DIR, lineage_name) + config_dir, constants.LIVE_DIR, lineage_name) for directory in (archive_dir, conf_dir, live_dir,): if not os.path.exists(directory): @@ -140,11 +144,11 @@ def make_lineage(self, testfile): os.symlink(os.path.join(archive_dir, '{0}1.pem'.format(kind)), os.path.join(live_dir, '{0}.pem'.format(kind))) - conf_path = os.path.join(self.config_dir, conf_dir, testfile) + conf_path = os.path.join(config_dir, conf_dir, testfile) with open(vector_path(testfile)) as src: with open(conf_path, 'w') as dst: dst.writelines( - line.replace('MAGICDIR', self.config_dir) for line in src) + line.replace('MAGICDIR', config_dir) for line in src) return conf_path @@ -241,3 +245,47 @@ class TempDirTestCase(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) + + +def lock_and_call(func, lock_path): + """Grab a lock for lock_path and call func. + + :param callable func: object to call after acquiring the lock + :param str lock_path: path to file or directory to lock + + """ + # Reload module to reset internal _LOCKS dictionary + reload_module(util) + + # start child and wait for it to grab the lock + cv = multiprocessing.Condition() + cv.acquire() + child_args = (cv, lock_path,) + child = multiprocessing.Process(target=hold_lock, args=child_args) + child.start() + cv.wait() + + # call func and terminate the child + func() + cv.notify() + cv.release() + child.join() + assert child.exitcode == 0 + + +def hold_lock(cv, lock_path): # pragma: no cover + """Acquire a file lock at lock_path and wait to release it. + + :param multiprocessing.Condition cv: condition for syncronization + :param str lock_path: path to the file lock + + """ + from certbot import lock + if os.path.isdir(lock_path): + my_lock = lock.lock_dir(lock_path) + else: + my_lock = lock.LockFile(lock_path) + cv.acquire() + cv.notify() + cv.wait() + my_lock.release() diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index f021c04cf..59a4f10b2 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -2,11 +2,13 @@ import argparse import errno import os +import shutil import stat import unittest import mock import six +from six.moves import reload_module # pylint: disable=import-error from certbot import errors import certbot.tests.util as test_util @@ -73,19 +75,52 @@ class ExeExistsTest(unittest.TestCase): self.assertFalse(self._call("exe")) -class MakeOrVerifyCoreDirTest(test_util.TempDirTestCase): +class LockDirUntilExit(test_util.TempDirTestCase): + """Tests for certbot.util.lock_dir_until_exit.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.util import lock_dir_until_exit + return lock_dir_until_exit(*args, **kwargs) + + def setUp(self): + super(LockDirUntilExit, self).setUp() + # reset global state from other tests + import certbot.util + reload_module(certbot.util) + + @mock.patch('certbot.util.logger') + @mock.patch('certbot.util.atexit_register') + def test_it(self, mock_register, mock_logger): + subdir = os.path.join(self.tempdir, 'subdir') + os.mkdir(subdir) + self._call(self.tempdir) + self._call(subdir) + self._call(subdir) + + self.assertEqual(mock_register.call_count, 1) + registered_func = mock_register.call_args[0][0] + shutil.rmtree(subdir) + registered_func() # exception not raised + # logger.debug is only called once because the second call + # to lock subdir was ignored because it was already locked + self.assertEqual(mock_logger.debug.call_count, 1) + + +class SetUpCoreDirTest(test_util.TempDirTestCase): """Tests for certbot.util.make_or_verify_core_dir.""" def _call(self, *args, **kwargs): - from certbot.util import make_or_verify_core_dir - return make_or_verify_core_dir(*args, **kwargs) + from certbot.util import set_up_core_dir + return set_up_core_dir(*args, **kwargs) - def test_success(self): + @mock.patch('certbot.util.lock_dir_until_exit') + def test_success(self, mock_lock): new_dir = os.path.join(self.tempdir, 'new') self._call(new_dir, 0o700, os.geteuid(), False) self.assertTrue(os.path.exists(new_dir)) + self.assertEqual(mock_lock.call_count, 1) - @mock.patch('certbot.main.util.make_or_verify_dir') + @mock.patch('certbot.util.make_or_verify_dir') def test_failure(self, mock_make_or_verify): mock_make_or_verify.side_effect = OSError self.assertRaises(errors.Error, self._call, diff --git a/certbot/util.py b/certbot/util.py index 55a75097f..1cbef7e80 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -20,6 +20,13 @@ import configargparse from certbot import constants from certbot import errors +from certbot import lock + +try: + from collections import OrderedDict +except ImportError: # pragma: no cover + # OrderedDict was added in Python 2.7 + from ordereddict import OrderedDict # pylint: disable=import-error logger = logging.getLogger(__name__) @@ -47,6 +54,11 @@ PERM_ERR_FMT = os.linesep.join(( # Stores importing process ID to be used by atexit_register() _INITIAL_PID = os.getpid() +# Maps paths to locked directories to their lock object. All locks in +# the dict are attempted to be cleaned up at program exit. If the +# program exits before the lock is cleaned up, it is automatically +# released, but the file isn't deleted. +_LOCKS = OrderedDict() def run_script(params, log=logger.error): @@ -103,20 +115,47 @@ def exe_exists(exe): return False -def make_or_verify_core_dir(directory, mode, uid, strict): - """Make sure directory exists with proper permissions. +def lock_dir_until_exit(dir_path): + """Lock the directory at dir_path until program exit. + + :param str dir_path: path to directory + + :raises errors.LockError: if the lock is held by another process + + """ + if not _LOCKS: # this is the first lock to be released at exit + atexit_register(_release_locks) + + if dir_path not in _LOCKS: + _LOCKS[dir_path] = lock.lock_dir(dir_path) + + +def _release_locks(): + for dir_lock in six.itervalues(_LOCKS): + try: + dir_lock.release() + except: # pylint: disable=bare-except + msg = 'Exception occurred releasing lock: {0!r}'.format(dir_lock) + logger.debug(msg, exc_info=True) + + +def set_up_core_dir(directory, mode, uid, strict): + """Ensure directory exists with proper permissions and is locked. :param str directory: Path to a directory. :param int mode: Directory mode. :param int uid: Directory owner. :param bool strict: require directory to be owned by current user + :raises .errors.LockError: if the directory cannot be locked :raises .errors.Error: if the directory cannot be made or verified """ try: make_or_verify_dir(directory, mode, uid, strict) + lock_dir_until_exit(directory) except OSError as error: + logger.debug("Exception was:", exc_info=True) raise errors.Error(PERM_ERR_FMT.format(error)) diff --git a/setup.py b/setup.py index b0cf57810..b7697dbd3 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,10 @@ install_requires = [ # env markers cause problems with older pip and setuptools if sys.version_info < (2, 7): - install_requires.append('argparse') + install_requires.extend([ + 'argparse', + 'ordereddict', + ]) dev_extras = [ # Pin astroid==1.3.5, pylint==1.4.2 as a workaround for #289 diff --git a/tests/lock_test.py b/tests/lock_test.py new file mode 100644 index 000000000..4bb2865b4 --- /dev/null +++ b/tests/lock_test.py @@ -0,0 +1,238 @@ +"""Tests to ensure the lock order is preserved.""" +import atexit +import functools +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile + +from certbot import lock +from certbot import util + +from certbot.tests import util as test_util + + +logger = logging.getLogger(__name__) + + +def main(): + """Run the lock tests.""" + dirs, base_cmd = set_up() + for subcommand in ('certonly', 'install', 'renew', 'run',): + logger.info('Testing subcommand: %s', subcommand) + test_command(base_cmd + [subcommand], dirs) + logger.info('Lock test ran successfully.') + + +def set_up(): + """Prepare tests to be run. + + Logging is set up and temporary directories are set up to contain a + basic Certbot and Nginx configuration. The directories are returned + in the order they should be locked by Certbot. If the Nginx plugin + is expected to work on the system, the Nginx directory is included, + otherwise, it is not. + + A Certbot command is also created that uses the temporary + directories. The returned command can be used to test different + subcommands by appending the desired command to the end. + + :returns: directories and command + :rtype: `tuple` of `list` + + """ + logging.basicConfig(format='%(message)s', level=logging.INFO) + config_dir, logs_dir, work_dir, nginx_dir = set_up_dirs() + command = set_up_command(config_dir, logs_dir, work_dir, nginx_dir) + + dirs = [logs_dir, config_dir, work_dir] + # Travis and Circle CI set CI to true so we + # will always test Nginx's lock during CI + if os.environ.get('CI') == 'true' or util.exe_exists('nginx'): + dirs.append(nginx_dir) + else: + logger.warning('Skipping Nginx lock tests') + + return dirs, command + + +def set_up_dirs(): + """Set up directories for tests. + + A temporary directory is created to contain the config, log, work, + and nginx directories. A sample renewal configuration is created in + the config directory and a basic Nginx config is placed in the Nginx + directory. The temporary directory containing all of these + directories is deleted when the program exits. + + :return value: config, log, work, and nginx directories + :rtype: `tuple` of `str` + + """ + temp_dir = tempfile.mkdtemp() + logger.debug('Created temporary directory: %s', temp_dir) + atexit.register(functools.partial(shutil.rmtree, temp_dir)) + + config_dir = os.path.join(temp_dir, 'config') + logs_dir = os.path.join(temp_dir, 'logs') + work_dir = os.path.join(temp_dir, 'work') + nginx_dir = os.path.join(temp_dir, 'nginx') + + for directory in (config_dir, logs_dir, work_dir, nginx_dir,): + os.mkdir(directory) + + test_util.make_lineage(config_dir, 'sample-renewal.conf') + set_up_nginx_dir(nginx_dir) + + return config_dir, logs_dir, work_dir, nginx_dir + + +def set_up_nginx_dir(root_path): + """Create a basic Nginx configuration in nginx_dir. + + :param str root_path: where the Nginx server root should be placed + + """ + # Get the root of the git repository + repo_root = check_call('git rev-parse --show-toplevel'.split()).strip() + conf_script = os.path.join( + repo_root, 'certbot-nginx', 'tests', 'boulder-integration.conf.sh') + # boulder-integration.conf.sh uses the root environment variable as + # the Nginx server root when writing paths + os.environ['root'] = root_path + with open(os.path.join(root_path, 'nginx.conf'), 'w') as f: + f.write(check_call(['/bin/sh', conf_script])) + del os.environ['root'] + + +def set_up_command(config_dir, logs_dir, work_dir, nginx_dir): + """Build the Certbot command to run for testing. + + You can test different subcommands by appending the desired command + to the returned list. + + :param str config_dir: path to the configuration directory + :param str logs_dir: path to the logs directory + :param str work_dir: path to the work directory + :param str nginx_dir: path to the nginx directory + + :returns: certbot command to execute for testing + :rtype: `list` of `str` + + """ + return ( + 'certbot --cert-path {0} --key-path {1} --config-dir {2} ' + '--logs-dir {3} --work-dir {4} --nginx-server-root {5} --debug ' + '--force-renewal --nginx --verbose '.format( + test_util.vector_path('cert.pem'), + test_util.vector_path('rsa512_key.pem'), + config_dir, logs_dir, work_dir, nginx_dir).split()) + + +def test_command(command, directories): + """Assert Certbot acquires locks in a specific order. + + command is run repeatedly testing that Certbot acquires locks on + directories in the order they appear in the parameter directories. + + :param list command: Certbot command to execute + :param list directories: list of directories Certbot should fail + to acquire the lock on in sorted order + + """ + locks = [lock.lock_dir(directory) for directory in directories] + for dir_path, dir_lock in zip(directories, locks): + check_error(command, dir_path) + dir_lock.release() + + +def check_error(command, dir_path): + """Run command and verify it fails to acquire the lock for dir_path. + + :param str command: certbot command to run + :param str dir_path: path to directory containing the lock Certbot + should fail on + + """ + ret, out, err = subprocess_call(command) + if ret == 0: + report_failure("Certbot didn't exit with a nonzero status!", out, err) + + match = re.search("Please see the logfile '(.*)' for more details", err) + if match is not None: + # Get error output from more verbose logfile + with open(match.group(1)) as f: + err = f.read() + + pattern = 'A lock on {0}.* is held by another process'.format(dir_path) + if not re.search(pattern, err): + err_msg = 'Directory path {0} not in error output!'.format(dir_path) + report_failure(err_msg, out, err) + + +def check_call(args): + """Simple imitation of subprocess.check_call. + + This function is only available in subprocess in Python 2.7+. + + :param list args: program and it's arguments to be run + + :returns: stdout output + :rtype: str + + """ + exit_code, out, err = subprocess_call(args) + if exit_code: + report_failure('Command exited with a nonzero status!', out, err) + return out + + +def report_failure(err_msg, out, err): + """Report a subprocess failure and exit. + + :param str err_msg: error message to report + :param str out: stdout output + :param str err: stderr output + + """ + logger.fatal(err_msg) + log_output(logging.INFO, out, err) + sys.exit(err_msg) + + +def subprocess_call(args): + """Run a command with subprocess and return the result. + + :param list args: program and it's arguments to be run + + :returns: return code, stdout output, stderr output + :rtype: tuple + + """ + process = subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + out, err = process.communicate() + logger.debug('Return code was %d', process.returncode) + log_output(logging.DEBUG, out, err) + return process.returncode, out, err + + +def log_output(level, out, err): + """Logs stdout and stderr output at the requested level. + + :param int level: logging level to use + :param str out: stdout output + :param str err: stderr output + + """ + if out: + logger.log(level, 'Stdout output was:\n%s', out) + if err: + logger.log(level, 'Stderr output was:\n%s', err) + + +if __name__ == "__main__": + main() diff --git a/tox.ini b/tox.ini index d393bb610..33cacf061 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ commands = nosetests -v certbot_nginx --processes=-1 pip install -e letshelp-certbot nosetests -v letshelp_certbot --processes=-1 + python tests/lock_test.py setenv = PYTHONPATH = {toxinidir} @@ -59,7 +60,7 @@ basepython = python2.7 # continue, but tox return code will reflect previous error commands = pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot - pylint --reports=n --rcfile=.pylintrc acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot + pylint --reports=n --rcfile=.pylintrc acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot tests/lock_test.py [testenv:mypy] basepython = python3.4 From 79d5c890c3caf242cb66d9b040deacae7460b4dc Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 1 May 2017 14:55:31 -0700 Subject: [PATCH 44/96] Add a timeout to prompts (#4601) * Add input_with_timeout * use input_with_timeout --- certbot/display/util.py | 43 ++++++++++++++--- certbot/tests/display/util_test.py | 75 +++++++++++++++++++++++------- 2 files changed, 95 insertions(+), 23 deletions(-) diff --git a/certbot/display/util.py b/certbot/display/util.py index 4d69f1263..7e797e71a 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -1,10 +1,10 @@ """Certbot display.""" import logging import os -import textwrap +import select import sys +import textwrap -import six import zope.interface from certbot import constants @@ -51,6 +51,37 @@ def _wrap_lines(msg): return os.linesep.join(fixed_l) + +def input_with_timeout(prompt=None, timeout=36000.0): + """Get user input with a timeout. + + Behaves the same as six.moves.input, however, an error is raised if + a user doesn't answer after timeout seconds. The default timeout + value was chosen to place it just under 12 hours for users following + our advice and running Certbot twice a day. + + :param str prompt: prompt to provide for input + :param float timeout: maximum number of seconds to wait for input + + :returns: user response + :rtype: str + + :raises errors.Error if no answer is given before the timeout + + """ + if prompt: + sys.stdout.write(prompt) + sys.stdout.flush() + + # select can only be used like this on UNIX + rlist, _, _ = select.select([sys.stdin], [], [], timeout) + if not rlist: + raise errors.Error( + "Timed out waiting for answer to prompt '{0}'".format(prompt)) + + return rlist[0].readline().rstrip('\n') + + @zope.interface.implementer(interfaces.IDisplay) class FileDisplay(object): """File-based display.""" @@ -83,7 +114,7 @@ class FileDisplay(object): line=os.linesep, frame=side_frame, msg=message)) if pause: if self._can_interact(force_interactive): - six.moves.input("Press Enter to Continue") + input_with_timeout("Press Enter to Continue") else: logger.debug("Not pausing for user confirmation") @@ -140,7 +171,7 @@ class FileDisplay(object): if self._return_default(message, default, cli_flag, force_interactive): return OK, default - ans = six.moves.input( + ans = input_with_timeout( textwrap.fill( "%s (Enter 'c' to cancel): " % message, 80, @@ -182,7 +213,7 @@ class FileDisplay(object): os.linesep, frame=side_frame, msg=message)) while True: - ans = six.moves.input("{yes}/{no}: ".format( + ans = input_with_timeout("{yes}/{no}: ".format( yes=_parens_around_char(yes_label), no=_parens_around_char(no_label))) @@ -388,7 +419,7 @@ class FileDisplay(object): input_msg = ("Press 1 [enter] to confirm the selection " "(press 'c' to cancel): ") while selection < 1: - ans = six.moves.input(input_msg) + ans = input_with_timeout(input_msg) if ans.startswith("c") or ans.startswith("C"): return CANCEL, -1 try: diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index f4d69b50d..a872917ec 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -1,8 +1,10 @@ """Test :mod:`certbot.display.util`.""" import inspect import os +import socket import unittest +import six import mock from certbot import errors @@ -15,6 +17,39 @@ CHOICES = [("First", "Description1"), ("Second", "Description2")] TAGS = ["tag1", "tag2", "tag3"] TAGS_CHOICES = [("1", "tag1"), ("2", "tag2"), ("3", "tag3")] + +class InputWithTimeoutTest(unittest.TestCase): + """Tests for certbot.display.util.input_with_timeout.""" + @classmethod + def _call(cls, *args, **kwargs): + from certbot.display.util import input_with_timeout + return input_with_timeout(*args, **kwargs) + + def setUp(self): + self.expected_msg = "foo bar" + self.stdin = six.StringIO(self.expected_msg + "\n") + + def test_input(self, prompt=None): + with mock.patch("certbot.display.util.select.select") as mock_select: + mock_select.return_value = ([self.stdin], [], [],) + self.assertEqual(display_util.input_with_timeout(prompt), + self.expected_msg) + + @mock.patch("certbot.display.util.sys.stdout") + def test_input_with_prompt(self, mock_stdout): + prompt = "test prompt: " + self.test_input(prompt) + mock_stdout.write.assert_called_once_with(prompt) + mock_stdout.flush.assert_called_once_with() + + def test_timeout(self): + stdin = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + stdin.bind(('', 0)) + stdin.listen(1) + with mock.patch("certbot.display.util.sys.stdin", stdin): + self.assertRaises(errors.Error, self._call, timeout=0.001) + + class FileOutputDisplayTest(unittest.TestCase): """Test stdout display. @@ -35,7 +70,8 @@ class FileOutputDisplayTest(unittest.TestCase): self.assertTrue("message" in string) def test_notification_pause(self): - with mock.patch("six.moves.input", return_value="enter"): + input_with_timeout = "certbot.display.util.input_with_timeout" + with mock.patch(input_with_timeout, return_value="enter"): self.displayer.notification("message", force_interactive=True) self.assertTrue("message" in self.mock_stdout.write.call_args[0][0]) @@ -72,13 +108,15 @@ class FileOutputDisplayTest(unittest.TestCase): self.assertEqual(result, (display_util.OK, default)) def test_input_cancel(self): - with mock.patch("six.moves.input", return_value="c"): + input_with_timeout = "certbot.display.util.input_with_timeout" + with mock.patch(input_with_timeout, return_value="c"): code, _ = self.displayer.input("message", force_interactive=True) self.assertTrue(code, display_util.CANCEL) def test_input_normal(self): - with mock.patch("six.moves.input", return_value="domain.com"): + input_with_timeout = "certbot.display.util.input_with_timeout" + with mock.patch(input_with_timeout, return_value="domain.com"): code, input_ = self.displayer.input("message", force_interactive=True) self.assertEqual(code, display_util.OK) @@ -104,23 +142,24 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer.input, "msg", cli_flag="--flag") def test_yesno(self): - with mock.patch("six.moves.input", return_value="Yes"): + input_with_timeout = "certbot.display.util.input_with_timeout" + with mock.patch(input_with_timeout, return_value="Yes"): self.assertTrue(self.displayer.yesno( "message", force_interactive=True)) - with mock.patch("six.moves.input", return_value="y"): + with mock.patch(input_with_timeout, return_value="y"): self.assertTrue(self.displayer.yesno( "message", force_interactive=True)) - with mock.patch("six.moves.input", side_effect=["maybe", "y"]): + with mock.patch(input_with_timeout, side_effect=["maybe", "y"]): self.assertTrue(self.displayer.yesno( "message", force_interactive=True)) - with mock.patch("six.moves.input", return_value="No"): + with mock.patch(input_with_timeout, return_value="No"): self.assertFalse(self.displayer.yesno( "message", force_interactive=True)) - with mock.patch("six.moves.input", side_effect=["cancel", "n"]): + with mock.patch(input_with_timeout, side_effect=["cancel", "n"]): self.assertFalse(self.displayer.yesno( "message", force_interactive=True)) - with mock.patch("six.moves.input", return_value="a"): + with mock.patch(input_with_timeout, return_value="a"): self.assertTrue(self.displayer.yesno( "msg", yes_label="Agree", force_interactive=True)) @@ -128,7 +167,7 @@ class FileOutputDisplayTest(unittest.TestCase): self.assertTrue(self._force_noninteractive( self.displayer.yesno, "message", default=True)) - @mock.patch("certbot.display.util.six.moves.input") + @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_valid(self, mock_input): mock_input.return_value = "2 1" code, tag_list = self.displayer.checklist( @@ -136,21 +175,21 @@ class FileOutputDisplayTest(unittest.TestCase): self.assertEqual( (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2"]))) - @mock.patch("certbot.display.util.six.moves.input") + @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_empty(self, mock_input): mock_input.return_value = "" code, tag_list = self.displayer.checklist("msg", TAGS, force_interactive=True) self.assertEqual( (code, set(tag_list)), (display_util.OK, set(["tag1", "tag2", "tag3"]))) - @mock.patch("certbot.display.util.six.moves.input") + @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_miss_valid(self, mock_input): mock_input.side_effect = ["10", "tag1 please", "1"] ret = self.displayer.checklist("msg", TAGS, force_interactive=True) self.assertEqual(ret, (display_util.OK, ["tag1"])) - @mock.patch("certbot.display.util.six.moves.input") + @mock.patch("certbot.display.util.input_with_timeout") def test_checklist_miss_quit(self, mock_input): mock_input.side_effect = ["10", "c"] @@ -182,7 +221,7 @@ class FileOutputDisplayTest(unittest.TestCase): self.displayer._scrub_checklist_input(list_, TAGS)) self.assertEqual(set_tags, exp[i]) - @mock.patch("certbot.display.util.six.moves.input") + @mock.patch("certbot.display.util.input_with_timeout") def test_directory_select(self, mock_input): # pylint: disable=star-args args = ["msg", "/var/www/html", "--flag", True] @@ -246,11 +285,12 @@ class FileOutputDisplayTest(unittest.TestCase): def test_get_valid_int_ans_valid(self): # pylint: disable=protected-access - with mock.patch("six.moves.input", return_value="1"): + input_with_timeout = "certbot.display.util.input_with_timeout" + with mock.patch(input_with_timeout, return_value="1"): self.assertEqual( self.displayer._get_valid_int_ans(1), (display_util.OK, 1)) ans = "2" - with mock.patch("six.moves.input", return_value=ans): + with mock.patch(input_with_timeout, return_value=ans): self.assertEqual( self.displayer._get_valid_int_ans(3), (display_util.OK, int(ans))) @@ -262,8 +302,9 @@ class FileOutputDisplayTest(unittest.TestCase): ["4", "one", "C"], ["c"], ] + input_with_timeout = "certbot.display.util.input_with_timeout" for ans in answers: - with mock.patch("six.moves.input", side_effect=ans): + with mock.patch(input_with_timeout, side_effect=ans): self.assertEqual( self.displayer._get_valid_int_ans(3), (display_util.CANCEL, -1)) From f57f35b1ddc488d2d28bbc387f1c27fe470fb2cc Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Tue, 26 Jul 2016 15:57:11 -0700 Subject: [PATCH 45/96] Start work on multivhost support in Apache * get through parsing * not slice * add mult vhost per file * idx line backwards * blocks be wrong * always close ifmod * let's not mess up indexes * don't double add multi * fix some lint, only dedupe multi * tests * fix lint * in progress bit flip * try to pick the right vhost * take Dominic's suggestion * don't redo search * add ancestor * we now support multiple vhosts * yay * add docstrings --- certbot-apache/certbot_apache/configurator.py | 176 ++++++++++------ certbot-apache/certbot_apache/obj.py | 3 +- .../certbot_apache/tests/configurator_test.py | 64 +++++- .../multi_vhosts/apache2/apache2.conf | 196 ++++++++++++++++++ .../multi_vhosts/apache2/envvars | 29 +++ .../multi_vhosts/apache2/ports.conf | 15 ++ .../apache2/sites-available/default.conf | 22 ++ .../apache2/sites-enabled/placeholder.conf | 0 certbot-apache/certbot_apache/tests/util.py | 17 ++ certbot/reverter.py | 2 +- 10 files changed, 453 insertions(+), 71 deletions(-) create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/apache2.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/envvars create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/ports.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/default.conf create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/placeholder.conf diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 39d25619d..aec1d692b 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -138,6 +138,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} + self._skeletons = {} @property def mod_ssl_conf(self): @@ -589,6 +590,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if realpath not in vhost_paths.keys(): vhs.append(new_vhost) vhost_paths[realpath] = new_vhost.filep + elif (realpath in vhost_paths.keys() + and new_vhost.path.endswith("]") and new_vhost not in vhs): + vhs.append(new_vhost) elif realpath == new_vhost.filep: # Prefer "real" vhost paths instead of symlinked ones # ex: sites-enabled/vh.conf -> sites-available/vh.conf @@ -792,20 +796,21 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): avail_fp = nonssl_vhost.filep ssl_fp = self._get_ssl_vhost_path(avail_fp) - self._copy_create_ssl_vhost_skeleton(avail_fp, ssl_fp) + vhost_num = -1 + if nonssl_vhost.path.endswith("]"): + # augeas doesn't zero index for whatever reason + vhost_num = int(nonssl_vhost.path[-2]) - 1 + self._copy_create_ssl_vhost_skeleton(avail_fp, ssl_fp, vhost_num) # Reload augeas to take into account the new vhost self.aug.load() # Get Vhost augeas path for new vhost vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % (self._escape(ssl_fp), parser.case_i("VirtualHost"))) - if len(vh_p) != 1: - logger.error("Error: should only be one vhost in %s", avail_fp) - raise errors.PluginError("Currently, we only support " - "configurations with one vhost per file") - else: - # This simplifies the process - vh_p = vh_p[0] + temp_vh = vh_p[0] + if self._skeletons[ssl_fp]: + temp_vh = vh_p[len(self._skeletons[ssl_fp]) -1] + vh_p = temp_vh # Update Addresses self._update_ssl_vhosts_addrs(vh_p) @@ -822,6 +827,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # We know the length is one because of the assertion above # Create the Vhost object ssl_vhost = self._create_vhost(vh_p) + ssl_vhost.ancestor = nonssl_vhost self.vhosts.append(ssl_vhost) # NOTE: Searches through Augeas seem to ruin changes to directives @@ -875,7 +881,38 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Sift line if it redirects the request to a HTTPS site return target.startswith("https://") - def _copy_create_ssl_vhost_skeleton(self, avail_fp, ssl_fp): + def _section_blocks(self, blocks): + """A helper function for _create_block_segments that makes + a list of line numbers to not include in the return. + + :param list blocks: A list of indexes of where vhosts start and end. + + """ + out = [] + while len(blocks) > 1: + start = blocks[0] + end = blocks[1] + 1 + out += range(start, end) + blocks = blocks[2:] + return out + + def _create_block_segments(self, orig_file_list, vhost_num): + """A helper function for _copy_create_ssl_vhost_skeleton + that slices the appropriate vhost from the origin conf file. + + :param list orig_file_list: the original file converted to a list of strings. + "param int vhost_num: Which vhost the vhost is in the origin multivhost file. + + """ + blocks = [idx for idx, line in enumerate(orig_file_list) + if line.lstrip().startswith(" skeleton. :param str avail_fp: Pointer to the original available non-ssl vhost @@ -891,65 +928,77 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): try: with open(avail_fp, "r") as orig_file: - with open(ssl_fp, "w") as new_file: - new_file.write("\n") + orig_file_list = [line for line in orig_file] + if vhost_num != -1: + orig_file_list = self._create_block_segments(orig_file_list, vhost_num) - comment = ("# Some rewrite rules in this file were " - "disabled on your HTTPS site,\n" - "# because they have the potential to create " - "redirection loops.\n") + if ssl_fp in self._skeletons: + bit = "a" + self._skeletons[ssl_fp].append(avail_fp) + else: + bit = "w" + self._skeletons[ssl_fp] = [avail_fp] - for line in orig_file: - A = line.lstrip().startswith("RewriteCond") - B = line.lstrip().startswith("RewriteRule") + with open(ssl_fp, bit) as new_file: + new_file.write("\n") - if not (A or B): - new_file.write(line) - continue + comment = ("# Some rewrite rules in this file were " + "disabled on your HTTPS site,\n" + "# because they have the potential to create " + "redirection loops.\n") - # A RewriteRule that doesn't need filtering - if B and not self._sift_rewrite_rule(line): - new_file.write(line) - continue + orig_file_list = iter(orig_file_list) + for line in orig_file_list: + A = line.lstrip().startswith("RewriteCond") + B = line.lstrip().startswith("RewriteRule") - # A RewriteRule that does need filtering - if B and self._sift_rewrite_rule(line): + if not (A or B): + new_file.write(line) + continue + + # A RewriteRule that doesn't need filtering + if B and not self._sift_rewrite_rule(line): + new_file.write(line) + continue + + # A RewriteRule that does need filtering + if B and self._sift_rewrite_rule(line): + if not sift: + new_file.write(comment) + sift = True + new_file.write("# " + line) + continue + + # We save RewriteCond(s) and their corresponding + # RewriteRule in 'chunk'. + # We then decide whether we comment out the entire + # chunk based on its RewriteRule. + chunk = [] + if A: + chunk.append(line) + line = next(orig_file_list) + + # RewriteCond(s) must be followed by one RewriteRule + while not line.lstrip().startswith("RewriteRule"): + chunk.append(line) + line = next(orig_file_list) + + # Now, current line must start with a RewriteRule + chunk.append(line) + + if self._sift_rewrite_rule(line): if not sift: new_file.write(comment) sift = True - new_file.write("# " + line) + + new_file.write(''.join( + ['# ' + l for l in chunk])) + continue + else: + new_file.write(''.join(chunk)) continue - # We save RewriteCond(s) and their corresponding - # RewriteRule in 'chunk'. - # We then decide whether we comment out the entire - # chunk based on its RewriteRule. - chunk = [] - if A: - chunk.append(line) - line = next(orig_file) - - # RewriteCond(s) must be followed by one RewriteRule - while not line.lstrip().startswith("RewriteRule"): - chunk.append(line) - line = next(orig_file) - - # Now, current line must start with a RewriteRule - chunk.append(line) - - if self._sift_rewrite_rule(line): - if not sift: - new_file.write(comment) - sift = True - - new_file.write(''.join( - ['# ' + l for l in chunk])) - continue - else: - new_file.write(''.join(chunk)) - continue - - new_file.write("\n") + new_file.write("\n") except IOError: logger.fatal("Error writing/reading to file in make_vhost_ssl") raise errors.PluginError("Unable to write/read in make_vhost_ssl") @@ -962,6 +1011,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "the potential to create redirection loops.".format(avail_fp, ssl_fp), reporter.MEDIUM_PRIORITY) + self.aug.set("/augeas/files%s/mtime" %(self._escape(ssl_fp)), "0") + self.aug.set("/augeas/files%s/mtime" %(self._escape(avail_fp)), "0") def _update_ssl_vhosts_addrs(self, vh_path): ssl_addrs = set() @@ -1008,12 +1059,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.parser.add_dir(vh_path, "Include", self.mod_ssl_conf) def _add_servername_alias(self, target_name, vhost): - fp = self._escape(vhost.filep) - vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % - (fp, parser.case_i("VirtualHost"))) - if not vh_p: - return - vh_path = vh_p[0] + vh_path = vhost.path if (self.parser.find_dir("ServerName", target_name, start=vh_path, exclude=False) or self.parser.find_dir("ServerAlias", target_name, @@ -1508,6 +1554,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def _get_http_vhost(self, ssl_vhost): """Find appropriate HTTP vhost for ssl_vhost.""" # First candidate vhosts filter + if ssl_vhost.ancestor: + return ssl_vhost.ancestor candidate_http_vhs = [ vhost for vhost in self.vhosts if not vhost.ssl ] diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index 30cb24844..fd743fe79 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -123,7 +123,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods strip_name = re.compile(r"^(?:.+://)?([^ :$]*)") def __init__(self, filep, path, addrs, ssl, enabled, name=None, - aliases=None, modmacro=False): + aliases=None, modmacro=False, ancestor=None): # pylint: disable=too-many-arguments """Initialize a VH.""" @@ -135,6 +135,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods self.ssl = ssl self.enabled = enabled self.modmacro = modmacro + self.ancestor = ancestor def get_names(self): """Return a set of all names.""" diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 45e701bd5..3dc3790a1 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -655,11 +655,6 @@ class MultipleVhostsTest(util.ApacheTest): len(self.config.parser.find_dir( directive, None, self.vh_truth[1].path, False)), 0) - def test_make_vhost_ssl_extra_vhs(self): - self.config.aug.match = mock.Mock(return_value=["p1", "p2"]) - self.assertRaises( - errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0]) - def test_make_vhost_ssl_bad_write(self): mock_open = mock.mock_open() # This calls open @@ -1345,6 +1340,65 @@ class AugeasVhostsTest(util.ApacheTest): self.config.choose_vhost(name) self.assertEqual(mock_select.call_count, 0) +class MultiVhostsTest(util.ApacheTest): + """Test vhosts with illegal names dependant on augeas version.""" + # pylint: disable=protected-access + + def setUp(self): # pylint: disable=arguments-differ + td = "debian_apache_2_4/multi_vhosts" + cr = "debian_apache_2_4/multi_vhosts/apache2" + vr = "debian_apache_2_4/multi_vhosts/apache2/sites-available" + super(MultiVhostsTest, self).setUp(test_dir=td, + config_root=cr, + vhost_root=vr) + + self.config = util.get_apache_configurator( + self.config_path, self.vhost_path, self.config_dir, self.work_dir) + self.vh_truth = util.get_vh_truth( + self.temp_dir, "debian_apache_2_4/multi_vhosts") + + def tearDown(self): + shutil.rmtree(self.temp_dir) + shutil.rmtree(self.config_dir) + shutil.rmtree(self.work_dir) + + def test_make_vhost_ssl(self): + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[1]) + + self.assertEqual( + ssl_vhost.filep, + os.path.join(self.config_path, "sites-available", + "default-le-ssl.conf")) + + self.assertEqual(ssl_vhost.path, + "/files" + ssl_vhost.filep + "/IfModule/VirtualHost") + self.assertEqual(len(ssl_vhost.addrs), 1) + self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs) + self.assertEqual(ssl_vhost.name, "banana.vomit.com") + self.assertTrue(ssl_vhost.ssl) + self.assertFalse(ssl_vhost.enabled) + + self.assertTrue(self.config.parser.find_dir( + "SSLCertificateFile", None, ssl_vhost.path, False)) + self.assertTrue(self.config.parser.find_dir( + "SSLCertificateKeyFile", None, ssl_vhost.path, False)) + + self.assertEqual(self.config.is_name_vhost(self.vh_truth[1]), + self.config.is_name_vhost(ssl_vhost)) + + def test_make_2nd_vhost_ssl(self): + _ = self.config.make_vhost_ssl(self.vh_truth[0]) + _ = self.config.make_vhost_ssl(self.vh_truth[1]) + self.assertEqual( + len(self.config._skeletons[self.config._get_ssl_vhost_path(self.vh_truth[0].filep)]), 2) + + def test_cover_is_stupid_and_I_hate_it(self): + http_vhost = obj.VirtualHost(None, None, None, False, False, name="Noah") + ssl_vhost = obj.VirtualHost(None, None, None, False, False, name="Noah") + self.config.vhosts.append(http_vhost) + self.assertEqual(self.config._get_http_vhost(ssl_vhost), http_vhost) + + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/apache2.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/apache2.conf new file mode 100644 index 000000000..2a5bb7be2 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/apache2.conf @@ -0,0 +1,196 @@ +# This is the main Apache server configuration file. It contains the +# configuration directives that give the server its instructions. +# See http://httpd.apache.org/docs/2.4/ for detailed information about +# the directives and /usr/share/doc/apache2/README.Debian about Debian specific +# hints. +# +# +# Summary of how the Apache 2 configuration works in Debian: +# The Apache 2 web server configuration in Debian is quite different to +# upstream's suggested way to configure the web server. This is because Debian's +# default Apache2 installation attempts to make adding and removing modules, +# virtual hosts, and extra configuration directives as flexible as possible, in +# order to make automating the changes and administering the server as easy as +# possible. + +# It is split into several files forming the configuration hierarchy outlined +# below, all located in the /etc/apache2/ directory: +# +# /etc/apache2/ +# |-- apache2.conf +# | `-- ports.conf +# |-- mods-enabled +# | |-- *.load +# | `-- *.conf +# |-- conf-enabled +# | `-- *.conf +# `-- sites-enabled +# `-- *.conf +# +# +# * apache2.conf is the main configuration file (this file). It puts the pieces +# together by including all remaining configuration files when starting up the +# web server. +# +# * ports.conf is always included from the main configuration file. It is +# supposed to determine listening ports for incoming connections which can be +# customized anytime. +# +# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/ +# directories contain particular configuration snippets which manage modules, +# global configuration fragments, or virtual host configurations, +# respectively. +# +# They are activated by symlinking available configuration files from their +# respective *-available/ counterparts. These should be managed by using our +# helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See +# their respective man pages for detailed information. +# +# * The binary is called apache2. Due to the use of environment variables, in +# the default configuration, apache2 needs to be started/stopped with +# /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not +# work with the default configuration. + + +# Global configuration + +# +# The accept serialization lock file MUST BE STORED ON A LOCAL DISK. +# +Mutex file:${APACHE_LOCK_DIR} default + +# +# PidFile: The file in which the server should record its process +# identification number when it starts. +# This needs to be set in /etc/apache2/envvars +# +PidFile ${APACHE_PID_FILE} + +# +# Timeout: The number of seconds before receives and sends time out. +# +Timeout 300 + +# +# KeepAlive: Whether or not to allow persistent connections (more than +# one request per connection). Set to "Off" to deactivate. +# +KeepAlive On + +# +# MaxKeepAliveRequests: The maximum number of requests to allow +# during a persistent connection. Set to 0 to allow an unlimited amount. +# We recommend you leave this number high, for maximum performance. +# +MaxKeepAliveRequests 100 + +# +# KeepAliveTimeout: Number of seconds to wait for the next request from the +# same client on the same connection. +# +KeepAliveTimeout 5 + + +# These need to be set in /etc/apache2/envvars +User ${APACHE_RUN_USER} +Group ${APACHE_RUN_GROUP} + +# +# HostnameLookups: Log the names of clients or just their IP addresses +# e.g., www.apache.org (on) or 204.62.129.132 (off). +# The default is off because it'd be overall better for the net if people +# had to knowingly turn this feature on, since enabling it means that +# each client request will result in AT LEAST one lookup request to the +# nameserver. +# +HostnameLookups Off + +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog ${APACHE_LOG_DIR}/error.log + +# +# LogLevel: Control the severity of messages logged to the error_log. +# Available values: trace8, ..., trace1, debug, info, notice, warn, +# error, crit, alert, emerg. +# It is also possible to configure the log level for particular modules, e.g. +# "LogLevel info ssl:warn" +# +LogLevel warn + +# Include module configuration: +IncludeOptional mods-enabled/*.load +IncludeOptional mods-enabled/*.conf + +# Include list of ports to listen on +Include ports.conf + + +# Sets the default security model of the Apache2 HTTPD server. It does +# not allow access to the root filesystem outside of /usr/share and /var/www. +# The former is used by web applications packaged in Debian, +# the latter may be used for local directories served by the web server. If +# your system is serving content from a sub-directory in /srv you must allow +# access here, or in any related virtual host. + + Options FollowSymLinks + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + +# AccessFileName: The name of the file to look for in each directory +# for additional configuration directives. See also the AllowOverride +# directive. +# +AccessFileName .htaccess + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# The following directives define some format nicknames for use with +# a CustomLog directive. +# +# These deviate from the Common Log Format definitions in that they use %O +# (the actual bytes sent including headers) instead of %b (the size of the +# requested file), because the latter makes it impossible to detect partial +# requests. +# +# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended. +# Use mod_remoteip instead. +# +LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined +LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %O" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent + +# Include of directories ignores editors' and dpkg's backup files, +# see README.Debian for details. + +# Include generic snippets of statements +IncludeOptional conf-enabled/*.conf + +# Include the virtual host configurations: +IncludeOptional sites-enabled/*.conf + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/envvars b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/envvars new file mode 100644 index 000000000..a13d9a89e --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/envvars @@ -0,0 +1,29 @@ +# envvars - default environment variables for apache2ctl + +# this won't be correct after changing uid +unset HOME + +# for supporting multiple apache2 instances +if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then + SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}" +else + SUFFIX= +fi + +# Since there is no sane way to get the parsed apache2 config in scripts, some +# settings are defined via environment variables and then used in apache2ctl, +# /etc/init.d/apache2, /etc/logrotate.d/apache2, etc. +export APACHE_RUN_USER=www-data +export APACHE_RUN_GROUP=www-data +# temporary state file location. This might be changed to /run in Wheezy+1 +export APACHE_PID_FILE=/var/run/apache2/apache2$SUFFIX.pid +export APACHE_RUN_DIR=/var/run/apache2$SUFFIX +export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX +# Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. +export APACHE_LOG_DIR=/var/log/apache2$SUFFIX + +## The locale used by some modules like mod_dav +export LANG=C + +export LANG + diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/ports.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/ports.conf new file mode 100644 index 000000000..5daec58c1 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/ports.conf @@ -0,0 +1,15 @@ +# If you just change the port or add more ports here, you will likely also +# have to change the VirtualHost statement in +# /etc/apache2/sites-enabled/000-default.conf + +Listen 80 + + + Listen 443 + + + + Listen 443 + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/default.conf new file mode 100644 index 000000000..6ab206b2d --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/default.conf @@ -0,0 +1,22 @@ + + + ServerName banana.vomit.net + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + + + + + ServerName banana.vomit.com + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/placeholder.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/placeholder.conf new file mode 100644 index 000000000..e69de29bb diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 3c33a0e19..90880ac5d 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -164,5 +164,22 @@ def get_vh_truth(temp_dir, config_name): set([obj.Addr.fromstring("10.2.3.4:443")]), True, True, "ocspvhost.com")] return vh_truth + if config_name == "debian_apache_2_4/multi_vhosts": + prefix = os.path.join( + temp_dir, config_name, "apache2/sites-available") + aug_pre = "/files" + prefix + vh_truth = [ + obj.VirtualHost( + os.path.join(prefix, "default.conf"), + os.path.join(aug_pre, "default.conf/VirtualHost[1]"), + set([obj.Addr.fromstring("*:80")]), + False, True, "ip-172-30-0-17"), + obj.VirtualHost( + os.path.join(prefix, "default.conf"), + os.path.join(aug_pre, "default.conf/VirtualHost[2]"), + set([obj.Addr.fromstring("*:80")]), + False, True, "banana.vomit.com")] + return vh_truth + return None # pragma: no cover diff --git a/certbot/reverter.py b/certbot/reverter.py index 32355782e..34feafc7e 100644 --- a/certbot/reverter.py +++ b/certbot/reverter.py @@ -491,7 +491,7 @@ class Reverter(object): else: logger.warning( "File: %s - Could not be found to be deleted %s - " - "LE probably shut down unexpectedly", + "Certbot probably shut down unexpectedly", os.linesep, path) except (IOError, OSError): logger.fatal( From 2613a8b5795e9171be662e63ad641e66283d484b Mon Sep 17 00:00:00 2001 From: Peter Eckersley Date: Wed, 28 Sep 2016 14:49:21 -0700 Subject: [PATCH 46/96] Continue work on Apache multivhost * Apache: do not assume directives will be CamelCased * Fixup * Elaborate * Simplify the definition of vh_p --- certbot-apache/certbot_apache/configurator.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index aec1d692b..e555e7610 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -805,12 +805,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Reload augeas to take into account the new vhost self.aug.load() # Get Vhost augeas path for new vhost - vh_p = self.aug.match("/files%s//* [label()=~regexp('%s')]" % - (self._escape(ssl_fp), parser.case_i("VirtualHost"))) - temp_vh = vh_p[0] - if self._skeletons[ssl_fp]: - temp_vh = vh_p[len(self._skeletons[ssl_fp]) -1] - vh_p = temp_vh + matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" % + (self._escape(ssl_fp), parser.case_i("VirtualHost"))) + skels = self._skeletons[ssl_fp] + vh_p = matches[len(skels) - 1] if skels else matches[0] # Update Addresses self._update_ssl_vhosts_addrs(vh_p) @@ -865,7 +863,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: bool """ - if not line.lstrip().startswith("RewriteRule"): + if not line.lower().lstrip().startswith("rewriterule"): return False # According to: http://httpd.apache.org/docs/2.4/rewrite/flags.html @@ -887,6 +885,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param list blocks: A list of indexes of where vhosts start and end. + For instance, turns [2,5,13,15] into [[2,3,4,5], [13,14,15]]. + """ out = [] while len(blocks) > 1: @@ -949,8 +949,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): orig_file_list = iter(orig_file_list) for line in orig_file_list: - A = line.lstrip().startswith("RewriteCond") - B = line.lstrip().startswith("RewriteRule") + A = line.lower().lstrip().startswith("rewritecond") + B = line.lower().lstrip().startswith("rewriterule") if not (A or B): new_file.write(line) @@ -979,7 +979,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): line = next(orig_file_list) # RewriteCond(s) must be followed by one RewriteRule - while not line.lstrip().startswith("RewriteRule"): + while not line.lower().lstrip().startswith("rewriterule"): chunk.append(line) line = next(orig_file_list) From 6b26015752c4715d00241d078932e100ae12e189 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 29 Sep 2016 19:32:22 -0700 Subject: [PATCH 47/96] Further Apache multivhost improvements * Don't filter vhosts on path if you've done so already * add get_internal_aug_path * Use relative augeas paths to determine if a file contains multiple virtual hosts --- certbot-apache/certbot_apache/configurator.py | 86 +++++++++++++------ .../certbot_apache/tests/configurator_test.py | 12 ++- 2 files changed, 71 insertions(+), 27 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index e555e7610..3aaafca4b 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -574,8 +574,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Search base config, and all included paths for VirtualHosts + file_paths = {} + internal_paths = defaultdict(set) vhs = [] - vhost_paths = {} for vhost_path in self.parser.parser_paths.keys(): paths = self.aug.match( ("/files%s//*[label()=~regexp('%s')]" % @@ -586,21 +587,33 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): new_vhost = self._create_vhost(path) if not new_vhost: continue + internal_path = get_internal_aug_path(new_vhost.path) realpath = os.path.realpath(new_vhost.filep) - if realpath not in vhost_paths.keys(): + if realpath not in file_paths: + file_paths[realpath] = new_vhost.filep + internal_paths[realpath].add(internal_path) vhs.append(new_vhost) - vhost_paths[realpath] = new_vhost.filep - elif (realpath in vhost_paths.keys() - and new_vhost.path.endswith("]") and new_vhost not in vhs): - vhs.append(new_vhost) - elif realpath == new_vhost.filep: + elif (realpath == new_vhost.filep and + realpath != file_paths[realpath]): # Prefer "real" vhost paths instead of symlinked ones # ex: sites-enabled/vh.conf -> sites-available/vh.conf # remove old (most likely) symlinked one - vhs = [v for v in vhs if v.filep != vhost_paths[realpath]] + new_vhs = [] + for v in vhs: + if v.filep == file_paths[realpath]: + internal_paths[realpath].remove( + get_internal_aug_path(v.path)) + else: + new_vhs.append(v) + vhs = new_vhs + + file_paths[realpath] = realpath + internal_paths[realpath].add(internal_path) + vhs.append(new_vhost) + elif internal_path not in internal_paths[realpath]: + internal_paths[realpath].add(internal_path) vhs.append(new_vhost) - vhost_paths[realpath] = realpath return vhs @@ -1867,25 +1880,46 @@ def get_file_path(vhost_path): :rtype: str """ - # Strip off /files/ - try: - if vhost_path.startswith("/files/"): - avail_fp = vhost_path[7:].split("/") - else: - return None - except AttributeError: - # If we received a None path + if vhost_path is None or not vhost_path.startswith("/files/"): return None - last_good = "" - # Loop through the path parts and validate after every addition - for p in avail_fp: - cur_path = last_good+"/"+p - if os.path.exists(cur_path): - last_good = cur_path - else: - break - return last_good + return _split_aug_path(vhost_path)[0] + + +def get_internal_aug_path(vhost_path): + """Get the Augeas path for a vhost with the file path removed. + + :param str vhost_path: Augeas virtual host path + + :returns: Augeas path to vhost relative to the containing file + :rtype: str + + """ + return _split_aug_path(vhost_path)[1] + + +def _split_aug_path(vhost_path): + """Splits an Augeas path into a file path and an internal path. + + After removing "/files", this function splits vhost_path into the + file path and the remaining Augeas path. + + :param str vhost_path: Augeas virtual host path + + :returns: file path and internal Augeas path + :rtype: `tuple` of `str` + + """ + # Strip off /files + file_path = vhost_path[6:] + internal_path = [] + + # Remove components from the end of file_path until it becomes valid + while not os.path.exists(file_path): + file_path, _, internal_path_part = file_path.rpartition("/") + internal_path.append(internal_path_part) + + return file_path, "/".join(reversed(internal_path)) def install_ssl_options_conf(options_ssl): diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 3dc3790a1..9ac03f1f4 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -138,6 +138,17 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(get_file_path("nonexistent"), None) self.assertEqual(self.config._create_vhost("nonexistent"), None) # pylint: disable=protected-access + def test_get_aug_internal_path(self): + from certbot_apache.configurator import get_internal_aug_path + internal_paths = [ + "VirtualHost", "IfModule/VirtualHost", "VirtualHost", "VirtualHost", + "Macro/VirtualHost", "IfModule/VirtualHost", "VirtualHost", + "IfModule/VirtualHost"] + + for i, internal_path in enumerate(internal_paths): + self.assertEqual( + get_internal_aug_path(self.vh_truth[i].path), internal_path) + def test_bad_servername_alias(self): ssl_vh1 = obj.VirtualHost( "fp1", "ap1", set([obj.Addr(("*", "443"))]), @@ -1399,6 +1410,5 @@ class MultiVhostsTest(util.ApacheTest): self.assertEqual(self.config._get_http_vhost(ssl_vhost), http_vhost) - if __name__ == "__main__": unittest.main() # pragma: no cover From 65c7a5a6f79cb1703faa3548367816a1f1c0e099 Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Wed, 28 Sep 2016 21:34:27 +0300 Subject: [PATCH 48/96] Add support for multivhosts in Apache * Case sensitivity fixes * Clean up merge leftovers * Get correct vhost paths when appending to already existing multivhost -le-ssl.conf * Test, lint and reverter fixes * Make py26 happy * Removed skeletons * Changed new vhost matching * Added span flag for augeas init * Extract VirtualHost using aug_span * Removed dead code * Fix tests to mitigate not being able to reload Augeas span values after write * Small fixes and test coverage * Implementing changes requested in review --- .../certbot_apache/augeas_configurator.py | 4 +- certbot-apache/certbot_apache/configurator.py | 311 ++++++++++-------- certbot-apache/certbot_apache/obj.py | 1 + .../certbot_apache/tests/configurator_test.py | 186 +++++------ .../apache2/sites-available/multi-vhost.conf | 38 +++ .../apache2/sites-enabled/default.conf | 1 + .../apache2/sites-enabled/multi-vhost.conf | 1 + .../apache2/sites-enabled/placeholder.conf | 0 certbot-apache/certbot_apache/tests/util.py | 19 +- 9 files changed, 318 insertions(+), 243 deletions(-) create mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/multi-vhost.conf create mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/default.conf create mode 120000 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/multi-vhost.conf delete mode 100644 certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/placeholder.conf diff --git a/certbot-apache/certbot_apache/augeas_configurator.py b/certbot-apache/certbot_apache/augeas_configurator.py index 3735284ef..e053d0468 100644 --- a/certbot-apache/certbot_apache/augeas_configurator.py +++ b/certbot-apache/certbot_apache/augeas_configurator.py @@ -47,7 +47,9 @@ class AugeasConfigurator(common.Plugin): loadpath=constants.AUGEAS_LENS_DIR, # Do not save backup (we do it ourselves), do not load # anything by default - flags=(augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)) + flags=(augeas.Augeas.NONE | + augeas.Augeas.NO_MODL_AUTOLOAD | + augeas.Augeas.ENABLE_SPAN)) self.recovery_routine() def check_parsing_errors(self, lens): diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 3aaafca4b..787564141 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -138,7 +138,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self._enhance_func = {"redirect": self._enable_redirect, "ensure-http-header": self._set_http_header, "staple-ocsp": self._enable_ocsp_stapling} - self._skeletons = {} @property def mod_ssl_conf(self): @@ -495,6 +494,32 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return "" + def _get_vhost_names(self, path): + """Helper method for getting the ServerName and + ServerAlias values from vhost in path + + :param path: Path to read ServerName and ServerAliases from + + :returns: Tuple including ServerName and `list` of ServerAlias strings + """ + + servername_match = self.parser.find_dir( + "ServerName", None, start=path, exclude=False) + serveralias_match = self.parser.find_dir( + "ServerAlias", None, start=path, exclude=False) + + serveraliases = [] + for alias in serveralias_match: + serveralias = self.parser.get_arg(alias) + serveraliases.append(serveralias) + + servername = None + if servername_match: + # Get last ServerName as each overwrites the previous + servername = self.parser.get_arg(servername_match[-1]) + + return (servername, serveraliases) + def _add_servernames(self, host): """Helper function for get_virtual_hosts(). @@ -502,22 +527,15 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :type host: :class:`~certbot_apache.obj.VirtualHost` """ - # Take the final ServerName as each overrides the previous - servername_match = self.parser.find_dir( - "ServerName", None, start=host.path, exclude=False) - serveralias_match = self.parser.find_dir( - "ServerAlias", None, start=host.path, exclude=False) - for alias in serveralias_match: - serveralias = self.parser.get_arg(alias) - if not host.modmacro: - host.aliases.add(serveralias) + servername, serveraliases = self._get_vhost_names(host.path) - if servername_match: - # Get last ServerName as each overwrites the previous - servername = self.parser.get_arg(servername_match[-1]) + for alias in serveraliases: if not host.modmacro: - host.name = servername + host.aliases.add(alias) + + if not host.modmacro: + host.name = servername def _create_vhost(self, path): """Used by get_virtual_hosts to create vhost objects @@ -582,7 +600,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ("/files%s//*[label()=~regexp('%s')]" % (vhost_path, parser.case_i("VirtualHost")))) paths = [path for path in paths if - os.path.basename(path.lower()) == "virtualhost"] + "virtualhost" in os.path.basename(path).lower()] for path in paths: new_vhost = self._create_vhost(path) if not new_vhost: @@ -809,23 +827,27 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): avail_fp = nonssl_vhost.filep ssl_fp = self._get_ssl_vhost_path(avail_fp) - vhost_num = -1 - if nonssl_vhost.path.endswith("]"): - # augeas doesn't zero index for whatever reason - vhost_num = int(nonssl_vhost.path[-2]) - 1 - self._copy_create_ssl_vhost_skeleton(avail_fp, ssl_fp, vhost_num) + orig_matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" % + (self._escape(ssl_fp), + parser.case_i("VirtualHost"))) + + self._copy_create_ssl_vhost_skeleton(nonssl_vhost, ssl_fp) # Reload augeas to take into account the new vhost self.aug.load() # Get Vhost augeas path for new vhost - matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" % - (self._escape(ssl_fp), parser.case_i("VirtualHost"))) - skels = self._skeletons[ssl_fp] - vh_p = matches[len(skels) - 1] if skels else matches[0] + new_matches = self.aug.match("/files%s//* [label()=~regexp('%s')]" % + (self._escape(ssl_fp), + parser.case_i("VirtualHost"))) + + vh_p = self._get_new_vh_path(orig_matches, new_matches) + + if not vh_p: + raise errors.PluginError( + "Could not reverse map the HTTPS VirtualHost to the original") # Update Addresses self._update_ssl_vhosts_addrs(vh_p) - # Add directives self._add_dummy_ssl_directives(vh_p) self.save() @@ -852,6 +874,20 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return ssl_vhost + def _get_new_vh_path(self, orig_matches, new_matches): + """ Helper method for make_vhost_ssl for matching augeas paths. Returns + VirtualHost path from new_matches that's not present in orig_matches. + + Paths are normalized, because augeas leaves indices out for paths + with only single directive with a similar key """ + + orig_matches = [i.replace("[1]", "") for i in orig_matches] + for match in new_matches: + if match.replace("[1]", "") not in orig_matches: + # Return the unmodified path + return match + return None + def _get_ssl_vhost_path(self, non_ssl_vh_fp): # Get filepath of new ssl_vhost if non_ssl_vh_fp.endswith(".conf"): @@ -892,43 +928,10 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # Sift line if it redirects the request to a HTTPS site return target.startswith("https://") - def _section_blocks(self, blocks): - """A helper function for _create_block_segments that makes - a list of line numbers to not include in the return. - - :param list blocks: A list of indexes of where vhosts start and end. - - For instance, turns [2,5,13,15] into [[2,3,4,5], [13,14,15]]. - - """ - out = [] - while len(blocks) > 1: - start = blocks[0] - end = blocks[1] + 1 - out += range(start, end) - blocks = blocks[2:] - return out - - def _create_block_segments(self, orig_file_list, vhost_num): - """A helper function for _copy_create_ssl_vhost_skeleton - that slices the appropriate vhost from the origin conf file. - - :param list orig_file_list: the original file converted to a list of strings. - "param int vhost_num: Which vhost the vhost is in the origin multivhost file. - - """ - blocks = [idx for idx, line in enumerate(orig_file_list) - if line.lstrip().startswith(" skeleton. - :param str avail_fp: Pointer to the original available non-ssl vhost + :param obj.VirtualHost vhost: Original VirtualHost object :param str ssl_fp: Full path where the new ssl_vhost will reside. A new file is created on the filesystem. @@ -936,81 +939,24 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # First register the creation so that it is properly removed if # configuration is rolled back - self.reverter.register_file_creation(False, ssl_fp) + if os.path.exists(ssl_fp): + notes = "Appended new VirtualHost directive to file %s" % ssl_fp + files = set() + files.add(ssl_fp) + self.reverter.add_to_checkpoint(files, notes) + else: + self.reverter.register_file_creation(False, ssl_fp) sift = False try: - with open(avail_fp, "r") as orig_file: - orig_file_list = [line for line in orig_file] - if vhost_num != -1: - orig_file_list = self._create_block_segments(orig_file_list, vhost_num) + orig_contents = self._get_vhost_block(vhost) + ssl_vh_contents, sift = self._sift_rewrite_rules(orig_contents) - if ssl_fp in self._skeletons: - bit = "a" - self._skeletons[ssl_fp].append(avail_fp) - else: - bit = "w" - self._skeletons[ssl_fp] = [avail_fp] - - with open(ssl_fp, bit) as new_file: + with open(ssl_fp, "a") as new_file: new_file.write("\n") - - comment = ("# Some rewrite rules in this file were " - "disabled on your HTTPS site,\n" - "# because they have the potential to create " - "redirection loops.\n") - - orig_file_list = iter(orig_file_list) - for line in orig_file_list: - A = line.lower().lstrip().startswith("rewritecond") - B = line.lower().lstrip().startswith("rewriterule") - - if not (A or B): - new_file.write(line) - continue - - # A RewriteRule that doesn't need filtering - if B and not self._sift_rewrite_rule(line): - new_file.write(line) - continue - - # A RewriteRule that does need filtering - if B and self._sift_rewrite_rule(line): - if not sift: - new_file.write(comment) - sift = True - new_file.write("# " + line) - continue - - # We save RewriteCond(s) and their corresponding - # RewriteRule in 'chunk'. - # We then decide whether we comment out the entire - # chunk based on its RewriteRule. - chunk = [] - if A: - chunk.append(line) - line = next(orig_file_list) - - # RewriteCond(s) must be followed by one RewriteRule - while not line.lower().lstrip().startswith("rewriterule"): - chunk.append(line) - line = next(orig_file_list) - - # Now, current line must start with a RewriteRule - chunk.append(line) - - if self._sift_rewrite_rule(line): - if not sift: - new_file.write(comment) - sift = True - - new_file.write(''.join( - ['# ' + l for l in chunk])) - continue - else: - new_file.write(''.join(chunk)) - continue - + new_file.write("\n".join(ssl_vh_contents)) + # The content does not include the closing tag, so add it + new_file.write("\n") new_file.write("\n") except IOError: logger.fatal("Error writing/reading to file in make_vhost_ssl") @@ -1021,11 +967,96 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): reporter.add_message( "Some rewrite rules copied from {0} were disabled in the " "vhost for your HTTPS site located at {1} because they have " - "the potential to create redirection loops.".format(avail_fp, - ssl_fp), - reporter.MEDIUM_PRIORITY) - self.aug.set("/augeas/files%s/mtime" %(self._escape(ssl_fp)), "0") - self.aug.set("/augeas/files%s/mtime" %(self._escape(avail_fp)), "0") + "the potential to create redirection loops.".format( + vhost.filep, ssl_fp), reporter.MEDIUM_PRIORITY) + self.aug.set("/augeas/files%s/mtime" % (self._escape(ssl_fp)), "0") + self.aug.set("/augeas/files%s/mtime" % (self._escape(vhost.filep)), "0") + + def _sift_rewrite_rules(self, contents): + """ Helper function for _copy_create_ssl_vhost_skeleton to prepare the + new HTTPS VirtualHost contents. Currently disabling the rewrites """ + + result = [] + sift = False + contents = iter(contents) + + comment = ("# Some rewrite rules in this file were " + "disabled on your HTTPS site,\n" + "# because they have the potential to create " + "redirection loops.\n") + + for line in contents: + A = line.lower().lstrip().startswith("rewritecond") + B = line.lower().lstrip().startswith("rewriterule") + + if not (A or B): + result.append(line) + continue + + # A RewriteRule that doesn't need filtering + if B and not self._sift_rewrite_rule(line): + result.append(line) + continue + + # A RewriteRule that does need filtering + if B and self._sift_rewrite_rule(line): + if not sift: + result.append(comment) + sift = True + result.append("# " + line) + continue + + # We save RewriteCond(s) and their corresponding + # RewriteRule in 'chunk'. + # We then decide whether we comment out the entire + # chunk based on its RewriteRule. + chunk = [] + if A: + chunk.append(line) + line = next(contents) + + # RewriteCond(s) must be followed by one RewriteRule + while not line.lower().lstrip().startswith("rewriterule"): + chunk.append(line) + line = next(contents) + + # Now, current line must start with a RewriteRule + chunk.append(line) + + if self._sift_rewrite_rule(line): + if not sift: + result.append(comment) + sift = True + + result.append('\n'.join( + ['# ' + l for l in chunk])) + continue + else: + result.append('\n'.join(chunk)) + continue + return result, sift + + def _get_vhost_block(self, vhost): + """ Helper method to get VirtualHost contents from the original file. + This is done with help of augeas span, which returns the span start and + end positions + + :returns: `list` of VirtualHost block content lines without closing tag + """ + + try: + span_val = self.aug.span(vhost.path) + except ValueError: + logger.fatal("Error while reading the VirtualHost %s from " + "file %s", vhost.name, vhost.filep, exc_info=True) + raise errors.PluginError("Unable to read VirtualHost from file") + span_filep = span_val[0] + span_start = span_val[5] + span_end = span_val[6] + with open(span_filep, 'r') as fh: + fh.seek(span_start) + vh_contents = fh.read(span_end-span_start) + return vh_contents.split("\n") def _update_ssl_vhosts_addrs(self, vh_path): ssl_addrs = set() @@ -1073,10 +1104,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def _add_servername_alias(self, target_name, vhost): vh_path = vhost.path - if (self.parser.find_dir("ServerName", target_name, - start=vh_path, exclude=False) or - self.parser.find_dir("ServerAlias", target_name, - start=vh_path, exclude=False)): + sname, saliases = self._get_vhost_names(vh_path) + if target_name == sname or target_name in saliases: return if self._has_matching_wildcard(vh_path, target_name): return @@ -1476,7 +1505,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for re_path in rewrite_engine_path_list: # A RewriteEngine directive may also be included in per # directory .htaccess files. We only care about the VirtualHost. - if 'VirtualHost' in re_path: + if 'virtualhost' in re_path.lower(): return self.parser.get_arg(re_path) return False @@ -1880,7 +1909,7 @@ def get_file_path(vhost_path): :rtype: str """ - if vhost_path is None or not vhost_path.startswith("/files/"): + if not vhost_path or not vhost_path.startswith("/files/"): return None return _split_aug_path(vhost_path)[0] diff --git a/certbot-apache/certbot_apache/obj.py b/certbot-apache/certbot_apache/obj.py index fd743fe79..1e3579858 100644 --- a/certbot-apache/certbot_apache/obj.py +++ b/certbot-apache/certbot_apache/obj.py @@ -112,6 +112,7 @@ class VirtualHost(object): # pylint: disable=too-few-public-methods :ivar bool ssl: SSLEngine on in vhost :ivar bool enabled: Virtual host is enabled :ivar bool modmacro: VirtualHost is using mod_macro + :ivar VirtualHost ancestor: A non-SSL VirtualHost this is based on https://httpd.apache.org/docs/2.4/vhosts/details.html diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index 9ac03f1f4..c930b97fd 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -580,6 +580,7 @@ class MultipleVhostsTest(util.ApacheTest): # already listens to the correct port self.assertEqual(mock_add_dir.call_count, 0) + def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) @@ -800,6 +801,15 @@ class MultipleVhostsTest(util.ApacheTest): def test_supported_enhancements(self): self.assertTrue(isinstance(self.config.supported_enhancements(), list)) + def test_find_http_vhost_without_ancestor(self): + # pylint: disable=protected-access + vhost = self.vh_truth[0] + vhost.ssl = True + vhost.ancestor = None + res = self.config._get_http_vhost(vhost) + self.assertEqual(self.vh_truth[0].name, res.name) + self.assertEqual(self.vh_truth[0].aliases, res.aliases) + @mock.patch("certbot_apache.configurator.ApacheConfigurator._get_http_vhost") @mock.patch("certbot_apache.display_ops.select_vhost") @mock.patch("certbot.util.exe_exists") @@ -999,8 +1009,9 @@ class MultipleVhostsTest(util.ApacheTest): # three args to rw_rule self.assertEqual(len(rw_rule), 3) - self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path)) - self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path)) + # [:-3] to remove the vhost index number + self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path[:-3])) + self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path[:-3])) self.assertTrue("rewrite_module" in self.config.parser.modules) @@ -1048,9 +1059,9 @@ class MultipleVhostsTest(util.ApacheTest): self.assertEqual(len(rw_engine), 1) # three args to rw_rule + 1 arg for the pre existing rewrite self.assertEqual(len(rw_rule), 5) - - self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path)) - self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path)) + # [:-3] to remove the vhost index number + self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path[:-3])) + self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path[:-3])) self.assertTrue("rewrite_module" in self.config.parser.modules) @@ -1158,89 +1169,6 @@ class MultipleVhostsTest(util.ApacheTest): not_rewriterule = "NotRewriteRule ^ ..." self.assertFalse(self.config._sift_rewrite_rule(not_rewriterule)) - @certbot_util.patch_get_utility() - def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") - - http_vhost = self.vh_truth[0] - - self.config.parser.add_dir( - http_vhost.path, "RewriteEngine", "on") - - self.config.parser.add_dir( - http_vhost.path, "RewriteRule", - ["^", - "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,NE,R=permanent]"]) - self.config.save() - - ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - - self.assertTrue(self.config.parser.find_dir( - "RewriteEngine", "on", ssl_vhost.path, False)) - - conf_text = open(ssl_vhost.filep).read() - commented_rewrite_rule = ("# RewriteRule ^ " - "https://%{SERVER_NAME}%{REQUEST_URI} " - "[L,NE,R=permanent]") - self.assertTrue(commented_rewrite_rule in conf_text) - mock_get_utility().add_message.assert_called_once_with(mock.ANY, - - mock.ANY) - @certbot_util.patch_get_utility() - def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility): - self.config.parser.modules.add("rewrite_module") - - http_vhost = self.vh_truth[0] - - self.config.parser.add_dir( - http_vhost.path, "RewriteEngine", "on") - - # Add a chunk that should not be commented out. - self.config.parser.add_dir(http_vhost.path, - "RewriteCond", ["%{DOCUMENT_ROOT}/%{REQUEST_FILENAME}", "!-f"]) - self.config.parser.add_dir( - http_vhost.path, "RewriteRule", - ["^(.*)$", "b://u%{REQUEST_URI}", "[P,NE,L]"]) - - # Add a chunk that should be commented out. - self.config.parser.add_dir(http_vhost.path, - "RewriteCond", ["%{HTTPS}", "!=on"]) - self.config.parser.add_dir(http_vhost.path, - "RewriteCond", ["%{HTTPS}", "!^$"]) - self.config.parser.add_dir( - http_vhost.path, "RewriteRule", - ["^", - "https://%{SERVER_NAME}%{REQUEST_URI}", - "[L,NE,R=permanent]"]) - - self.config.save() - - ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) - - conf_line_set = set(open(ssl_vhost.filep).read().splitlines()) - - not_commented_cond1 = ("RewriteCond " - "%{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f") - not_commented_rewrite_rule = ("RewriteRule " - "^(.*)$ b://u%{REQUEST_URI} [P,NE,L]") - - commented_cond1 = "# RewriteCond %{HTTPS} !=on" - commented_cond2 = "# RewriteCond %{HTTPS} !^$" - commented_rewrite_rule = ("# RewriteRule ^ " - "https://%{SERVER_NAME}%{REQUEST_URI} " - "[L,NE,R=permanent]") - - self.assertTrue(not_commented_cond1 in conf_line_set) - self.assertTrue(not_commented_rewrite_rule in conf_line_set) - - self.assertTrue(commented_cond1 in conf_line_set) - self.assertTrue(commented_cond2 in conf_line_set) - self.assertTrue(commented_rewrite_rule in conf_line_set) - mock_get_utility().add_message.assert_called_once_with(mock.ANY, - mock.ANY) - - def get_achalls(self): """Return testing achallenges.""" account_key = self.rsa512jwk @@ -1351,6 +1279,12 @@ class AugeasVhostsTest(util.ApacheTest): self.config.choose_vhost(name) self.assertEqual(mock_select.call_count, 0) + def test_augeas_span_error(self): + broken_vhost = self.config.vhosts[0] + broken_vhost.path = broken_vhost.path + "/nonexistent" + self.assertRaises(errors.PluginError, self.config.make_vhost_ssl, + broken_vhost) + class MultiVhostsTest(util.ApacheTest): """Test vhosts with illegal names dependant on augeas version.""" # pylint: disable=protected-access @@ -1397,17 +1331,73 @@ class MultiVhostsTest(util.ApacheTest): self.assertEqual(self.config.is_name_vhost(self.vh_truth[1]), self.config.is_name_vhost(ssl_vhost)) - def test_make_2nd_vhost_ssl(self): - _ = self.config.make_vhost_ssl(self.vh_truth[0]) - _ = self.config.make_vhost_ssl(self.vh_truth[1]) - self.assertEqual( - len(self.config._skeletons[self.config._get_ssl_vhost_path(self.vh_truth[0].filep)]), 2) + mock_path = "certbot_apache.configurator.ApacheConfigurator._get_new_vh_path" + with mock.patch(mock_path) as mock_getpath: + mock_getpath.return_value = None + self.assertRaises(errors.PluginError, self.config.make_vhost_ssl, + self.vh_truth[1]) - def test_cover_is_stupid_and_I_hate_it(self): - http_vhost = obj.VirtualHost(None, None, None, False, False, name="Noah") - ssl_vhost = obj.VirtualHost(None, None, None, False, False, name="Noah") - self.config.vhosts.append(http_vhost) - self.assertEqual(self.config._get_http_vhost(ssl_vhost), http_vhost) + def test_get_new_path(self): + with_index_1 = ["/path[1]/section[1]"] + without_index = ["/path/section"] + with_index_2 = ["/path[2]/section[2]"] + self.assertEqual(self.config._get_new_vh_path(without_index, + with_index_1), + None) + self.assertEqual(self.config._get_new_vh_path(without_index, + with_index_2), + with_index_2[0]) + + both = with_index_1 + with_index_2 + self.assertEqual(self.config._get_new_vh_path(without_index, both), + with_index_2[0]) + + @certbot_util.patch_get_utility() + def test_make_vhost_ssl_with_existing_rewrite_rule(self, mock_get_utility): + self.config.parser.modules.add("rewrite_module") + + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[4]) + + self.assertTrue(self.config.parser.find_dir( + "RewriteEngine", "on", ssl_vhost.path, False)) + + conf_text = open(ssl_vhost.filep).read() + commented_rewrite_rule = ("# RewriteRule \"^/secrets/(.+)\" " + "\"https://new.example.com/docs/$1\" [R,L]") + uncommented_rewrite_rule = ("RewriteRule \"^/docs/(.+)\" " + "\"http://new.example.com/docs/$1\" [R,L]") + self.assertTrue(commented_rewrite_rule in conf_text) + self.assertTrue(uncommented_rewrite_rule in conf_text) + mock_get_utility().add_message.assert_called_once_with(mock.ANY, + mock.ANY) + + @certbot_util.patch_get_utility() + def test_make_vhost_ssl_with_existing_rewrite_conds(self, mock_get_utility): + self.config.parser.modules.add("rewrite_module") + + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[3]) + + conf_lines = open(ssl_vhost.filep).readlines() + conf_line_set = [l.strip() for l in conf_lines] + not_commented_cond1 = ("RewriteCond " + "%{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f") + not_commented_rewrite_rule = ("RewriteRule " + "^(.*)$ b://u%{REQUEST_URI} [P,NE,L]") + + commented_cond1 = "# RewriteCond %{HTTPS} !=on" + commented_cond2 = "# RewriteCond %{HTTPS} !^$" + commented_rewrite_rule = ("# RewriteRule ^ " + "https://%{SERVER_NAME}%{REQUEST_URI} " + "[L,NE,R=permanent]") + + self.assertTrue(not_commented_cond1 in conf_line_set) + self.assertTrue(not_commented_rewrite_rule in conf_line_set) + + self.assertTrue(commented_cond1 in conf_line_set) + self.assertTrue(commented_cond2 in conf_line_set) + self.assertTrue(commented_rewrite_rule in conf_line_set) + mock_get_utility().add_message.assert_called_once_with(mock.ANY, + mock.ANY) if __name__ == "__main__": diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/multi-vhost.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/multi-vhost.conf new file mode 100644 index 000000000..5f2b727bf --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-available/multi-vhost.conf @@ -0,0 +1,38 @@ + + ServerName 1.multi.vhost.tld + ServerAlias first.multi.vhost.tld + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined + + + + + ServerName 2.multi.vhost.tld + ServerAlias second.multi.vhost.tld + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined +RewriteEngine on +RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_FILENAME} !-f +RewriteRule ^(.*)$ b://u%{REQUEST_URI} [P,NE,L] +RewriteCond %{HTTPS} !=on +RewriteCond %{HTTPS} !^$ +RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [L,NE,R=permanent] + + + + + ServerName 3.multi.vhost.tld + ServerAlias third.multi.vhost.tld + ServerAdmin webmaster@localhost + DocumentRoot /var/www/html + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined +RewriteEngine on +RewriteRule "^/secrets/(.+)" "https://new.example.com/docs/$1" [R,L] +RewriteRule "^/docs/(.+)" "http://new.example.com/docs/$1" [R,L] + + diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/default.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/default.conf new file mode 120000 index 000000000..032e6bcf0 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/default.conf @@ -0,0 +1 @@ +../sites-available/default.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/multi-vhost.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/multi-vhost.conf new file mode 120000 index 000000000..7f0910ff4 --- /dev/null +++ b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/multi-vhost.conf @@ -0,0 +1 @@ +../sites-available/multi-vhost.conf \ No newline at end of file diff --git a/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/placeholder.conf b/certbot-apache/certbot_apache/tests/testdata/debian_apache_2_4/multi_vhosts/apache2/sites-enabled/placeholder.conf deleted file mode 100644 index e69de29bb..000000000 diff --git a/certbot-apache/certbot_apache/tests/util.py b/certbot-apache/certbot_apache/tests/util.py index 90880ac5d..d40152ad5 100644 --- a/certbot-apache/certbot_apache/tests/util.py +++ b/certbot-apache/certbot_apache/tests/util.py @@ -178,8 +178,21 @@ def get_vh_truth(temp_dir, config_name): os.path.join(prefix, "default.conf"), os.path.join(aug_pre, "default.conf/VirtualHost[2]"), set([obj.Addr.fromstring("*:80")]), - False, True, "banana.vomit.com")] + False, True, "banana.vomit.com"), + obj.VirtualHost( + os.path.join(prefix, "multi-vhost.conf"), + os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[1]"), + set([obj.Addr.fromstring("*:80")]), + False, True, "1.multi.vhost.tld"), + obj.VirtualHost( + os.path.join(prefix, "multi-vhost.conf"), + os.path.join(aug_pre, "multi-vhost.conf/IfModule/VirtualHost"), + set([obj.Addr.fromstring("*:80")]), + False, True, "2.multi.vhost.tld"), + obj.VirtualHost( + os.path.join(prefix, "multi-vhost.conf"), + os.path.join(aug_pre, "multi-vhost.conf/VirtualHost[2]"), + set([obj.Addr.fromstring("*:80")]), + False, True, "3.multi.vhost.tld")] return vh_truth - - return None # pragma: no cover From 70168742435f50cba1e7f123147b7b395b166265 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 2 May 2017 17:56:56 -0700 Subject: [PATCH 49/96] Switch to using include directive for Nginx constants (#4557) * Switch to using include directive for Nginx constants * remove deprecated comment * give better error message when attempting to insert an existing directive * make code more readable * add docstrings * allow a duplicated directive if it's identical * comment out precisely repeated directives * add comments --- certbot-nginx/certbot_nginx/configurator.py | 19 ++- certbot-nginx/certbot_nginx/parser.py | 122 +++++++++++------- .../certbot_nginx/tests/configurator_test.py | 14 +- .../certbot_nginx/tests/parser_test.py | 45 ++----- .../certbot_nginx/tests/tls_sni_01_test.py | 8 +- certbot-nginx/certbot_nginx/tls_sni_01.py | 6 +- 6 files changed, 108 insertions(+), 106 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index 8608fb66a..dd61575d4 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -153,10 +153,10 @@ class NginxConfigurator(common.Plugin): # Make sure configuration is valid self.config_test() - # temp_install must be run before creating the NginxParser - temp_install(self.mod_ssl_conf) - self.parser = parser.NginxParser( - self.conf('server-root'), self.mod_ssl_conf) + + self.parser = parser.NginxParser(self.conf('server-root')) + + install_ssl_options_conf(self.mod_ssl_conf) # Set Version if self.version is None: @@ -452,14 +452,11 @@ class NginxConfigurator(common.Plugin): snakeoil_cert, snakeoil_key = self._get_snakeoil_paths() - # the options file doesn't have a newline at the beginning, but there - # needs to be one when it's dropped into the file ssl_block = ( [['\n ', 'listen', ' ', '{0} ssl'.format(self.config.tls_sni_01_port)], ['\n ', 'ssl_certificate', ' ', snakeoil_cert], ['\n ', 'ssl_certificate_key', ' ', snakeoil_key], - ['\n']] + - self.parser.loc["ssl_options"]) + ['\n ', 'include', ' ', self.mod_ssl_conf]]) self.parser.add_server_directives( vhost, ssl_block, replace=False) @@ -677,7 +674,7 @@ class NginxConfigurator(common.Plugin): "Configures Nginx to authenticate and install HTTPS.{0}" "Server root: {root}{0}" "Version: {version}".format( - os.linesep, root=self.parser.loc["root"], + os.linesep, root=self.parser.config_root, version=".".join(str(i) for i in self.version)) ) @@ -865,8 +862,8 @@ def nginx_restart(nginx_ctl, nginx_conf): time.sleep(1) -def temp_install(options_ssl): - """Temporary install for convenience.""" +def install_ssl_options_conf(options_ssl): + """Copy Certbot's SSL options file into the system's config dir if required.""" # Check to make sure options-ssl.conf is installed if not os.path.isfile(options_ssl): shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl) diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 9f1a08b3b..558275b0d 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -24,10 +24,10 @@ class NginxParser(object): """ - def __init__(self, root, ssl_options): + def __init__(self, root): self.parsed = {} self.root = os.path.abspath(root) - self.loc = self._set_locations(ssl_options) + self.config_root = self._find_config_root() # Parse nginx.conf and included files. # TODO: Check sites-available/ as well. For now, the configurator does @@ -39,7 +39,7 @@ class NginxParser(object): """ self.parsed = {} - self._parse_recursively(self.loc["root"]) + self._parse_recursively(self.config_root) def _parse_recursively(self, filepath): """Parses nginx config files recursively by looking at 'include' @@ -209,40 +209,8 @@ class NginxParser(object): logger.debug("Could not parse file: %s due to %s", item, err) return trees - def _parse_ssl_options(self, ssl_options): - if ssl_options is not None: - try: - with open(ssl_options) as _file: - return nginxparser.load(_file).spaced - except IOError: - logger.warn("Missing NGINX TLS options file: %s", ssl_options) - except pyparsing.ParseBaseException as err: - logger.debug("Could not parse file: %s due to %s", ssl_options, err) - return [] - - def _set_locations(self, ssl_options): - """Set default location for directives. - - Locations are given as file_paths - .. todo:: Make sure that files are included - - """ - root = self._find_config_root() - default = root - - nginx_temp = os.path.join(self.root, "nginx_ports.conf") - if os.path.isfile(nginx_temp): - listen = nginx_temp - name = nginx_temp - else: - listen = default - name = default - - return {"root": root, "default": default, "listen": listen, - "name": name, "ssl_options": self._parse_ssl_options(ssl_options)} - def _find_config_root(self): - """Find the Nginx Configuration Root file.""" + """Return the Nginx Configuration Root file.""" location = ['nginx.conf'] for name in location: @@ -344,6 +312,16 @@ class NginxParser(object): except errors.MisconfigurationError as err: raise errors.MisconfigurationError("Problem in %s: %s" % (filename, str(err))) +def _parse_ssl_options(ssl_options): + if ssl_options is not None: + try: + with open(ssl_options) as _file: + return nginxparser.load(_file) + except IOError: + logger.warn("Missing NGINX TLS options file: %s", ssl_options) + except pyparsing.ParseBaseException as err: + logger.debug("Could not parse file: %s due to %s", ssl_options, err) + return [] def _do_for_subarray(entry, condition, func, path=None): """Executes a function for a subarray of a nested array if it matches @@ -501,11 +479,11 @@ def _add_directives(block, directives, replace): block.append(nginxparser.UnspacedList('\n')) -REPEATABLE_DIRECTIVES = set(['server_name', 'listen', 'include']) +INCLUDE = 'include' +REPEATABLE_DIRECTIVES = set(['server_name', 'listen', INCLUDE]) COMMENT = ' managed by Certbot' COMMENT_BLOCK = [' ', '#', COMMENT] - def _comment_directive(block, location): """Add a comment to the end of the line at location.""" next_entry = block[location + 1] if location + 1 < len(block) else None @@ -521,6 +499,28 @@ def _comment_directive(block, location): if next_entry is not None and "\n" not in next_entry: block.insert(location + 2, '\n') +def _comment_out_directive(block, location, include_location): + """Comment out the line at location, with a note of explanation.""" + comment_message = ' duplicated in {0}'.format(include_location) + # add the end comment + # create a dumpable object out of block[location] (so it includes the ;) + directive = block[location] + new_dir_block = nginxparser.UnspacedList([]) # just a wrapper + new_dir_block.append(directive) + dumped = nginxparser.dumps(new_dir_block) + commented = dumped + ' #' + comment_message # add the comment directly to the one-line string + new_dir = nginxparser.loads(commented) # reload into UnspacedList + + # add the beginning comment + insert_location = 0 + if new_dir[0].spaced[0] != new_dir[0][0]: # if there's whitespace at the beginning + insert_location = 1 + new_dir[0].spaced.insert(insert_location, "# ") # comment out the line + new_dir[0].spaced.append(";") # directly add in the ;, because now dumping won't work properly + dumped = nginxparser.dumps(new_dir) + new_dir = nginxparser.loads(dumped) # reload into an UnspacedList + + block[location] = new_dir[0] # set the now-single-line-comment directive back in place def _add_directive(block, directive, replace): """Adds or replaces a single directive in a config block. @@ -534,10 +534,15 @@ def _add_directive(block, directive, replace): block.append(directive) return - # Find the index of a config line where the name of the directive matches - # the name of the directive we want to add. If no line exists, use None. - location = next((index for index, line in enumerate(block) - if line and line[0] == directive[0]), None) + def find_location(direc): + """ Find the index of a config line where the name of the directive matches + the name of the directive we want to add. If no line exists, use None. + """ + return next((index for index, line in enumerate(block) \ + if line and line[0] == direc[0]), None) + + location = find_location(directive) + if replace: if location is None: raise errors.MisconfigurationError( @@ -549,15 +554,38 @@ def _add_directive(block, directive, replace): # Append directive. Fail if the name is not a repeatable directive name, # and there is already a copy of that directive with a different value # in the config file. + + # handle flat include files + directive_name = directive[0] - if location is None or (isinstance(directive_name, str) and - directive_name in REPEATABLE_DIRECTIVES): + def can_append(loc, dir_name): + """ Can we append this directive to the block? """ + return loc is None or (isinstance(dir_name, str) and dir_name in REPEATABLE_DIRECTIVES) + + err_fmt = 'tried to insert directive "{0}" but found conflicting "{1}".' + + # Give a better error message about the specific directive than Nginx's "fail to restart" + if directive_name == INCLUDE: + # in theory, we might want to do this recursively, but in practice, that's really not + # necessary because we know what file we're talking about (and if we don't recurse, we + # just give a worse error message) + included_directives = _parse_ssl_options(directive[1]) + + for included_directive in included_directives: + included_dir_loc = find_location(included_directive) + included_dir_name = included_directive[0] + if not can_append(included_dir_loc, included_dir_name): + if block[included_dir_loc] != included_directive: + raise errors.MisconfigurationError(err_fmt.format(included_directive, + block[included_dir_loc])) + else: + _comment_out_directive(block, included_dir_loc, directive[1]) + + if can_append(location, directive_name): block.append(directive) _comment_directive(block, len(block) - 1) elif block[location] != directive: - raise errors.MisconfigurationError( - 'tried to insert directive "{0}" but found ' - 'conflicting "{1}".'.format(directive, block[location])) + raise errors.MisconfigurationError(err_fmt.format(directive, block[location])) def _apply_global_addr_ssl(addr_to_ssl, parsed_server): """Apply global sslishness information to the parsed server block diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 69f728b53..906d276b1 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -45,8 +45,6 @@ class NginxConfiguratorTest(util.NginxTest): def test_prepare(self): self.assertEqual((1, 6, 2), self.config.version) self.assertEqual(8, len(self.config.parser.parsed)) - # ensure we successfully parsed a file for ssl_options - self.assertTrue(self.config.parser.loc["ssl_options"]) @mock.patch("certbot_nginx.configurator.util.exe_exists") @mock.patch("certbot_nginx.configurator.subprocess.Popen") @@ -225,8 +223,8 @@ class NginxConfiguratorTest(util.NginxTest): ['listen', '5001', 'ssl'], ['ssl_certificate', 'example/fullchain.pem'], - ['ssl_certificate_key', 'example/key.pem']] + - util.filter_comments(self.config.parser.loc["ssl_options"]) + ['ssl_certificate_key', 'example/key.pem'], + ['include', self.config.mod_ssl_conf]] ]], parsed_example_conf) self.assertEqual([['server_name', 'somename', 'alias', 'another.alias']], @@ -243,8 +241,8 @@ class NginxConfiguratorTest(util.NginxTest): ['index', 'index.html', 'index.htm']]], ['listen', '5001', 'ssl'], ['ssl_certificate', '/etc/nginx/fullchain.pem'], - ['ssl_certificate_key', '/etc/nginx/key.pem']] + - util.filter_comments(self.config.parser.loc["ssl_options"]) + ['ssl_certificate_key', '/etc/nginx/key.pem'], + ['include', self.config.mod_ssl_conf]] ], 2)) @@ -267,8 +265,8 @@ class NginxConfiguratorTest(util.NginxTest): ['listen', '80'], ['listen', '5001', 'ssl'], ['ssl_certificate', 'summer/fullchain.pem'], - ['ssl_certificate_key', 'summer/key.pem']] + - util.filter_comments(self.config.parser.loc["ssl_options"]) + ['ssl_certificate_key', 'summer/key.pem'], + ['include', self.config.mod_ssl_conf]] ], parsed_migration_conf[0]) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 8a8bd0ff1..1ed9a8edf 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -27,22 +27,22 @@ class NginxParserTest(util.NginxTest): def test_root_normalized(self): path = os.path.join(self.temp_dir, "etc_nginx/////" "ubuntu_nginx/../../etc_nginx") - nparser = parser.NginxParser(path, None) + nparser = parser.NginxParser(path) self.assertEqual(nparser.root, self.config_path) def test_root_absolute(self): - nparser = parser.NginxParser(os.path.relpath(self.config_path), None) + nparser = parser.NginxParser(os.path.relpath(self.config_path)) self.assertEqual(nparser.root, self.config_path) def test_root_no_trailing_slash(self): - nparser = parser.NginxParser(self.config_path + os.path.sep, None) + nparser = parser.NginxParser(self.config_path + os.path.sep) self.assertEqual(nparser.root, self.config_path) def test_load(self): """Test recursive conf file parsing. """ - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) nparser.load() self.assertEqual(set([nparser.abs_path(x) for x in ['foo.conf', 'nginx.conf', 'server.conf', @@ -62,13 +62,13 @@ class NginxParserTest(util.NginxTest): 'sites-enabled/example.com')]) def test_abs_path(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) self.assertEqual('/etc/nginx/*', nparser.abs_path('/etc/nginx/*')) self.assertEqual(os.path.join(self.config_path, 'foo/bar/'), nparser.abs_path('foo/bar/')) def test_filedump(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) nparser.filedump('test', lazy=False) # pylint: disable=protected-access parsed = nparser._parse_files(nparser.abs_path( @@ -106,7 +106,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(paths, result) def test_get_vhosts_global_ssl(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) vhosts = nparser.get_vhosts() vhost = obj.VirtualHost(nparser.abs_path('sites-enabled/globalssl.com'), @@ -117,7 +117,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(vhost, globalssl_com) def test_get_vhosts(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) vhosts = nparser.get_vhosts() vhost1 = obj.VirtualHost(nparser.abs_path('nginx.conf'), @@ -161,7 +161,7 @@ class NginxParserTest(util.NginxTest): self.assertEqual(vhost2, somename) def test_has_ssl_on_directive(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(None, None, None, None, None, [['listen', 'myhost default_server'], ['server_name', 'www.example.org'], @@ -181,7 +181,7 @@ class NginxParserTest(util.NginxTest): self.assertTrue(nparser.has_ssl_on_directive(mock_vhost)) def test_add_server_directives(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) mock_vhost = obj.VirtualHost(nparser.abs_path('nginx.conf'), None, None, None, set(['localhost', @@ -231,7 +231,7 @@ class NginxParserTest(util.NginxTest): replace=False) def test_replace_server_directives(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) target = set(['.example.com', 'example.*']) filep = nparser.abs_path('sites-enabled/example.com') mock_vhost = obj.VirtualHost(filep, None, None, None, target, None, [0]) @@ -330,33 +330,12 @@ class NginxParserTest(util.NginxTest): self.assertEqual(len(server['addrs']), 0) def test_parse_server_global_ssl_applied(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) + nparser = parser.NginxParser(self.config_path) server = nparser.parse_server([ ['listen', '443'] ]) self.assertTrue(server['ssl']) - def test_ssl_options_should_be_parsed_ssl_directives(self): - nparser = parser.NginxParser(self.config_path, self.ssl_options) - self.assertEqual(nginxparser.UnspacedList(nparser.loc["ssl_options"]), - [['ssl_session_cache', 'shared:le_nginx_SSL:1m'], - ['ssl_session_timeout', '1440m'], - ['ssl_protocols', 'TLSv1', 'TLSv1.1', 'TLSv1.2'], - ['ssl_prefer_server_ciphers', 'on'], - ['ssl_ciphers', '"ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-'+ - 'RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:'+ - 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-'+ - 'SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-'+ - 'SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-'+ - 'SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:'+ - 'ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-'+ - 'AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:'+ - 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-'+ - 'SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-'+ - 'RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:'+ - 'AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:'+ - 'AES256-SHA:DES-CBC3-SHA:!DSS"'] - ]) if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py index 7a2de44a2..85db584b3 100644 --- a/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py +++ b/certbot-nginx/certbot_nginx/tests/tls_sni_01_test.py @@ -89,7 +89,7 @@ class TlsSniPerformTest(util.NginxTest): # Make sure challenge config is included in main config http = self.sni.configurator.parser.parsed[ - self.sni.configurator.parser.loc["root"]][-1] + self.sni.configurator.parser.config_root][-1] self.assertTrue( util.contains_at_depth(http, ['include', self.sni.challenge_conf], 1)) @@ -112,7 +112,7 @@ class TlsSniPerformTest(util.NginxTest): mock_setup_cert.call_args_list[index], mock.call(achall)) http = self.sni.configurator.parser.parsed[ - self.sni.configurator.parser.loc["root"]][-1] + self.sni.configurator.parser.config_root][-1] self.assertTrue(['include', self.sni.challenge_conf] in http[1]) self.assertFalse( util.contains_at_depth(http, ['server_name', 'another.alias'], 3)) @@ -137,7 +137,7 @@ class TlsSniPerformTest(util.NginxTest): self.sni.configurator.parser.load() http = self.sni.configurator.parser.parsed[ - self.sni.configurator.parser.loc["root"]][-1] + self.sni.configurator.parser.config_root][-1] self.assertTrue(['include', self.sni.challenge_conf] in http[1]) vhosts = self.sni.configurator.parser.get_vhosts() @@ -154,7 +154,7 @@ class TlsSniPerformTest(util.NginxTest): self.assertEqual(len(vhs), 2) def test_mod_config_fail(self): - root = self.sni.configurator.parser.loc["root"] + root = self.sni.configurator.parser.config_root self.sni.configurator.parser.parsed[root] = [['include', 'foo.conf']] # pylint: disable=protected-access self.assertRaises( diff --git a/certbot-nginx/certbot_nginx/tls_sni_01.py b/certbot-nginx/certbot_nginx/tls_sni_01.py index 347d9f21f..48e117bba 100644 --- a/certbot-nginx/certbot_nginx/tls_sni_01.py +++ b/certbot-nginx/certbot_nginx/tls_sni_01.py @@ -91,7 +91,7 @@ class NginxTlsSni01(common.TLSSNI01): # already in the main config included = False include_directive = ['\n', 'include', ' ', self.challenge_conf] - root = self.configurator.parser.loc["root"] + root = self.configurator.parser.config_root bucket_directive = ['\n', 'server_names_hash_bucket_size', ' ', '128'] @@ -157,6 +157,6 @@ class NginxTlsSni01(common.TLSSNI01): self.configurator.config.work_dir, 'error.log')], ['ssl_certificate', ' ', self.get_cert_path(achall)], ['ssl_certificate_key', ' ', self.get_key_path(achall)], - [['location', ' ', '/'], [['root', ' ', document_root]]]] + - self.configurator.parser.loc["ssl_options"]) + ['include', ' ', self.configurator.mod_ssl_conf], + [['location', ' ', '/'], [['root', ' ', document_root]]]]) return [['server'], block] From a5bd0cf50ca6c823b70e71cbe3e9c061d5ed1525 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 2 May 2017 18:37:54 -0700 Subject: [PATCH 50/96] Add a test for #4557 (#4609) --- .../certbot_nginx/tests/parser_test.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 1ed9a8edf..7b33b1075 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -302,6 +302,33 @@ class NginxParserTest(util.NginxTest): COMMENT_BLOCK, ["\n", "e", " ", "f"]]) + def test_comment_out_directive(self): + server_block = nginxparser.loads(""" + server { + listen 80; + root /var/www/html; + index star.html; + + server_name *.functorkitten.xyz; + ssl_session_timeout 1440m; ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + + ssl_prefer_server_ciphers on; + }""") + block = server_block[0][1] + from certbot_nginx.parser import _comment_out_directive + _comment_out_directive(block, 4, "blah1") + _comment_out_directive(block, 5, "blah2") + _comment_out_directive(block, 6, "blah3") + self.assertEqual(block.spaced, [ + ['\n ', 'listen', ' ', '80'], + ['\n ', 'root', ' ', '/var/www/html'], + ['\n ', 'index', ' ', 'star.html'], + ['\n\n ', 'server_name', ' ', '*.functorkitten.xyz'], + ['\n ', '#', ' ssl_session_timeout 1440m; # duplicated in blah1'], + [' ', '#', ' ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # duplicated in blah2'], + ['\n\n ', '#', ' ssl_prefer_server_ciphers on; # duplicated in blah3'], + '\n ']) + def test_parse_server_raw_ssl(self): server = parser._parse_server_raw([ #pylint: disable=protected-access ['listen', '443'] From 13c88f1c0288622053c1170b09c76fd091a5e6a7 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 May 2017 14:44:15 -0700 Subject: [PATCH 51/96] Properly handle EOF in input (#4612) * properly handle eof * cleanup InputWithTimeoutTest * add test_eof * add comment about mimicking getpass --- certbot/display/util.py | 7 ++++++- certbot/tests/display/util_test.py | 15 +++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/certbot/display/util.py b/certbot/display/util.py index 7e797e71a..5b01dd8d4 100644 --- a/certbot/display/util.py +++ b/certbot/display/util.py @@ -69,6 +69,8 @@ def input_with_timeout(prompt=None, timeout=36000.0): :raises errors.Error if no answer is given before the timeout """ + # use of sys.stdin and sys.stdout to mimic six.moves.input based on + # https://github.com/python/cpython/blob/baf7bb30a02aabde260143136bdf5b3738a1d409/Lib/getpass.py#L129 if prompt: sys.stdout.write(prompt) sys.stdout.flush() @@ -79,7 +81,10 @@ def input_with_timeout(prompt=None, timeout=36000.0): raise errors.Error( "Timed out waiting for answer to prompt '{0}'".format(prompt)) - return rlist[0].readline().rstrip('\n') + line = rlist[0].readline() + if not line: + raise EOFError + return line.rstrip('\n') @zope.interface.implementer(interfaces.IDisplay) diff --git a/certbot/tests/display/util_test.py b/certbot/tests/display/util_test.py index a872917ec..1dfc21c30 100644 --- a/certbot/tests/display/util_test.py +++ b/certbot/tests/display/util_test.py @@ -2,6 +2,7 @@ import inspect import os import socket +import tempfile import unittest import six @@ -25,15 +26,17 @@ class InputWithTimeoutTest(unittest.TestCase): from certbot.display.util import input_with_timeout return input_with_timeout(*args, **kwargs) - def setUp(self): - self.expected_msg = "foo bar" - self.stdin = six.StringIO(self.expected_msg + "\n") + def test_eof(self): + with tempfile.TemporaryFile("r+") as f: + with mock.patch("certbot.display.util.sys.stdin", new=f): + self.assertRaises(EOFError, self._call) def test_input(self, prompt=None): + expected = "foo bar" + stdin = six.StringIO(expected + "\n") with mock.patch("certbot.display.util.select.select") as mock_select: - mock_select.return_value = ([self.stdin], [], [],) - self.assertEqual(display_util.input_with_timeout(prompt), - self.expected_msg) + mock_select.return_value = ([stdin], [], [],) + self.assertEqual(self._call(prompt), expected) @mock.patch("certbot.display.util.sys.stdout") def test_input_with_prompt(self, mock_stdout): From 4d0cf8000ae14f6662e52fda14eb390474cab6e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 3 May 2017 18:42:47 -0700 Subject: [PATCH 52/96] make a copy of keys in all python versions (#4614) * make a copy of keys in all python versions * documentation++ --- certbot-apache/certbot_apache/configurator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 3dd9aae5f..2885cd0ef 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -603,7 +603,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): file_paths = {} internal_paths = defaultdict(set) vhs = [] - for vhost_path in self.parser.parser_paths.keys(): + # Make a list of parser paths because the parser_paths + # dictionary may be modified during the loop. + for vhost_path in list(self.parser.parser_paths): paths = self.aug.match( ("/files%s//*[label()=~regexp('%s')]" % (vhost_path, parser.case_i("VirtualHost")))) From 0db668f67be69ac5407c823fa15d017ea29cca71 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 May 2017 16:35:37 -0700 Subject: [PATCH 53/96] remove unnecessary closes causing logging problems (#4616) --- certbot/tests/main_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/certbot/tests/main_test.py b/certbot/tests/main_test.py index 23cff7edd..7c2016178 100644 --- a/certbot/tests/main_test.py +++ b/certbot/tests/main_test.py @@ -410,8 +410,6 @@ class MainTest(test_util.TempDirTestCase): # pylint: disable=too-many-public-me finally: output = toy_out.getvalue() or toy_err.getvalue() self.assertTrue("certbot" in output, "Output is {0}".format(output)) - toy_out.close() - toy_err.close() def _cli_missing_flag(self, args, message): "Ensure that a particular error raises a missing cli flag error containing message" From 4be7efbf745c34ab13ee38c6f2fd5f08345de531 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 4 May 2017 16:52:13 -0700 Subject: [PATCH 54/96] Release 0.14.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-auto | 183 ++++++++++-------- certbot-compatibility-test/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- docs/cli-help.txt | 125 ++++++------ letsencrypt-auto | 183 ++++++++++-------- letsencrypt-auto-source/certbot-auto.asc | 14 +- letsencrypt-auto-source/letsencrypt-auto | 26 +-- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/letsencrypt-auto-requirements.txt | 24 +-- 12 files changed, 298 insertions(+), 267 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index 8d8d1a049..e38cadf2c 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0.dev0' +version = '0.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 19f0c74f8..dcf604c55 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0.dev0' +version = '0.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-auto b/certbot-auto index fc8007c9e..0db142a78 100755 --- a/certbot-auto +++ b/certbot-auto @@ -15,6 +15,11 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed + +# HOME might not be defined when being run through something like systemd +if [ -z "$HOME" ]; then + HOME=~root +fi if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME=~/.local/share fi @@ -23,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.13.0" +LE_AUTO_VERSION="0.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -59,7 +64,7 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) + --noninteractive|--non-interactive|renew) ASSUME_YES=1;; --quiet) QUIET=1;; @@ -93,6 +98,16 @@ if [ "$QUIET" = 1 ]; then ASSUME_YES=1 fi +say() { + if [ "$QUIET" != 1 ]; then + echo "$@" + fi +} + +error() { + echo "$@" +} + # Support for busybox and others where there is no "command", # but "which" instead if command -v command > /dev/null 2>&1 ; then @@ -100,7 +115,7 @@ if command -v command > /dev/null 2>&1 ; then elif which which > /dev/null 2>&1 ; then export EXISTS="which" else - echo "Cannot find command nor which... please install one!" + error "Cannot find command nor which... please install one!" exit 1 fi @@ -145,17 +160,17 @@ if [ -n "${LE_AUTO_SUDO+x}" ]; then ;; '') ;; # Nothing to do for plain root method. *) - echo "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." + error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." exit 1 esac - echo "Using preset root authorization mechanism '$LE_AUTO_SUDO'." + say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." else if test "`id -u`" -ne "0" ; then if $EXISTS sudo 1>/dev/null 2>&1; then SUDO=sudo SUDO_ENV="CERTBOT_AUTO=$0" else - echo \"sudo\" is not available, will use \"su\" for installation steps... + say \"sudo\" is not available, will use \"su\" for installation steps... SUDO=su_sudo fi else @@ -165,7 +180,7 @@ fi BootstrapMessage() { # Arguments: Platform name - echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" + say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" } ExperimentalBootstrap() { @@ -176,11 +191,11 @@ ExperimentalBootstrap() { $2 fi else - echo "FATAL: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - echo "Alternatively, you can install OS dependencies yourself and run this script" - echo "again with --no-bootstrap." + error "FATAL: $1 support is very experimental at present..." + error "if you would like to work on improving it, please ensure you have backups" + error "and then run this script again with the --debug flag!" + error "Alternatively, you can install OS dependencies yourself and run this script" + error "again with --no-bootstrap." exit 1 fi } @@ -191,15 +206,15 @@ DeterminePythonVersion() { $EXISTS "$LE_PYTHON" > /dev/null && break done if [ "$?" != "0" ]; then - echo "Cannot find any Pythons; please install one!" + error "Cannot find any Pythons; please install one!" exit 1 fi export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt 26 ]; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version 2.6." exit 1 fi } @@ -227,7 +242,7 @@ BootstrapDebCommon() { QUIET_FLAG='-qq' fi - $SUDO apt-get $QUIET_FLAG update || echo apt-get update hit problems but continuing anyway... + $SUDO apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -255,7 +270,7 @@ BootstrapDebCommon() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" - echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then @@ -315,7 +330,7 @@ BootstrapDebCommon() { if ! $EXISTS virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting + error Failed to install a working \"virtualenv\" command, exiting exit 1 fi } @@ -335,7 +350,7 @@ BootstrapRpmCommon() { tool=yum else - echo "Neither yum nor dnf found. Aborting bootstrap!" + error "Neither yum nor dnf found. Aborting bootstrap!" exit 1 fi @@ -349,7 +364,7 @@ BootstrapRpmCommon() { if ! $SUDO $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." if ! $SUDO $tool list epel-release >/dev/null 2>&1; then - echo "Please enable this repository and try running Certbot again." + error "Enable the EPEL repository and try running Certbot again." exit 1 fi if [ "$ASSUME_YES" = 1 ]; then @@ -361,7 +376,7 @@ BootstrapRpmCommon() { sleep 1s fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG epel-release; then - echo "Could not enable EPEL. Aborting bootstrap!" + error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi @@ -403,7 +418,7 @@ BootstrapRpmCommon() { fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG $pkgs; then - echo "Could not install OS dependencies. Aborting bootstrap!" + error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi } @@ -508,15 +523,15 @@ BootstrapFreeBsd() { BootstrapMac() { if hash brew 2>/dev/null; then - echo "Using Homebrew to install dependencies..." + say "Using Homebrew to install dependencies..." pkgman=brew pkgcmd="brew install" elif hash port 2>/dev/null; then - echo "Using MacPorts to install dependencies..." + say "Using MacPorts to install dependencies..." pkgman=port pkgcmd="$SUDO port install" else - echo "No Homebrew/MacPorts; installing Homebrew..." + say "No Homebrew/MacPorts; installing Homebrew..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" pkgman=brew pkgcmd="brew install" @@ -527,26 +542,26 @@ BootstrapMac() { -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. - echo "Installing python..." + say "Installing python..." $pkgcmd python fi # Workaround for _dlopen not finding augeas on macOS if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - echo "Applying augeas workaround" + say "Applying augeas workaround" $SUDO mkdir -p /usr/local/lib/ $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then - echo "pip not installed" - echo "Installing pip..." + say "pip not installed" + say "Installing pip..." curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python fi if ! hash virtualenv 2>/dev/null; then - echo "virtualenv not installed." - echo "Installing with pip..." + say "virtualenv not installed." + say "Installing with pip..." pip install virtualenv fi } @@ -566,7 +581,7 @@ BootstrapMageiaCommon() { libpython-devel \ python-virtualenv then - echo "Could not install Python dependencies. Aborting bootstrap!" + error "Could not install Python dependencies. Aborting bootstrap!" exit 1 fi @@ -578,7 +593,7 @@ BootstrapMageiaCommon() { libffi-devel \ rootcerts then - echo "Could not install additional dependencies. Aborting bootstrap!" + error "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi } @@ -605,11 +620,11 @@ Bootstrap() { BootstrapMessage "Archlinux" BootstrapArchCommon else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S certbot certbot-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." + error "Please use pacman to install letsencrypt packages:" + error "# pacman -S certbot certbot-apache" + error + error "If you would like to use the virtualenv way, please run the script again with the" + error "--debug flag." exit 1 fi elif [ -f /etc/manjaro-release ]; then @@ -625,11 +640,11 @@ Bootstrap() { elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" - echo - echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info." + error "Sorry, I don't know how to bootstrap Certbot on your operating system!" + error + error "You will need to install OS dependencies, configure virtualenv, and run pip install manually." + error "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + error "for more info." exit 1 fi } @@ -649,7 +664,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # grep for both certbot and letsencrypt until certbot and shim packages have been released INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) if [ -z "$INSTALLED_VERSION" ]; then - echo "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 + error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 "$VENV_BIN/letsencrypt" --version exit 1 fi @@ -657,7 +672,7 @@ if [ "$1" = "--le-auto-phase2" ]; then INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Creating virtual environment..." + say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then @@ -666,7 +681,7 @@ if [ "$1" = "--le-auto-phase2" ]; then virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi - echo "Installing Python packages..." + say "Installing Python packages..." TEMP_DIR=$(TempDir) trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. @@ -845,18 +860,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.13.0 \ - --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ - --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 -certbot==0.13.0 \ - --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ - --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 -certbot-apache==0.13.0 \ - --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ - --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 -certbot-nginx==0.13.0 \ - --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ - --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 +acme==0.14.0 \ + --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ + --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 +certbot==0.14.0 \ + --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ + --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 +certbot-apache==0.14.0 \ + --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ + --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f +certbot-nginx==0.14.0 \ + --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ + --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1022,42 +1037,40 @@ UNLIKELY_EOF set -e if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) - echo "Had a problem while installing Python packages." + error "Had a problem while installing Python packages." if [ "$VERBOSE" != 1 ]; then - echo - echo "pip prints the following errors: " - echo "=====================================================" - echo "$PIP_OUT" - echo "=====================================================" - echo - echo "Certbot has problem setting up the virtual environment." + error + error "pip prints the following errors: " + error "=====================================================" + error "$PIP_OUT" + error "=====================================================" + error + error "Certbot has problem setting up the virtual environment." if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - echo - echo "Based on your pip output, the problem can likely be fixed by " - echo "increasing the available memory." + error + error "Based on your pip output, the problem can likely be fixed by " + error "increasing the available memory." else - echo - echo "We were not be able to guess the right solution from your pip " - echo "output." + error + error "We were not be able to guess the right solution from your pip " + error "output." fi - echo - echo "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - echo "for possible solutions." - echo "You may also find some support resources at https://certbot.eff.org/support/ ." + error + error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" + error "for possible solutions." + error "You may also find some support resources at https://certbot.eff.org/support/ ." fi rm -rf "$VENV_PATH" exit 1 fi - echo "Installation succeeded." + say "Installation succeeded." fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - if [ "$QUIET" != 1 ]; then - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" - fi + say "Requesting root privileges to run certbot..." + say " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop @@ -1084,7 +1097,7 @@ else Bootstrap fi if [ "$OS_PACKAGES_ONLY" = 1 ]; then - echo "OS packages installed." + say "OS packages installed." exit 0 fi @@ -1227,9 +1240,9 @@ UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - echo "WARNING: unable to check for updates." + error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -1238,7 +1251,7 @@ UNLIKELY_EOF # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing certbot-auto..." + say "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index dfd05bfd9..47495a0dc 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0.dev0' +version = '0.14.0' install_requires = [ 'certbot', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 63b6f16af..4c3c45cd9 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0.dev0' +version = '0.14.0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 228a1a2a6..1245bf10b 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.14.0.dev0' +__version__ = '0.14.0' diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 3154bbda6..2318a7255 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -85,11 +85,16 @@ optional arguments: --user-agent USER_AGENT Set a custom user agent string for the client. User agent strings allow the CA to collect high level - statistics about success rates by OS and plugin. If - you wish to hide your server OS version from the Let's - Encrypt server, set this to "". (default: - CertbotACMEClient/0.13.0 (Ubuntu 16.04.2 LTS) - Authenticator/XXX Installer/YYY) + statistics about success rates by OS, plugin and use + case, and to know when to deprecate support for past + Python versions and flags. If you wish to hide this + information from the Let's Encrypt server, set this to + "". (default: CertbotACMEClient/0.14.0 (certbot; + Ubuntu 16.04.2 LTS) Authenticator/XXX Installer/YYY + (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags + encoded in the user agent are: --duplicate, --force- + renew, --allow-subset-of-names, -n, and whether any + hooks are set. automation: Arguments for automating execution & other tweaks @@ -269,8 +274,8 @@ renew: "/etc/letsencrypt/live/example.com") containing the new certs and keys; the shell variable $RENEWED_DOMAINS will contain a space-delimited list - of renewed cert domains (for example, - "example.com www.example.com") (default: None) + of renewed cert domains (for example, "example.com + www.example.com" (default: None) --disable-hook-validation Ordinarily the commands specified for --pre-hook /--post-hook/--renew-hook will be checked for @@ -375,59 +380,6 @@ plugins: --webroot Obtain certs by placing files in a webroot directory. (default: False) -nginx: - Nginx Web Server plugin - Alpha - - --nginx-server-root NGINX_SERVER_ROOT - Nginx server root directory. (default: /etc/nginx) - --nginx-ctl NGINX_CTL - Path to the 'nginx' binary, used for 'configtest' and - retrieving nginx version number. (default: nginx) - -standalone: - Spin up a temporary webserver - -manual: - Authenticate through manual configuration or custom shell scripts. When - using shell scripts, an authenticator script must be provided. The - environment variables available to this script are $CERTBOT_DOMAIN which - contains the domain being authenticated, $CERTBOT_VALIDATION which is the - validation string, and $CERTBOT_TOKEN which is the filename of the - resource requested when performing an HTTP-01 challenge. An additional - cleanup script can also be provided and can use the additional variable - $CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth - script. - - --manual-auth-hook MANUAL_AUTH_HOOK - Path or command to execute for the authentication - script (default: None) - --manual-cleanup-hook MANUAL_CLEANUP_HOOK - Path or command to execute for the cleanup script - (default: None) - --manual-public-ip-logging-ok - Automatically allows public IP logging (default: Ask) - -webroot: - Place files in webroot directory - - --webroot-path WEBROOT_PATH, -w WEBROOT_PATH - public_html / webroot path. This can be specified - multiple times to handle different domains; each - domain will have the webroot path that preceded it. - For instance: `-w /var/www/example -d example.com -d - www.example.com -w /var/www/thing -d thing.net -d - m.thing.net` (default: Ask) - --webroot-map WEBROOT_MAP - JSON dictionary mapping domains to webroot paths; this - implies -d for each entry. You may need to escape this - from your shell. E.g.: --webroot-map - '{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' - This option is merged with, but takes precedence over, - -w / -d entries. At present, if you put webroot-map in - a config file, it needs to be on a single line, like: - webroot-map = {"example.com":"/var/www"}. (default: - {}) - apache: Apache Web Server plugin - Beta @@ -458,5 +410,58 @@ apache: Let installer handle enabling sites for you.(Only Ubuntu/Debian currently) (default: True) +manual: + Authenticate through manual configuration or custom shell scripts. When + using shell scripts, an authenticator script must be provided. The + environment variables available to this script are $CERTBOT_DOMAIN which + contains the domain being authenticated, $CERTBOT_VALIDATION which is the + validation string, and $CERTBOT_TOKEN which is the filename of the + resource requested when performing an HTTP-01 challenge. An additional + cleanup script can also be provided and can use the additional variable + $CERTBOT_AUTH_OUTPUT which contains the stdout output from the auth + script. + + --manual-auth-hook MANUAL_AUTH_HOOK + Path or command to execute for the authentication + script (default: None) + --manual-cleanup-hook MANUAL_CLEANUP_HOOK + Path or command to execute for the cleanup script + (default: None) + --manual-public-ip-logging-ok + Automatically allows public IP logging (default: Ask) + +nginx: + Nginx Web Server plugin - Alpha + + --nginx-server-root NGINX_SERVER_ROOT + Nginx server root directory. (default: /etc/nginx) + --nginx-ctl NGINX_CTL + Path to the 'nginx' binary, used for 'configtest' and + retrieving nginx version number. (default: nginx) + null: Null Installer + +standalone: + Spin up a temporary webserver + +webroot: + Place files in webroot directory + + --webroot-path WEBROOT_PATH, -w WEBROOT_PATH + public_html / webroot path. This can be specified + multiple times to handle different domains; each + domain will have the webroot path that preceded it. + For instance: `-w /var/www/example -d example.com -d + www.example.com -w /var/www/thing -d thing.net -d + m.thing.net` (default: Ask) + --webroot-map WEBROOT_MAP + JSON dictionary mapping domains to webroot paths; this + implies -d for each entry. You may need to escape this + from your shell. E.g.: --webroot-map + '{"eg1.is,m.eg1.is":"/www/eg1/", "eg2.is":"/www/eg2"}' + This option is merged with, but takes precedence over, + -w / -d entries. At present, if you put webroot-map in + a config file, it needs to be on a single line, like: + webroot-map = {"example.com":"/var/www"}. (default: + {}) diff --git a/letsencrypt-auto b/letsencrypt-auto index fc8007c9e..0db142a78 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -15,6 +15,11 @@ set -e # Work even if somebody does "sh thisscript.sh". # Note: you can set XDG_DATA_HOME or VENV_PATH before running this script, # if you want to change where the virtual environment will be installed + +# HOME might not be defined when being run through something like systemd +if [ -z "$HOME" ]; then + HOME=~root +fi if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME=~/.local/share fi @@ -23,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.13.0" +LE_AUTO_VERSION="0.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -59,7 +64,7 @@ for arg in "$@" ; do NO_BOOTSTRAP=1;; --help) HELP=1;; - --noninteractive|--non-interactive) + --noninteractive|--non-interactive|renew) ASSUME_YES=1;; --quiet) QUIET=1;; @@ -93,6 +98,16 @@ if [ "$QUIET" = 1 ]; then ASSUME_YES=1 fi +say() { + if [ "$QUIET" != 1 ]; then + echo "$@" + fi +} + +error() { + echo "$@" +} + # Support for busybox and others where there is no "command", # but "which" instead if command -v command > /dev/null 2>&1 ; then @@ -100,7 +115,7 @@ if command -v command > /dev/null 2>&1 ; then elif which which > /dev/null 2>&1 ; then export EXISTS="which" else - echo "Cannot find command nor which... please install one!" + error "Cannot find command nor which... please install one!" exit 1 fi @@ -145,17 +160,17 @@ if [ -n "${LE_AUTO_SUDO+x}" ]; then ;; '') ;; # Nothing to do for plain root method. *) - echo "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." + error "Error: unknown root authorization mechanism '$LE_AUTO_SUDO'." exit 1 esac - echo "Using preset root authorization mechanism '$LE_AUTO_SUDO'." + say "Using preset root authorization mechanism '$LE_AUTO_SUDO'." else if test "`id -u`" -ne "0" ; then if $EXISTS sudo 1>/dev/null 2>&1; then SUDO=sudo SUDO_ENV="CERTBOT_AUTO=$0" else - echo \"sudo\" is not available, will use \"su\" for installation steps... + say \"sudo\" is not available, will use \"su\" for installation steps... SUDO=su_sudo fi else @@ -165,7 +180,7 @@ fi BootstrapMessage() { # Arguments: Platform name - echo "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" + say "Bootstrapping dependencies for $1... (you can skip this with --no-bootstrap)" } ExperimentalBootstrap() { @@ -176,11 +191,11 @@ ExperimentalBootstrap() { $2 fi else - echo "FATAL: $1 support is very experimental at present..." - echo "if you would like to work on improving it, please ensure you have backups" - echo "and then run this script again with the --debug flag!" - echo "Alternatively, you can install OS dependencies yourself and run this script" - echo "again with --no-bootstrap." + error "FATAL: $1 support is very experimental at present..." + error "if you would like to work on improving it, please ensure you have backups" + error "and then run this script again with the --debug flag!" + error "Alternatively, you can install OS dependencies yourself and run this script" + error "again with --no-bootstrap." exit 1 fi } @@ -191,15 +206,15 @@ DeterminePythonVersion() { $EXISTS "$LE_PYTHON" > /dev/null && break done if [ "$?" != "0" ]; then - echo "Cannot find any Pythons; please install one!" + error "Cannot find any Pythons; please install one!" exit 1 fi export LE_PYTHON PYVER=`"$LE_PYTHON" -V 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'` if [ "$PYVER" -lt 26 ]; then - echo "You have an ancient version of Python entombed in your operating system..." - echo "This isn't going to work; you'll need at least version 2.6." + error "You have an ancient version of Python entombed in your operating system..." + error "This isn't going to work; you'll need at least version 2.6." exit 1 fi } @@ -227,7 +242,7 @@ BootstrapDebCommon() { QUIET_FLAG='-qq' fi - $SUDO apt-get $QUIET_FLAG update || echo apt-get update hit problems but continuing anyway... + $SUDO apt-get $QUIET_FLAG update || error apt-get update hit problems but continuing anyway... # virtualenv binary can be found in different packages depending on # distro version (#346) @@ -255,7 +270,7 @@ BootstrapDebCommon() { # ARGS: BACKPORT_NAME="$1" BACKPORT_SOURCELINE="$2" - echo "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." + say "To use the Apache Certbot plugin, augeas needs to be installed from $BACKPORT_NAME." if ! grep -v -e ' *#' /etc/apt/sources.list | grep -q "$BACKPORT_NAME" ; then # This can theoretically error if sources.list.d is empty, but in that case we don't care. if ! grep -v -e ' *#' /etc/apt/sources.list.d/* 2>/dev/null | grep -q "$BACKPORT_NAME"; then @@ -315,7 +330,7 @@ BootstrapDebCommon() { if ! $EXISTS virtualenv > /dev/null ; then - echo Failed to install a working \"virtualenv\" command, exiting + error Failed to install a working \"virtualenv\" command, exiting exit 1 fi } @@ -335,7 +350,7 @@ BootstrapRpmCommon() { tool=yum else - echo "Neither yum nor dnf found. Aborting bootstrap!" + error "Neither yum nor dnf found. Aborting bootstrap!" exit 1 fi @@ -349,7 +364,7 @@ BootstrapRpmCommon() { if ! $SUDO $tool list *virtualenv >/dev/null 2>&1; then echo "To use Certbot, packages from the EPEL repository need to be installed." if ! $SUDO $tool list epel-release >/dev/null 2>&1; then - echo "Please enable this repository and try running Certbot again." + error "Enable the EPEL repository and try running Certbot again." exit 1 fi if [ "$ASSUME_YES" = 1 ]; then @@ -361,7 +376,7 @@ BootstrapRpmCommon() { sleep 1s fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG epel-release; then - echo "Could not enable EPEL. Aborting bootstrap!" + error "Could not enable EPEL. Aborting bootstrap!" exit 1 fi fi @@ -403,7 +418,7 @@ BootstrapRpmCommon() { fi if ! $SUDO $tool install $yes_flag $QUIET_FLAG $pkgs; then - echo "Could not install OS dependencies. Aborting bootstrap!" + error "Could not install OS dependencies. Aborting bootstrap!" exit 1 fi } @@ -508,15 +523,15 @@ BootstrapFreeBsd() { BootstrapMac() { if hash brew 2>/dev/null; then - echo "Using Homebrew to install dependencies..." + say "Using Homebrew to install dependencies..." pkgman=brew pkgcmd="brew install" elif hash port 2>/dev/null; then - echo "Using MacPorts to install dependencies..." + say "Using MacPorts to install dependencies..." pkgman=port pkgcmd="$SUDO port install" else - echo "No Homebrew/MacPorts; installing Homebrew..." + say "No Homebrew/MacPorts; installing Homebrew..." ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" pkgman=brew pkgcmd="brew install" @@ -527,26 +542,26 @@ BootstrapMac() { -o "$(which python)" = "/usr/bin/python" ]; then # We want to avoid using the system Python because it requires root to use pip. # python.org, MacPorts or HomeBrew Python installations should all be OK. - echo "Installing python..." + say "Installing python..." $pkgcmd python fi # Workaround for _dlopen not finding augeas on macOS if [ "$pkgman" = "port" ] && ! [ -e "/usr/local/lib/libaugeas.dylib" ] && [ -e "/opt/local/lib/libaugeas.dylib" ]; then - echo "Applying augeas workaround" + say "Applying augeas workaround" $SUDO mkdir -p /usr/local/lib/ $SUDO ln -s /opt/local/lib/libaugeas.dylib /usr/local/lib/ fi if ! hash pip 2>/dev/null; then - echo "pip not installed" - echo "Installing pip..." + say "pip not installed" + say "Installing pip..." curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python fi if ! hash virtualenv 2>/dev/null; then - echo "virtualenv not installed." - echo "Installing with pip..." + say "virtualenv not installed." + say "Installing with pip..." pip install virtualenv fi } @@ -566,7 +581,7 @@ BootstrapMageiaCommon() { libpython-devel \ python-virtualenv then - echo "Could not install Python dependencies. Aborting bootstrap!" + error "Could not install Python dependencies. Aborting bootstrap!" exit 1 fi @@ -578,7 +593,7 @@ BootstrapMageiaCommon() { libffi-devel \ rootcerts then - echo "Could not install additional dependencies. Aborting bootstrap!" + error "Could not install additional dependencies. Aborting bootstrap!" exit 1 fi } @@ -605,11 +620,11 @@ Bootstrap() { BootstrapMessage "Archlinux" BootstrapArchCommon else - echo "Please use pacman to install letsencrypt packages:" - echo "# pacman -S certbot certbot-apache" - echo - echo "If you would like to use the virtualenv way, please run the script again with the" - echo "--debug flag." + error "Please use pacman to install letsencrypt packages:" + error "# pacman -S certbot certbot-apache" + error + error "If you would like to use the virtualenv way, please run the script again with the" + error "--debug flag." exit 1 fi elif [ -f /etc/manjaro-release ]; then @@ -625,11 +640,11 @@ Bootstrap() { elif [ -f /etc/product ] && grep -q "Joyent Instance" /etc/product ; then ExperimentalBootstrap "Joyent SmartOS Zone" BootstrapSmartOS else - echo "Sorry, I don't know how to bootstrap Certbot on your operating system!" - echo - echo "You will need to install OS dependencies, configure virtualenv, and run pip install manually." - echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" - echo "for more info." + error "Sorry, I don't know how to bootstrap Certbot on your operating system!" + error + error "You will need to install OS dependencies, configure virtualenv, and run pip install manually." + error "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites" + error "for more info." exit 1 fi } @@ -649,7 +664,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # grep for both certbot and letsencrypt until certbot and shim packages have been released INSTALLED_VERSION=$("$VENV_BIN/letsencrypt" --version 2>&1 | grep "^certbot\|^letsencrypt" | cut -d " " -f 2) if [ -z "$INSTALLED_VERSION" ]; then - echo "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 + error "Error: couldn't get currently installed version for $VENV_BIN/letsencrypt: " 1>&2 "$VENV_BIN/letsencrypt" --version exit 1 fi @@ -657,7 +672,7 @@ if [ "$1" = "--le-auto-phase2" ]; then INSTALLED_VERSION="none" fi if [ "$LE_AUTO_VERSION" != "$INSTALLED_VERSION" ]; then - echo "Creating virtual environment..." + say "Creating virtual environment..." DeterminePythonVersion rm -rf "$VENV_PATH" if [ "$VERBOSE" = 1 ]; then @@ -666,7 +681,7 @@ if [ "$1" = "--le-auto-phase2" ]; then virtualenv --no-site-packages --python "$LE_PYTHON" "$VENV_PATH" > /dev/null fi - echo "Installing Python packages..." + say "Installing Python packages..." TEMP_DIR=$(TempDir) trap 'rm -rf "$TEMP_DIR"' EXIT # There is no $ interpolation due to quotes on starting heredoc delimiter. @@ -845,18 +860,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.13.0 \ - --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ - --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 -certbot==0.13.0 \ - --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ - --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 -certbot-apache==0.13.0 \ - --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ - --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 -certbot-nginx==0.13.0 \ - --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ - --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 +acme==0.14.0 \ + --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ + --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 +certbot==0.14.0 \ + --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ + --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 +certbot-apache==0.14.0 \ + --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ + --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f +certbot-nginx==0.14.0 \ + --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ + --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db UNLIKELY_EOF # ------------------------------------------------------------------------- @@ -1022,42 +1037,40 @@ UNLIKELY_EOF set -e if [ "$PIP_STATUS" != 0 ]; then # Report error. (Otherwise, be quiet.) - echo "Had a problem while installing Python packages." + error "Had a problem while installing Python packages." if [ "$VERBOSE" != 1 ]; then - echo - echo "pip prints the following errors: " - echo "=====================================================" - echo "$PIP_OUT" - echo "=====================================================" - echo - echo "Certbot has problem setting up the virtual environment." + error + error "pip prints the following errors: " + error "=====================================================" + error "$PIP_OUT" + error "=====================================================" + error + error "Certbot has problem setting up the virtual environment." if `echo $PIP_OUT | grep -q Killed` || `echo $PIP_OUT | grep -q "allocate memory"` ; then - echo - echo "Based on your pip output, the problem can likely be fixed by " - echo "increasing the available memory." + error + error "Based on your pip output, the problem can likely be fixed by " + error "increasing the available memory." else - echo - echo "We were not be able to guess the right solution from your pip " - echo "output." + error + error "We were not be able to guess the right solution from your pip " + error "output." fi - echo - echo "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" - echo "for possible solutions." - echo "You may also find some support resources at https://certbot.eff.org/support/ ." + error + error "Consult https://certbot.eff.org/docs/install.html#problems-with-python-virtual-environment" + error "for possible solutions." + error "You may also find some support resources at https://certbot.eff.org/support/ ." fi rm -rf "$VENV_PATH" exit 1 fi - echo "Installation succeeded." + say "Installation succeeded." fi if [ -n "$SUDO" ]; then # SUDO is su wrapper or sudo - if [ "$QUIET" != 1 ]; then - echo "Requesting root privileges to run certbot..." - echo " $VENV_BIN/letsencrypt" "$@" - fi + say "Requesting root privileges to run certbot..." + say " $VENV_BIN/letsencrypt" "$@" fi if [ -z "$SUDO_ENV" ] ; then # SUDO is su wrapper / noop @@ -1084,7 +1097,7 @@ else Bootstrap fi if [ "$OS_PACKAGES_ONLY" = 1 ]; then - echo "OS packages installed." + say "OS packages installed." exit 0 fi @@ -1227,9 +1240,9 @@ UNLIKELY_EOF # --------------------------------------------------------------------------- DeterminePythonVersion if ! REMOTE_VERSION=`"$LE_PYTHON" "$TEMP_DIR/fetch.py" --latest-version` ; then - echo "WARNING: unable to check for updates." + error "WARNING: unable to check for updates." elif [ "$LE_AUTO_VERSION" != "$REMOTE_VERSION" ]; then - echo "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." + say "Upgrading certbot-auto $LE_AUTO_VERSION to $REMOTE_VERSION..." # Now we drop into Python so we don't have to install even more # dependencies (curl, etc.), for better flow control, and for the option of @@ -1238,7 +1251,7 @@ UNLIKELY_EOF # Install new copy of certbot-auto. # TODO: Deal with quotes in pathnames. - echo "Replacing certbot-auto..." + say "Replacing certbot-auto..." # Clone permissions with cp. chmod and chown don't have a --reference # option on macOS or BSD, and stat -c on Linux is stat -f on macOS and BSD: $SUDO cp -p "$0" "$TEMP_DIR/letsencrypt-auto.permission-clone" diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index bf267dc25..236e4dcdc 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQEcBAABCAAGBQJY5WxEAAoJEE0XyZXNl3XyoDYH/joyJ/7cS4+SoTEiPpVcDnK+ -YJVhxP6pir6GaRvl+ebWlo7ichS4c0Kye8e5BPVj5RtZbDT88iplMZ2EyUmeA579 -8Z96p9qoEANeGWiPe+KCDXRHJfCAsphcHSLTeS8lXgG8SP13p7hsML6hn3gosRdu -OG4/SnFBDLLwu4YwUVom4U+Z+dYS1jQstge4sexr85jCX/Lds7M5WM/lFiYMBsJ8 -uZd/IGKwb7jvsc4u58Ruj9xiTcchaxn15NMJR7R967Mt5ortSvZ3C6Cv3NyubJmB -hmGQVU+eNBTeEwPSIN8xAf3fcwh2wlRMaTZOy5nJ3IoDdSQuwO9IGxxdkNDSegE= -=8KUq +iQEcBAABCAAGBQJZC76WAAoJEE0XyZXNl3XyXhcIAJ1+gPoWZmXjFcC4by2tDBoM +Lkxf5BNxq8aq7qSohU8SqSo6ShDkWh9ci390n+jbOX1R503uQL1egGbEAJbziFYq +vym6j0AmqM+2/YcWmcj3J7RYtDOV1sUPKD2pgUxWtvQrd9iZ1235WMzBF/uBprzm +qAtFwF04V2H3kkC4e7+jAEkFzs1TJ8fYumqqqw0NgSwM6bikfurpRyf8qR2RVYWt +e3GOTxyBVbjhp2UPy/O8Xx7iBD3m+t9mJgsCJ9l8s7xKot6LF7+WrJkn0A3cfKcR +LSTataKedsP3u1jOgP3y2ujumBlDlDRuXn6vK/YKNYNnHte5B9mstSzoDGgRvHE= +=3Jgs -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 50c80f0a4..0db142a78 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -28,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.14.0.dev0" +LE_AUTO_VERSION="0.14.0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -860,18 +860,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.13.0 \ - --hash=sha256:103ce8bed43aad1a9655ed815df09bbeab86ee16cc82137b44d9dac68faa394f \ - --hash=sha256:7489b3e20d02da0a389aedb82408ffb6b76294e41d833db85591b9f779539815 -certbot==0.13.0 \ - --hash=sha256:65d0d9d158972aff7746d4ef80a20465a14c54ae8bcb879216970c2a1b34503c \ - --hash=sha256:f63ad7747edaca2fb7d60c28882e44d2f48ff1cca9b9c7c251ad47e2189c00f3 -certbot-apache==0.13.0 \ - --hash=sha256:22f7c1dc93439384c0874960081d66957910c6dc737a9facbd9fcbc46c545874 \ - --hash=sha256:b43b04b53005e7218a09a0ba4d97581fab369e929472fa49fb55d29d0ab54589 -certbot-nginx==0.13.0 \ - --hash=sha256:9d0ab4eeb98b0ebad70ba116b32268342ad343d82d64990a652ff8072959b044 \ - --hash=sha256:f026a8faee8397a22c5d4a7623a6ef7c7e780ed63a3bdf9940f43f7823aa2a72 +acme==0.14.0 \ + --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ + --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 +certbot==0.14.0 \ + --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ + --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 +certbot-apache==0.14.0 \ + --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ + --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f +certbot-nginx==0.14.0 \ + --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ + --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index 723cc2f8ca6c97bde3befac7184eaa55565dfd11..cb07e9573ee0c8b116c5b9a8d758cc0e2b10961d 100644 GIT binary patch literal 256 zcmV+b0ssD}*cqby5KtZP(Mxv3J)FkHt>eYFn0e(bA5G*B{HO!8qxTDY5xsI2*#Mgb z5U%;nxm4m=scfQB7aLexo@l4Q2a?^K3zSQFK=7TboMlyK zu)t&mKKnf*3aVCnqzu@VE>eW$VD5r6{8T*}hy-5gBVxQ^RM!@X_)XDdF46=vEGDm1 zt$DKaS%!b_${M$E{07#~CZwuL&&vru10`97U;Qrc0@V1C=`Dfr+8M{H_isyB`Ki9b G^;U^#=z%T( literal 256 zcmV+b0ssCL!AY$O$G=(YPL-?HISZVL;uLSvo4ezuc`p!goY|_rlo`sQjXmnLA??J3 zG3#@-hsQdZ3;EW6kL#){9%73pZgfPO^V{n`wAQwiKV_n~hA*hATMAtDbe^FV?7KHMU7&ibA zY)(7YTayW2J7xLFKD;j4uGs>>g0Ud^f#2rW8lI&1oBpWdE0zySF(Jdneg#Byz|O6G z%PaA(4sIzeHzBn34||(^k7n Date: Thu, 4 May 2017 16:52:29 -0700 Subject: [PATCH 55/96] Bump version to 0.15.0 --- acme/setup.py | 2 +- certbot-apache/setup.py | 2 +- certbot-compatibility-test/setup.py | 2 +- certbot-nginx/setup.py | 2 +- certbot/__init__.py | 2 +- letsencrypt-auto-source/letsencrypt-auto | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acme/setup.py b/acme/setup.py index e38cadf2c..0938c5c68 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0' +version = '0.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index dcf604c55..9429ada2e 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0' +version = '0.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot-compatibility-test/setup.py b/certbot-compatibility-test/setup.py index 47495a0dc..c5ec54e22 100644 --- a/certbot-compatibility-test/setup.py +++ b/certbot-compatibility-test/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0' +version = '0.15.0.dev0' install_requires = [ 'certbot', diff --git a/certbot-nginx/setup.py b/certbot-nginx/setup.py index 4c3c45cd9..4329eb858 100644 --- a/certbot-nginx/setup.py +++ b/certbot-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.14.0' +version = '0.15.0.dev0' # Please update tox.ini when modifying dependency version requirements install_requires = [ diff --git a/certbot/__init__.py b/certbot/__init__.py index 1245bf10b..5128bf096 100644 --- a/certbot/__init__.py +++ b/certbot/__init__.py @@ -1,4 +1,4 @@ """Certbot client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.14.0' +__version__ = '0.15.0.dev0' diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 0db142a78..d4433d525 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -28,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.14.0" +LE_AUTO_VERSION="0.15.0.dev0" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates From 51ae69698d676e01a08260c0e2e593815a0ee1e0 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Fri, 5 May 2017 23:49:54 +0800 Subject: [PATCH 56/96] Allow boulder-fetch.sh run with `ip` from iproute2 (#4620) --- tests/boulder-fetch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/boulder-fetch.sh b/tests/boulder-fetch.sh index d9a979667..60538362e 100755 --- a/tests/boulder-fetch.sh +++ b/tests/boulder-fetch.sh @@ -13,6 +13,7 @@ fi cd ${BOULDERPATH} FAKE_DNS=$(ifconfig docker0 | grep "inet addr:" | cut -d: -f2 | awk '{ print $1}') [ -z "$FAKE_DNS" ] && FAKE_DNS=$(ifconfig docker0 | grep "inet " | xargs | cut -d ' ' -f 2) +[ -z "$FAKE_DNS" ] && FAKE_DNS=$(ip addr show dev docker0 | grep "inet " | xargs | cut -d ' ' -f 2 | cut -d '/' -f 1) [ -z "$FAKE_DNS" ] && echo Unable to find the IP for docker0 && exit 1 sed -i "s/FAKE_DNS: .*/FAKE_DNS: ${FAKE_DNS}/" docker-compose.yml docker-compose up -d From d8fbd4f31ddac0f34794c819992a7752d94ce0a3 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 5 May 2017 10:10:28 -0700 Subject: [PATCH 57/96] Add 0.14.0 release notes (#4618) --- CHANGELOG.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8ee45024..1b8d5c0c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,39 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.14.0 - 2017-05-04 + +### Added + +* Python 3.3+ support for all Certbot packages. `certbot-auto` still currently +only supports Python 2, but the `acme`, `certbot`, `certbot-apache`, and +`certbot-nginx` packages on PyPI now fully support Python 2.6, 2.7, and 3.3+. +* Certbot's Apache plugin now handles multiple virtual hosts per file. +* Lockfiles to prevent multiple versions of Certbot running simultaneously. + +### Changed + +* When converting an HTTP virtual host to HTTPS in Apache, Certbot only copies +the virtual host rather than the entire contents of the file it's contained +in. +* The Nginx plugin now includes SSL/TLS directives in a separate file located +in Certbot's configuration directory rather than copying the contents of the +file into every modified `server` block. + +### Fixed + +* Ensure logging is configured before parts of Certbot attempt to log any +messages. +* Support for the `--quiet` flag in `certbot-auto`. +* Reverted a change made in a previous release to make the `acme` and `certbot` +packages always depend on `argparse`. This dependency is conditional again on +the user's Python version. +* Small bugs in the Nginx plugin such as properly handling empty `server` +blocks and setting `server_names_hash_bucket_size` during challenges. + +As always, a more complete list of changes can be found on GitHub: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.14.0+is%3Aclosed + ## 0.13.0 - 2017-04-06 ### Added From 1d876aba231579a90563a39b753cda6cba45366e Mon Sep 17 00:00:00 2001 From: Noah Swartz Date: Mon, 8 May 2017 10:54:19 -0700 Subject: [PATCH 58/96] update README (#4623) --- README.rst | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index c51168216..ff6fafe36 100644 --- a/README.rst +++ b/README.rst @@ -96,18 +96,10 @@ ACME spec: http://ietf-wg-acme.github.io/acme/ ACME working area in github: https://github.com/ietf-wg-acme/acme - -Mailing list: `client-dev`_ (to subscribe without a Google account, send an -email to client-dev+subscribe@letsencrypt.org) - |build-status| |coverage| |docs| |container| .. _Freenode: https://webchat.freenode.net?channels=%23letsencrypt -.. _OFTC: https://webchat.oftc.net?channels=%23certbot - -.. _client-dev: https://groups.google.com/a/letsencrypt.org/forum/#!forum/client-dev - .. |build-status| image:: https://travis-ci.org/certbot/certbot.svg?branch=master :target: https://travis-ci.org/certbot/certbot :alt: Travis CI status @@ -141,7 +133,7 @@ Current Features * Supports multiple web servers: - apache/2.x (beta support for auto-configuration) - - nginx/0.8.48+ (alpha support for auto-configuration) + - nginx/0.8.48+ (alpha support for auto-configuration, beta support in 0.14.0) - webroot (adds files to webroot directories in order to prove control of domains and obtain certs) - standalone (runs its own simple webserver to prove you control a domain) @@ -157,7 +149,7 @@ Current Features runs https only (Apache only) * Fully automated. * Configuration changes are logged and can be reverted. -* Supports ncurses and text (-t) UI, or can be driven entirely from the +* Supports an interactive text UI, or can be driven entirely from the command line. * Free and Open Source Software, made with Python. From c6fcb017b840e46110f45ddb3c6aa93fe46ddea7 Mon Sep 17 00:00:00 2001 From: Yen Chi Hsuan Date: Tue, 9 May 2017 01:55:03 +0800 Subject: [PATCH 59/96] Use universal_newlines=True whereever the output is used (#4626) --- certbot/util.py | 3 ++- letshelp-certbot/letshelp_certbot/apache.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/certbot/util.py b/certbot/util.py index 1cbef7e80..e343fdcb1 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -429,7 +429,8 @@ def get_python_os_info(): elif os_type.startswith('darwin'): os_ver = subprocess.Popen( ["sw_vers", "-productVersion"], - stdout=subprocess.PIPE + stdout=subprocess.PIPE, + universal_newlines=True, ).communicate()[0].rstrip('\n') elif os_type.startswith('freebsd'): # eg "9.3-RC3-p1" diff --git a/letshelp-certbot/letshelp_certbot/apache.py b/letshelp-certbot/letshelp_certbot/apache.py index 2391a30bb..b13057ca5 100755 --- a/letshelp-certbot/letshelp_certbot/apache.py +++ b/letshelp-certbot/letshelp_certbot/apache.py @@ -183,19 +183,22 @@ def setup_tempdir(args): config_fd.write(args.config_file + "\n") proc = subprocess.Popen([args.apache_ctl, "-v"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) with open(os.path.join(tempdir, "version"), "w") as version_fd: version_fd.write(proc.communicate()[0]) proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f", args.config_file, "-M"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) with open(os.path.join(tempdir, "modules"), "w") as modules_fd: modules_fd.write(proc.communicate()[0]) proc = subprocess.Popen([args.apache_ctl, "-d", args.server_root, "-f", args.config_file, "-t", "-D", "DUMP_VHOSTS"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) with open(os.path.join(tempdir, "vhosts"), "w") as vhosts_fd: vhosts_fd.write(proc.communicate()[0]) @@ -231,7 +234,8 @@ def locate_config(apache_ctl): """ try: proc = subprocess.Popen([apache_ctl, "-V"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) output, _ = proc.communicate() except OSError: sys.exit(_NO_APACHECTL) From 6670f828ef7c0e96d67d612e52905e8be6e3a674 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 8 May 2017 11:54:12 -0700 Subject: [PATCH 60/96] Deduplicate package lists in tox.ini (#4608) Use substitution of values form other sections[1] to deduplicate information in tox.ini, including pip install arguments and package paths. 1 - https://tox.readthedocs.io/en/latest/config.html#substitution-for-values-from-other-sections --- tox.ini | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tox.ini b/tox.ini index 33cacf061..b073f03b8 100644 --- a/tox.ini +++ b/tox.ini @@ -9,21 +9,40 @@ envlist = modification,py{26,33,34,35,36},cover,lint # nosetest -v => more verbose output, allows to detect busy waiting # loops, especially on Travis -[testenv] +[base] # packages installed separately to ensure that downstream deps problems # are detected, c.f. #1002 -commands = +core_commands = pip install -e acme[dev] nosetests -v acme --processes=-1 pip install -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 +core_install_args = -e acme[dev] -e .[dev] +core_paths = acme/acme certbot + +plugin_commands = pip install -e certbot-apache nosetests -v certbot_apache --processes=-1 --process-timeout=80 pip install -e certbot-nginx nosetests -v certbot_nginx --processes=-1 +plugin_install_args = -e certbot-apache -e certbot-nginx +plugin_paths = certbot-apache/certbot_apache certbot-nginx/certbot_nginx + +compatibility_install_args = -e certbot-compatibility-test +compatibility_paths = certbot-compatibility-test/certbot_compatibility_test + +other_commands = pip install -e letshelp-certbot nosetests -v letshelp_certbot --processes=-1 python tests/lock_test.py +other_install_args = -e letshelp-certbot +other_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py + +[testenv] +commands = + {[base]core_commands} + {[base]plugin_commands} + {[base]other_commands} setenv = PYTHONPATH = {toxinidir} @@ -44,12 +63,12 @@ deps = [testenv:py27_install] basepython = python2.7 commands = - pip install -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e letshelp-certbot + pip install {[base]core_install_args} {[base]plugin_install_args} {[base]other_install_args} [testenv:cover] basepython = python2.7 commands = - pip install -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e letshelp-certbot + pip install {[base]core_install_args} {[base]plugin_install_args} {[base]other_install_args} ./tox.cover.sh [testenv:lint] @@ -59,25 +78,25 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot - pylint --reports=n --rcfile=.pylintrc acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot tests/lock_test.py + pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:mypy] basepython = python3.4 commands = pip install mypy - pip install -q -e acme[dev] -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot - mypy --py2 --ignore-missing-imports acme/acme certbot certbot-apache/certbot_apache certbot-nginx/certbot_nginx certbot-compatibility-test/certbot_compatibility_test letshelp-certbot/letshelp_certbot + pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:apacheconftest] #basepython = python2.7 commands = - pip install -e acme -e .[dev] -e certbot-apache -e certbot-nginx -e certbot-compatibility-test -e letshelp-certbot + pip install {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules [testenv:nginxroundtrip] commands = - pip install -e acme[dev] -e .[dev] -e certbot-nginx + pip install {[base]core_install_args} -e certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata # This is a duplication of the command line in testenv:le_auto to From 3752ed4ee2127c749d63a8e678578d3fd55477f5 Mon Sep 17 00:00:00 2001 From: Alexander Krotov Date: Wed, 10 May 2017 21:43:56 +0300 Subject: [PATCH 61/96] ServerName and ServerAlias are directives, not directories (#4632) --- certbot-apache/certbot_apache/display_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/display_ops.py b/certbot-apache/certbot_apache/display_ops.py index 22aafc0fe..6bcb64dd5 100644 --- a/certbot-apache/certbot_apache/display_ops.py +++ b/certbot-apache/certbot_apache/display_ops.py @@ -91,7 +91,7 @@ def _vhost_menu(domain, vhosts): msg = ("Encountered vhost ambiguity but unable to ask for user guidance in " "non-interactive mode. Currently Certbot needs each vhost to be " "in its own conf file, and may need vhosts to be explicitly " - "labelled with ServerName or ServerAlias directories.") + "labelled with ServerName or ServerAlias directives.") logger.warning(msg) raise errors.MissingCommandlineFlag(msg) From db6defe614ea5a84ea07831b8913b5e5e56ae906 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Tue, 2 May 2017 10:12:23 -0700 Subject: [PATCH 62/96] Cloudflare DNS Authenticator Implement an Authenticator which can fulfill a dns-01 challenge using the Cloudflare API. Applicable only for domains using Cloudflare for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-cloudflare -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-cloudflare -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting cloudflare interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Credentials file missing e-mail address. * Credentials file with blank API key. * Credentials file with incorrect e-mail address. * Credentials file with malformed API key. * Credentials file with invalid API key. * Domain name not registered to Cloudflare account. --- certbot-dns-cloudflare/LICENSE.txt | 190 +++++++++++ certbot-dns-cloudflare/MANIFEST.in | 3 + certbot-dns-cloudflare/README.rst | 1 + .../certbot_dns_cloudflare/__init__.py | 1 + .../certbot_dns_cloudflare/dns_cloudflare.py | 198 +++++++++++ .../dns_cloudflare_test.py | 174 ++++++++++ certbot-dns-cloudflare/docs/.gitignore | 1 + certbot-dns-cloudflare/docs/Makefile | 20 ++ certbot-dns-cloudflare/docs/api.rst | 8 + .../docs/api/dns_cloudflare.rst | 5 + certbot-dns-cloudflare/docs/conf.py | 180 ++++++++++ certbot-dns-cloudflare/docs/index.rst | 27 ++ certbot-dns-cloudflare/docs/make.bat | 36 ++ certbot-dns-cloudflare/setup.cfg | 2 + certbot-dns-cloudflare/setup.py | 69 ++++ certbot/cli.py | 2 + certbot/plugins/disco.py | 1 + certbot/plugins/dns_common.py | 323 ++++++++++++++++++ certbot/plugins/dns_common_test.py | 233 +++++++++++++ certbot/plugins/dns_test_common.py | 63 ++++ certbot/plugins/selection.py | 4 +- docs/api/plugins/dns_common.rst | 5 + tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 18 +- 26 files changed, 1562 insertions(+), 8 deletions(-) create mode 100644 certbot-dns-cloudflare/LICENSE.txt create mode 100644 certbot-dns-cloudflare/MANIFEST.in create mode 100644 certbot-dns-cloudflare/README.rst create mode 100644 certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py create mode 100644 certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py create mode 100644 certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare_test.py create mode 100644 certbot-dns-cloudflare/docs/.gitignore create mode 100644 certbot-dns-cloudflare/docs/Makefile create mode 100644 certbot-dns-cloudflare/docs/api.rst create mode 100644 certbot-dns-cloudflare/docs/api/dns_cloudflare.rst create mode 100644 certbot-dns-cloudflare/docs/conf.py create mode 100644 certbot-dns-cloudflare/docs/index.rst create mode 100644 certbot-dns-cloudflare/docs/make.bat create mode 100644 certbot-dns-cloudflare/setup.cfg create mode 100644 certbot-dns-cloudflare/setup.py create mode 100644 certbot/plugins/dns_common.py create mode 100644 certbot/plugins/dns_common_test.py create mode 100644 certbot/plugins/dns_test_common.py create mode 100644 docs/api/plugins/dns_common.rst diff --git a/certbot-dns-cloudflare/LICENSE.txt b/certbot-dns-cloudflare/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-cloudflare/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-cloudflare/MANIFEST.in b/certbot-dns-cloudflare/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-cloudflare/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-cloudflare/README.rst b/certbot-dns-cloudflare/README.rst new file mode 100644 index 000000000..ff69eeaac --- /dev/null +++ b/certbot-dns-cloudflare/README.rst @@ -0,0 +1 @@ +Cloudflare DNS Authenticator plugin for Certbot diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py new file mode 100644 index 000000000..f4820e1ca --- /dev/null +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/__init__.py @@ -0,0 +1 @@ +"""Cloudflare DNS Authenticator""" diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py new file mode 100644 index 000000000..f8f674693 --- /dev/null +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py @@ -0,0 +1,198 @@ +"""DNS Authenticator for Cloudflare.""" +import logging + +import CloudFlare +import zope.interface + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common + +logger = logging.getLogger(__name__) + +ACCOUNT_URL = 'https://www.cloudflare.com/a/account/my-account' + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Cloudflare + + This Authenticator uses the Cloudflare API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using Cloudflare for DNS).' + ttl = 120 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): + super(Authenticator, cls).add_parser_arguments(add) + add('credentials', help='Cloudflare credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Cloudflare API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'Cloudflare credentials INI file', + { + 'email': 'email address associated with Cloudflare account', + 'api-key': 'API key for Cloudflare account, obtained from {0}'.format(ACCOUNT_URL) + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_cloudflare_client().add_txt_record(domain, validation_name, validation, self.ttl) + + def _cleanup(self, domain, validation_name, validation): + self._get_cloudflare_client().del_txt_record(domain, validation_name, validation) + + def _get_cloudflare_client(self): + return _CloudflareClient(self.credentials.conf('email'), self.credentials.conf('api-key')) + + +class _CloudflareClient(object): + """ + Encapsulates all communication with the Cloudflare API. + """ + + def __init__(self, email, api_key): + self.cf = CloudFlare.CloudFlare(email, api_key) + + def add_txt_record(self, domain, record_name, record_content, record_ttl): + """ + Add a TXT record using the supplied information. + + :param str domain: The domain to use to look up the Cloudflare zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL (number of seconds that the record may be cached). + :raises certbot.errors.PluginError: if an error occurs communicating with the Cloudflare API + """ + + zone_id = self._find_zone_id(domain) + + data = {'type': 'TXT', + 'name': record_name, + 'content': record_content, + 'ttl': record_ttl} + + try: + logger.debug('Attempting to add record to zone %s: %s', zone_id, data) + self.cf.zones.dns_records.post(zone_id, data=data) # zones | pylint: disable=no-member + except CloudFlare.exceptions.CloudFlareAPIError as e: + logger.error('Encountered CloudFlareAPIError adding TXT record: %d %s', e, e) + raise errors.PluginError('Error communicating with the Cloudflare API: {0}'.format(e)) + + record_id = self._find_txt_record_id(zone_id, record_name, record_content) + logger.debug('Successfully added TXT record with record_id: %s', record_id) + + def del_txt_record(self, domain, record_name, record_content): + """ + Delete a TXT record using the supplied information. + + Note that both the record's name and content are used to ensure that similar records + created concurrently (e.g., due to concurrent invocations of this plugin) are not deleted. + + Failures are logged, but not raised. + + :param str domain: The domain to use to look up the Cloudflare zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + """ + + try: + zone_id = self._find_zone_id(domain) + except errors.PluginError as e: + logger.debug('Encountered error finding zone_id during deletion: %s', e) + return + + if zone_id: + record_id = self._find_txt_record_id(zone_id, record_name, record_content) + if record_id: + try: + # zones | pylint: disable=no-member + self.cf.zones.dns_records.delete(zone_id, record_id) + logger.debug('Successfully deleted TXT record.') + except CloudFlare.exceptions.CloudFlareAPIError as e: + logger.warn('Encountered CloudFlareAPIError deleting TXT record: %s', e) + else: + logger.debug('TXT record not found; no cleanup needed.') + else: + logger.debug('Zone not found; no cleanup needed.') + + def _find_zone_id(self, domain): + """ + Find the zone_id for a given domain. + + :param str domain: The domain for which to find the zone_id. + :returns: The zone_id, if found. + :rtype: str + :raises certbot.errors.PluginError: if no zone_id is found. + """ + + zone_name_guesses = dns_common.base_domain_name_guesses(domain) + + for zone_name in zone_name_guesses: + params = {'name': zone_name, + 'per_page': 1} + + try: + zones = self.cf.zones.get(params=params) # zones | pylint: disable=no-member + except CloudFlare.exceptions.CloudFlareAPIError as e: + code = int(e) + hint = None + + if code == 6003: + hint = 'Did you copy your entire API key?' + elif code == 9103: + hint = 'Did you enter the correct email address?' + + raise errors.PluginError('Error determining zone_id: {0} {1}. Please confirm that ' + 'you have supplied valid Cloudflare API credentials.{2}' + .format(code, e, ' ({0})'.format(hint) if hint else '')) + + if len(zones) > 0: + zone_id = zones[0]['id'] + logger.debug('Found zone_id of %s for %s using name %s', zone_id, domain, zone_name) + return zone_id + + raise errors.PluginError('Unable to determine zone_id for {0} using zone names: {1}. ' + 'Please confirm that the domain name has been entered correctly ' + 'and is already associated with the supplied Cloudflare account.' + .format(domain, zone_name_guesses)) + + def _find_txt_record_id(self, zone_id, record_name, record_content): + """ + Find the record_id for a TXT record with the given name and content. + + :param str zone_id: The zone_id which contains the record. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :returns: The record_id, if found. + :rtype: str + """ + + params = {'type': 'TXT', + 'name': record_name, + 'content': record_content, + 'per_page': 1} + try: + # zones | pylint: disable=no-member + records = self.cf.zones.dns_records.get(zone_id, params=params) + except CloudFlare.exceptions.CloudFlareAPIError as e: + logger.debug('Encountered CloudFlareAPIError getting TXT record_id: %s', e) + records = [] + + if len(records) > 0: + # Cleanup is returning the system to the state we found it. If, for some reason, + # there are multiple matching records, we only delete one because we only added one. + return records[0]['id'] + else: + logger.debug('Unable to find TXT record.') diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare_test.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare_test.py new file mode 100644 index 000000000..e60d6ff8b --- /dev/null +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare_test.py @@ -0,0 +1,174 @@ +"""Tests for certbot_dns_cloudflare.dns_cloudflare.""" + +import os +import unittest + +import CloudFlare +import mock + +from certbot import errors +from certbot.plugins import dns_test_common +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +API_ERROR = CloudFlare.exceptions.CloudFlareAPIError(1000, '', '') +API_KEY = 'an-api-key' +EMAIL = 'example@example.com' + + +class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): + + def setUp(self): + from certbot_dns_cloudflare.dns_cloudflare import Authenticator + + super(AuthenticatorTest, self).setUp() + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"cloudflare_email": EMAIL, "cloudflare_api_key": API_KEY}, path) + + self.config = mock.MagicMock(cloudflare_credentials=path, + cloudflare_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "cloudflare") + + self.mock_client = mock.MagicMock() + # _get_cloudflare_client | pylint: disable=protected-access + self.auth._get_cloudflare_client = mock.MagicMock(return_value=self.mock_client) + + def test_perform(self): + self.auth.perform([self.achall]) + + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + def test_cleanup(self): + # _attempt_cleanup | pylint: disable=protected-access + self.auth._attempt_cleanup = True + self.auth.cleanup([self.achall]) + + expected = [mock.call.del_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + +class CloudflareClientTest(unittest.TestCase): + record_name = "foo" + record_content = "bar" + record_ttl = 42 + zone_id = 1 + record_id = 2 + + def setUp(self): + from certbot_dns_cloudflare.dns_cloudflare import _CloudflareClient + + self.cloudflare_client = _CloudflareClient(EMAIL, API_KEY) + + self.cf = mock.MagicMock() + self.cloudflare_client.cf = self.cf + + def test_add_txt_record(self): + self.cf.zones.get.return_value = [{'id': self.zone_id}] + + self.cloudflare_client.add_txt_record(DOMAIN, self.record_name, self.record_content, + self.record_ttl) + + self.cf.zones.dns_records.post.assert_called_with(self.zone_id, data=mock.ANY) + + post_data = self.cf.zones.dns_records.post.call_args[1]['data'] + + self.assertEqual('TXT', post_data['type']) + self.assertEqual(self.record_name, post_data['name']) + self.assertEqual(self.record_content, post_data['content']) + self.assertEqual(self.record_ttl, post_data['ttl']) + + def test_add_txt_record_error(self): + self.cf.zones.get.return_value = [{'id': self.zone_id}] + + self.cf.zones.dns_records.post.side_effect = API_ERROR + + self.assertRaises( + errors.PluginError, + self.cloudflare_client.add_txt_record, + DOMAIN, self.record_name, self.record_content, self.record_ttl) + + def test_add_txt_record_error_during_zone_lookup(self): + self.cf.zones.get.side_effect = API_ERROR + + self.assertRaises( + errors.PluginError, + self.cloudflare_client.add_txt_record, + DOMAIN, self.record_name, self.record_content, self.record_ttl) + + def test_add_txt_record_zone_not_found(self): + self.cf.zones.get.return_value = [] + + self.assertRaises( + errors.PluginError, + self.cloudflare_client.add_txt_record, + DOMAIN, self.record_name, self.record_content, self.record_ttl) + + def test_del_txt_record(self): + self.cf.zones.get.return_value = [{'id': self.zone_id}] + self.cf.zones.dns_records.get.return_value = [{'id': self.record_id}] + + self.cloudflare_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + expected = [mock.call.zones.get(params=mock.ANY), + mock.call.zones.dns_records.get(self.zone_id, params=mock.ANY), + mock.call.zones.dns_records.delete(self.zone_id, self.record_id)] + + self.assertEqual(expected, self.cf.mock_calls) + + get_data = self.cf.zones.dns_records.get.call_args[1]['params'] + + self.assertEqual('TXT', get_data['type']) + self.assertEqual(self.record_name, get_data['name']) + self.assertEqual(self.record_content, get_data['content']) + + def test_del_txt_record_error_during_zone_lookup(self): + self.cf.zones.get.side_effect = API_ERROR + + self.cloudflare_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_error_during_delete(self): + self.cf.zones.get.return_value = [{'id': self.zone_id}] + self.cf.zones.dns_records.get.return_value = [{'id': self.record_id}] + self.cf.zones.dns_records.delete.side_effect = API_ERROR + + self.cloudflare_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + expected = [mock.call.zones.get(params=mock.ANY), + mock.call.zones.dns_records.get(self.zone_id, params=mock.ANY), + mock.call.zones.dns_records.delete(self.zone_id, self.record_id)] + + self.assertEqual(expected, self.cf.mock_calls) + + def test_del_txt_record_error_during_get(self): + self.cf.zones.get.return_value = [{'id': self.zone_id}] + self.cf.zones.dns_records.get.side_effect = API_ERROR + + self.cloudflare_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + expected = [mock.call.zones.get(params=mock.ANY), + mock.call.zones.dns_records.get(self.zone_id, params=mock.ANY)] + + self.assertEqual(expected, self.cf.mock_calls) + + def test_del_txt_record_no_record(self): + self.cf.zones.get.return_value = [{'id': self.zone_id}] + self.cf.zones.dns_records.get.return_value = [] + + self.cloudflare_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + expected = [mock.call.zones.get(params=mock.ANY), + mock.call.zones.dns_records.get(self.zone_id, params=mock.ANY)] + + self.assertEqual(expected, self.cf.mock_calls) + + def test_del_txt_record_no_zone(self): + self.cf.zones.get.return_value = [{'id': None}] + + self.cloudflare_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + expected = [mock.call.zones.get(params=mock.ANY)] + + self.assertEqual(expected, self.cf.mock_calls) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-cloudflare/docs/.gitignore b/certbot-dns-cloudflare/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-cloudflare/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-cloudflare/docs/Makefile b/certbot-dns-cloudflare/docs/Makefile new file mode 100644 index 000000000..091abbfe7 --- /dev/null +++ b/certbot-dns-cloudflare/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-cloudflare +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/certbot-dns-cloudflare/docs/api.rst b/certbot-dns-cloudflare/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-cloudflare/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-cloudflare/docs/api/dns_cloudflare.rst b/certbot-dns-cloudflare/docs/api/dns_cloudflare.rst new file mode 100644 index 000000000..939d4c0b4 --- /dev/null +++ b/certbot-dns-cloudflare/docs/api/dns_cloudflare.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_cloudflare.dns_cloudflare` +-------------------------------------- + +.. automodule:: certbot_dns_cloudflare.dns_cloudflare + :members: diff --git a/certbot-dns-cloudflare/docs/conf.py b/certbot-dns-cloudflare/docs/conf.py new file mode 100644 index 000000000..aa7809246 --- /dev/null +++ b/certbot-dns-cloudflare/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-cloudflare documentation build configuration file, created by +# sphinx-quickstart on Tue May 9 10:20:04 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-cloudflare' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-cloudflaredoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-cloudflare.tex', u'certbot-dns-cloudflare Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-cloudflare', u'certbot-dns-cloudflare Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-cloudflare', u'certbot-dns-cloudflare Documentation', + author, 'certbot-dns-cloudflare', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-cloudflare/docs/index.rst b/certbot-dns-cloudflare/docs/index.rst new file mode 100644 index 000000000..e75106a01 --- /dev/null +++ b/certbot-dns-cloudflare/docs/index.rst @@ -0,0 +1,27 @@ +.. certbot-dns-cloudflare documentation master file, created by + sphinx-quickstart on Tue May 9 10:20:04 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-cloudflare's documentation! +================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_cloudflare + :members: + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-cloudflare/docs/make.bat b/certbot-dns-cloudflare/docs/make.bat new file mode 100644 index 000000000..88867c770 --- /dev/null +++ b/certbot-dns-cloudflare/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-cloudflare + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-cloudflare/setup.cfg b/certbot-dns-cloudflare/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-cloudflare/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-cloudflare/setup.py b/certbot-dns-cloudflare/setup.py new file mode 100644 index 000000000..26570a0ff --- /dev/null +++ b/certbot-dns-cloudflare/setup.py @@ -0,0 +1,69 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.15.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + 'cloudflare>=1.5.1', + 'mock', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-cloudflare', + version=version, + description="Cloudflare DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-cloudflare = certbot_dns_cloudflare.dns_cloudflare:Authenticator', + ], + }, + test_suite='certbot_dns_cloudflare', +) diff --git a/certbot/cli.py b/certbot/cli.py index 1845164b8..f802244ae 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1221,6 +1221,8 @@ def _plugins_parsing(helpful, plugins): help='Provide laborious manual instructions for obtaining a cert') helpful.add(["plugins", "certonly"], "--webroot", action="store_true", help='Obtain certs by placing files in a webroot directory.') + helpful.add(["plugins", "certonly"], "--dns-cloudflare", action="store_true", + help='Obtain certs using a DNS TXT record (if you are using Cloudflare for DNS).') # things should not be reorder past/pre this comment: # plugins_group should be displayed in --help before plugin diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 6bf4bd369..5d1f9cdd1 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -28,6 +28,7 @@ class PluginEntryPoint(object): PREFIX_FREE_DISTRIBUTIONS = [ "certbot", "certbot-apache", + "certbot-dns-cloudflare", "certbot-nginx", ] """Distributions for which prefix will be omitted.""" diff --git a/certbot/plugins/dns_common.py b/certbot/plugins/dns_common.py new file mode 100644 index 000000000..de8f695c7 --- /dev/null +++ b/certbot/plugins/dns_common.py @@ -0,0 +1,323 @@ +"""Common code for DNS Authenticator Plugins.""" + +import abc +import logging +import os +import stat +from time import sleep + +import configobj +import zope.interface +from acme import challenges + +from certbot import errors +from certbot import interfaces +from certbot.display import ops +from certbot.display import util as display_util +from certbot.plugins import common + +logger = logging.getLogger(__name__) + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class DNSAuthenticator(common.Plugin): + """Base class for DNS Authenticators""" + + def __init__(self, config, name): + super(DNSAuthenticator, self).__init__(config, name) + + self._attempt_cleanup = False + + @classmethod + def add_parser_arguments(cls, add): + add('propagation-seconds', + default=10, + type=int, + help='The number of seconds to wait for DNS to propagate before asking the ACME server ' + 'to verify the DNS record.') + + def get_chall_pref(self, unused_domain): # pylint: disable=missing-docstring,no-self-use + return [challenges.DNS01] + + def prepare(self): # pylint: disable=missing-docstring + pass + + def perform(self, achalls): # pylint: disable=missing-docstring + self._setup_credentials() + + self._attempt_cleanup = True + + responses = [] + for achall in achalls: + domain = achall.domain + validation_domain_name = achall.validation_domain_name(domain) + validation = achall.validation(achall.account_key) + + self._perform(domain, validation_domain_name, validation) + responses.append(achall.response(achall.account_key)) + + # DNS updates take time to propagate and checking to see if the update has occurred is not + # reliable (the machine this code is running on might be able to see an update before + # the ACME server). So: we sleep for a short amount of time we believe to be long enough. + logger.info("Waiting %d seconds for DNS changes to propagate", + self.conf('propagation-seconds')) + sleep(self.conf('propagation-seconds')) + + return responses + + def cleanup(self, achalls): # pylint: disable=missing-docstring + if self._attempt_cleanup: + for achall in achalls: + domain = achall.domain + validation_domain_name = achall.validation_domain_name(domain) + validation = achall.validation(achall.account_key) + + self._cleanup(domain, validation_domain_name, validation) + + @abc.abstractmethod + def _setup_credentials(self): # pragma: no cover + """ + Establish credentials, prompting if necessary. + """ + raise NotImplementedError() + + @abc.abstractmethod + def _perform(self, domain, validation_domain_name, validation): # pragma: no cover + """ + Performs a dns-01 challenge by creating a DNS TXT record. + + :param str domain: The domain being validated. + :param str validation_domain_name: The validation record domain name. + :param str validation: The validation record content. + :raises errors.PluginError: If the challenge cannot be performed + """ + raise NotImplementedError() + + @abc.abstractmethod + def _cleanup(self, domain, validation_domain_name, validation): # pragma: no cover + """ + Deletes the DNS TXT record which would have been created by `_perform_achall`. + + Fails gracefully if no such record exists. + + :param str domain: The domain being validated. + :param str validation_domain_name: The validation record domain name. + :param str validation: The validation record content. + """ + raise NotImplementedError() + + def _configure(self, key, label): + """ + Ensure that a configuration value is available. + + If necessary, prompts the user and stores the result. + + :param str key: The configuration key. + :param str label: The user-friendly label for this piece of information. + """ + + configured_value = self.conf(key) + if not configured_value: + new_value = self._prompt_for_data(label) + + setattr(self.config, self.dest(key), new_value) + + def _configure_file(self, key, label, validator=None): + """ + Ensure that a configuration value is available for a path. + + If necessary, prompts the user and stores the result. + + :param str key: The configuration key. + :param str label: The user-friendly label for this piece of information. + """ + + configured_value = self.conf(key) + if not configured_value: + new_value = self._prompt_for_file(label, validator) + + setattr(self.config, self.dest(key), os.path.abspath(os.path.expanduser(new_value))) + + def _configure_credentials(self, key, label, required_variables=None): + """ + As `_configure_file`, but for a credential configuration file. + + If necessary, prompts the user and stores the result. + + Always stores absolute paths to avoid issues during renewal. + + :param str key: The configuration key. + :param str label: The user-friendly label for this piece of information. + :param dict required_variables: Map of variable which must be present to error to display. + """ + + def __validator(filename): + if required_variables: + CredentialsConfiguration(filename, self.dest).require(required_variables) + + self._configure_file(key, label, __validator) + + credentials_configuration = CredentialsConfiguration(self.conf(key), self.dest) + if required_variables: + credentials_configuration.require(required_variables) + + return credentials_configuration + + @staticmethod + def _prompt_for_data(label): + """ + Prompt the user for a piece of information. + + :param str label: The user-friendly label for this piece of information. + :returns: The user's response (guaranteed non-empty). + :rtype: str + """ + + def __validator(i): + if not i: + raise errors.PluginError('Please enter your {0}.'.format(label)) + + code, response = ops.validated_input( + __validator, + 'Input your {0}'.format(label), + force_interactive=True) + + if code == display_util.OK: + return response + else: + raise errors.PluginError('{0} required to proceed.'.format(label)) + + @staticmethod + def _prompt_for_file(label, validator=None): + """ + Prompt the user for a path. + + :param str label: The user-friendly label for the file. + :param callable validator: A method which will be called to validate the supplied input + after it has been validated to be a non-empty path to an existing file. Should throw a + `~certbot.errors.PluginError` to indicate any issue. + :returns: The user's response (guaranteed to exist). + :rtype: str + """ + + def __validator(filename): + if not filename: + raise errors.PluginError('Please enter a valid path to your {0}.'.format(label)) + + filename = os.path.expanduser(filename) + + validate_file(filename) + + if validator: + validator(filename) + + code, response = ops.validated_directory( + __validator, + 'Input the path to your {0}'.format(label), + force_interactive=True) + + if code == display_util.OK: + return response + else: + raise errors.PluginError('{0} required to proceed.'.format(label)) + + +class CredentialsConfiguration(object): + """Represents a user-supplied filed which stores API credentials.""" + + def __init__(self, filename, mapper=lambda x: x): + """ + :param str filename: A path to the configuration file. + :param callable mapper: A transformation to apply to configuration key names + :raises errors.PluginError: If the file does not exist or is not a valid format. + """ + validate_file_permissions(filename) + + try: + self.confobj = configobj.ConfigObj(filename) + except configobj.ConfigObjError as e: + logger.debug("Error parsing credentials configuration: %s", e, exc_info=True) + raise errors.PluginError("Error parsing credentials configuration: {0}".format(e)) + + self.mapper = mapper + + def require(self, required_variables): + """Ensures that the supplied set of variables are all present in the file. + + :param dict required_variables: Map of variable which must be present to error to display. + :raises errors.PluginError: If one or more are missing. + """ + messages = [] + + for var in required_variables: + if not self._has(var): + messages.append('Property "{0}" not found (should be {1}).' + .format(self.mapper(var), required_variables[var])) + elif not self._get(var): + messages.append('Property "{0}" not set (should be {1}).' + .format(self.mapper(var), required_variables[var])) + + if messages: + raise errors.PluginError( + 'Missing {0} in credentials configuration file {1}:\n * {2}'.format( + 'property' if len(messages) == 1 else 'properties', + self.confobj.filename, + '\n * '.join(messages) + ) + ) + + def conf(self, var): + """Find a configuration value for variable `var`, as transformed by `mapper`. + + :param str var: The variable to get. + :returns: The value of the variable. + :rtype: str + """ + + return self._get(var) + + def _has(self, var): + return self.mapper(var) in self.confobj + + def _get(self, var): + return self.confobj.get(self.mapper(var)) + + +def validate_file(filename): + """Ensure that the specified file exists.""" + + if not os.path.exists(filename): + raise errors.PluginError('File not found: {0}'.format(filename)) + + if not os.path.isfile(filename): + raise errors.PluginError('Path is not a file: {0}'.format(filename)) + + +def validate_file_permissions(filename): + """Ensure that the specified file exists and warn about unsafe permissions.""" + + validate_file(filename) + + permissions = stat.S_IMODE(os.stat(filename).st_mode) + if permissions & stat.S_IRWXO: + logger.warning('Unsafe permissions on credentials configuration file: %s', filename) + + +def base_domain_name_guesses(domain): + """Return a list of progressively less-specific domain names. + + One of these will probably be the domain name known to the DNS provider. + + :Example: + + >>> base_domain_name_guesses('foo.bar.baz.example.com') + ['foo.bar.baz.example.com', 'bar.baz.example.com', 'baz.example.com', 'example.com', 'com'] + + :param str domain: The domain for which to return guesses. + :returns: The a list of less specific domain names. + :rtype: list + """ + + fragments = domain.split('.') + return ['.'.join(fragments[i:]) for i in range(0, len(fragments))] diff --git a/certbot/plugins/dns_common_test.py b/certbot/plugins/dns_common_test.py new file mode 100644 index 000000000..9b0f0c875 --- /dev/null +++ b/certbot/plugins/dns_common_test.py @@ -0,0 +1,233 @@ +"""Tests for certbot.plugins.dns_common.""" + +import collections +import logging +import os +import unittest + +import mock + +from certbot import errors +from certbot.display import util as display_util +from certbot.plugins import dns_common +from certbot.plugins import dns_test_common +from certbot.tests import util + + +class DNSAuthenticatorTest(util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): + # pylint: disable=protected-access + + class _FakeDNSAuthenticator(dns_common.DNSAuthenticator): + _setup_credentials = mock.MagicMock() + _perform = mock.MagicMock() + _cleanup = mock.MagicMock() + + def __init__(self, *args, **kwargs): + # pylint: disable=protected-access + super(DNSAuthenticatorTest._FakeDNSAuthenticator, self).__init__(*args, **kwargs) + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'A fake authenticator for testing.' + + class _FakeConfig(object): + fake_propagation_seconds = 0 + fake_config_key = 1 + fake_other_key = None + fake_file_path = None + + def setUp(self): + super(DNSAuthenticatorTest, self).setUp() + + self.config = DNSAuthenticatorTest._FakeConfig() + + self.auth = DNSAuthenticatorTest._FakeDNSAuthenticator(self.config, "fake") + + def test_perform(self): + self.auth.perform([self.achall]) + + self.auth._perform.assert_called_once_with(dns_test_common.DOMAIN, mock.ANY, mock.ANY) + + def test_cleanup(self): + self.auth._attempt_cleanup = True + + self.auth.cleanup([self.achall]) + + self.auth._cleanup.assert_called_once_with(dns_test_common.DOMAIN, mock.ANY, mock.ANY) + + @util.patch_get_utility() + def test_prompt(self, mock_get_utility): + mock_display = mock_get_utility() + mock_display.input.side_effect = ((display_util.OK, "",), + (display_util.OK, "value",)) + + self.auth._configure("other_key", "") + self.assertEqual(self.auth.config.fake_other_key, "value") + + @util.patch_get_utility() + def test_prompt_canceled(self, mock_get_utility): + mock_display = mock_get_utility() + mock_display.input.side_effect = ((display_util.CANCEL, "c",),) + + self.assertRaises(errors.PluginError, self.auth._configure, "other_key", "") + + @util.patch_get_utility() + def test_prompt_file(self, mock_get_utility): + path = os.path.join(self.tempdir, 'file.ini') + open(path, "wb").close() + + mock_display = mock_get_utility() + mock_display.directory_select.side_effect = ((display_util.OK, "",), + (display_util.OK, "not-a-file.ini",), + (display_util.OK, self.tempdir), + (display_util.OK, path,)) + + self.auth._configure_file("file_path", "") + self.assertEqual(self.auth.config.fake_file_path, path) + + @util.patch_get_utility() + def test_prompt_file_canceled(self, mock_get_utility): + mock_display = mock_get_utility() + mock_display.directory_select.side_effect = ((display_util.CANCEL, "c",),) + + self.assertRaises(errors.PluginError, self.auth._configure_file, "file_path", "") + + def test_configure_credentials(self): + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"fake_test": "value"}, path) + setattr(self.config, "fake_credentials", path) + + credentials = self.auth._configure_credentials("credentials", "", {"test": ""}) + + self.assertEqual(credentials.conf("test"), "value") + + @util.patch_get_utility() + def test_prompt_credentials(self, mock_get_utility): + bad_path = os.path.join(self.tempdir, 'bad-file.ini') + dns_test_common.write({"fake_other": "other_value"}, bad_path) + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"fake_test": "value"}, path) + setattr(self.config, "fake_credentials", "") + + mock_display = mock_get_utility() + mock_display.directory_select.side_effect = ((display_util.OK, "",), + (display_util.OK, "not-a-file.ini",), + (display_util.OK, self.tempdir), + (display_util.OK, bad_path), + (display_util.OK, path,)) + + credentials = self.auth._configure_credentials("credentials", "", {"test": ""}) + self.assertEqual(credentials.conf("test"), "value") + + +class CredentialsConfigurationTest(util.TempDirTestCase): + class _MockLoggingHandler(logging.Handler): + messages = None + + def __init__(self, *args, **kwargs): + self.reset() + logging.Handler.__init__(self, *args, **kwargs) + + def emit(self, record): + self.messages[record.levelname.lower()].append(record.getMessage()) + + def reset(self): + """Allows the handler to be reset between tests.""" + self.messages = collections.defaultdict(list) + + def test_valid_file(self): + path = os.path.join(self.tempdir, 'too-permissive-file.ini') + + dns_test_common.write({"test": "value", "other": 1}, path) + + credentials_configuration = dns_common.CredentialsConfiguration(path) + self.assertEqual("value", credentials_configuration.conf("test")) + self.assertEqual("1", credentials_configuration.conf("other")) + + def test_nonexistent_file(self): + path = os.path.join(self.tempdir, 'not-a-file.ini') + + self.assertRaises(errors.PluginError, dns_common.CredentialsConfiguration, path) + + def test_valid_file_with_unsafe_permissions(self): + log = self._MockLoggingHandler() + dns_common.logger.addHandler(log) + + path = os.path.join(self.tempdir, 'too-permissive-file.ini') + open(path, "wb").close() + + dns_common.CredentialsConfiguration(path) + + self.assertEqual(1, len([_ for _ in log.messages['warning'] if _.startswith("Unsafe")])) + + +class CredentialsConfigurationRequireTest(util.TempDirTestCase): + + def setUp(self): + super(CredentialsConfigurationRequireTest, self).setUp() + + self.path = os.path.join(self.tempdir, 'file.ini') + + def _write(self, values): + dns_test_common.write(values, self.path) + + def test_valid(self): + self._write({"test": "value", "other": 1}) + + credentials_configuration = dns_common.CredentialsConfiguration(self.path) + credentials_configuration.require({"test": "", "other": ""}) + + def test_valid_but_extra(self): + self._write({"test": "value", "other": 1}) + + credentials_configuration = dns_common.CredentialsConfiguration(self.path) + credentials_configuration.require({"test": ""}) + + def test_valid_empty(self): + self._write({}) + + credentials_configuration = dns_common.CredentialsConfiguration(self.path) + credentials_configuration.require({}) + + def test_missing(self): + self._write({}) + + credentials_configuration = dns_common.CredentialsConfiguration(self.path) + self.assertRaises(errors.PluginError, credentials_configuration.require, {"test": ""}) + + def test_blank(self): + self._write({"test": ""}) + + credentials_configuration = dns_common.CredentialsConfiguration(self.path) + self.assertRaises(errors.PluginError, credentials_configuration.require, {"test": ""}) + + def test_typo(self): + self._write({"tets": "typo!"}) + + credentials_configuration = dns_common.CredentialsConfiguration(self.path) + self.assertRaises(errors.PluginError, credentials_configuration.require, {"test": ""}) + + +class DomainNameGuessTest(unittest.TestCase): + + def test_simple_case(self): + self.assertTrue( + 'example.com' in + dns_common.base_domain_name_guesses("example.com") + ) + + def test_sub_domain(self): + self.assertTrue( + 'example.com' in + dns_common.base_domain_name_guesses("foo.bar.baz.example.com") + ) + + def test_second_level_domain(self): + self.assertTrue( + 'example.co.uk' in + dns_common.base_domain_name_guesses("foo.bar.baz.example.co.uk") + ) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot/plugins/dns_test_common.py b/certbot/plugins/dns_test_common.py new file mode 100644 index 000000000..d8cd29404 --- /dev/null +++ b/certbot/plugins/dns_test_common.py @@ -0,0 +1,63 @@ +"""Base test class for DNS authenticators.""" + +import os + +import configobj +import mock +import six +from acme import challenges +from acme import jose + +from certbot import achallenges +from certbot.tests import acme_util +from certbot.tests import util as test_util + +DOMAIN = 'example.com' +KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + + +class BaseAuthenticatorTest(object): + """ + A base test class to reduce duplication between test code for DNS Authenticator Plugins. + + Assumes: + * That subclasses also subclass unittest.TestCase + * That the authenticator is stored as self.auth + """ + + achall = achallenges.KeyAuthorizationAnnotatedChallenge( + challb=acme_util.DNS01, domain=DOMAIN, account_key=KEY) + + def test_more_info(self): + # pylint: disable=no-member + self.assertTrue(isinstance(self.auth.more_info(), six.string_types)) + + def test_get_chall_pref(self): + # pylint: disable=no-member + self.assertEqual(self.auth.get_chall_pref(None), [challenges.DNS01]) + + def test_parser_arguments(self): + m = mock.MagicMock() + + # pylint: disable=no-member + self.auth.add_parser_arguments(m) + + m.assert_any_call('propagation-seconds', type=int, default=mock.ANY, help=mock.ANY) + + +def write(values, path): + """Write the specified values to a config file. + + :param dict values: A map of values to write. + :param str path: Where to write the values. + """ + + config = configobj.ConfigObj() + + for key in values: + config[key] = values[key] + + with open(path, "wb") as f: + config.write(outfile=f) + + os.chmod(path, 0o600) diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index d138001e6..29ab97a8d 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -133,7 +133,7 @@ def choose_plugin(prepared, question): else: return None -noninstaller_plugins = ["webroot", "manual", "standalone"] +noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -237,6 +237,8 @@ def cli_plugin_requests(config): req_auth = set_configurator(req_auth, "webroot") if config.manual: req_auth = set_configurator(req_auth, "manual") + if config.dns_cloudflare: + req_auth = set_configurator(req_auth, "dns-cloudflare") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) return req_auth, req_inst diff --git a/docs/api/plugins/dns_common.rst b/docs/api/plugins/dns_common.rst new file mode 100644 index 000000000..ee3945e74 --- /dev/null +++ b/docs/api/plugins/dns_common.rst @@ -0,0 +1,5 @@ +:mod:`certbot.plugins.dns_common` +--------------------------------- + +.. automodule:: certbot.plugins.dns_common + :members: diff --git a/tools/venv.sh b/tools/venv.sh index c9d8fdb9d..2b5f4272c 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -14,6 +14,7 @@ fi -e acme[dev] \ -e .[dev,docs] \ -e certbot-apache \ + -e certbot-dns-cloudflare \ -e certbot-nginx \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tools/venv3.sh b/tools/venv3.sh index 08358aa48..2063d99d8 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -13,6 +13,7 @@ fi -e acme[dev] \ -e .[dev,docs] \ -e certbot-apache \ + -e certbot-dns-cloudflare \ -e certbot-nginx \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tox.cover.sh b/tox.cover.sh index 7243c4708..81bf19d5a 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_nginx letshelp_certbot" else pkgs="$@" fi @@ -21,6 +21,8 @@ cover () { min=100 elif [ "$1" = "certbot_apache" ]; then min=100 + elif [ "$1" = "certbot_dns_cloudflare" ]; then + min=98 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "letshelp_certbot" ]; then diff --git a/tox.ini b/tox.ini index b073f03b8..a956b19a9 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,12 @@ plugin_commands = plugin_install_args = -e certbot-apache -e certbot-nginx plugin_paths = certbot-apache/certbot_apache certbot-nginx/certbot_nginx +dns_plugin_commands = + pip install -e certbot-dns-cloudflare + nosetests -v certbot_dns_cloudflare --processes=-1 +dns_plugin_install_args = -e certbot-dns-cloudflare +dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare + compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test @@ -63,12 +69,12 @@ deps = [testenv:py27_install] basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]other_install_args} + pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} [testenv:cover] basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]other_install_args} + pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} ./tox.cover.sh [testenv:lint] @@ -78,15 +84,15 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} - pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]compatibility_paths} {[base]other_paths} + pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:mypy] basepython = python3.4 commands = pip install mypy - pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} - mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]compatibility_paths} {[base]other_paths} + pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:apacheconftest] #basepython = python2.7 From 7955274126fca72a97be44e6cec295229c962bbe Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Wed, 10 May 2017 10:00:00 -0700 Subject: [PATCH 63/96] Script to create docs directory for new packages. --- tools/sphinx-quickstart.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 tools/sphinx-quickstart.sh diff --git a/tools/sphinx-quickstart.sh b/tools/sphinx-quickstart.sh new file mode 100755 index 000000000..d67c45b6f --- /dev/null +++ b/tools/sphinx-quickstart.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo "Usage: $(basename $0) project-name" + exit 1 +fi + +PROJECT=$1 + +yes "n" | sphinx-quickstart --dot _ --project $PROJECT --author "Certbot Project" -v 0 --release 0 --language en --suffix .rst --master index --ext-autodoc --ext-intersphinx --ext-todo --ext-coverage --ext-viewcode --makefile --batchfile $PROJECT/docs + +cd $PROJECT/docs +sed -i -e "s|\# import os|import os|" conf.py +sed -i -e "s|\# needs_sphinx = '1.0'|needs_sphinx = '1.0'|" conf.py +sed -i -e "s|intersphinx_mapping = {'https://docs.python.org/': None}|intersphinx_mapping = {\n 'python': ('https://docs.python.org/', None),\n 'acme': ('https://acme-python.readthedocs.org/en/latest/', None),\n 'certbot': ('https://certbot.eff.org/docs/', None),\n}|" conf.py +sed -i -e "s|html_theme = 'alabaster'|\n# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs\n# on_rtd is whether we are on readthedocs.org\non_rtd = os.environ.get('READTHEDOCS', None) == 'True'\nif not on_rtd: # only import and set the theme if we're building docs locally\n import sphinx_rtd_theme\n html_theme = 'sphinx_rtd_theme'\n html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]\n# otherwise, readthedocs.org uses their theme by default, so no need to specify it|" conf.py +sed -i -e "s|# Add any paths that contain templates here, relative to this directory.|autodoc_member_order = 'bysource'\nautodoc_default_flags = ['show-inheritance', 'private-members']\n\n# Add any paths that contain templates here, relative to this directory.|" conf.py +sed -i -e "s|# The name of the Pygments (syntax highlighting) style to use.|default_role = 'py:obj'\n\n# The name of the Pygments (syntax highlighting) style to use.|" conf.py +echo "/_build/" >> .gitignore +echo "================= +API Documentation +================= + +.. toctree:: + :glob: + + api/**" > api.rst +sed -i -e "s| :caption: Contents:| :caption: Contents:\n\n.. toctree::\n :maxdepth: 1\n\n api\n\n.. automodule:: ${PROJECT//-/_}\n :members:|" index.rst + +echo "Suggested next steps: +* Add API docs to: $PROJECT/docs/api/ +* Run: git add $PROJECT/docs" From be7e99a46193e2b9305fe0f679b4edd178954d6b Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 11 May 2017 10:06:05 -0700 Subject: [PATCH 64/96] Pin dependency versions when using tools/venv.sh (#4629) * Revert "Pin python-augeas version to avoid error with 1.0.0 (#4422)" This reverts commit 1c51ae25887f2dc3168a38d1f0042363cd7ac1e3. * make dependency-requirements * separate certbot and dependency requirements * fix build.py * update hashin comment * simplify release pinning * separate letsencrypt dependency * pin hashes in venv * error out when bad things happen * use pinned dependencies in tox * Revert "pin hashes in venv" This reverts commit 1cd38a9e50da94e6959728bf57aaf642807bc9c7. * use pip_install.sh in venv_common * quote pip install args * bump mock version --- certbot-apache/setup.py | 2 +- letsencrypt-auto-source/build.py | 7 ++-- letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++------ .../letsencrypt-auto.template | 4 ++- .../pieces/certbot-requirements.txt | 12 +++++++ ...ements.txt => dependency-requirements.txt} | 32 ++++--------------- .../pieces/letsencrypt-requirements.txt | 10 ++++++ tools/_venv_common.sh | 2 +- tools/pip_install.sh | 13 ++++++++ tools/release.sh | 9 ++---- tox.ini | 26 ++++++++------- 11 files changed, 83 insertions(+), 58 deletions(-) create mode 100644 letsencrypt-auto-source/pieces/certbot-requirements.txt rename letsencrypt-auto-source/pieces/{letsencrypt-auto-requirements.txt => dependency-requirements.txt} (88%) create mode 100644 letsencrypt-auto-source/pieces/letsencrypt-requirements.txt create mode 100755 tools/pip_install.sh diff --git a/certbot-apache/setup.py b/certbot-apache/setup.py index 9429ada2e..ccd7cbc2a 100644 --- a/certbot-apache/setup.py +++ b/certbot-apache/setup.py @@ -11,7 +11,7 @@ install_requires = [ 'acme=={0}'.format(version), 'certbot=={0}'.format(version), 'mock', - 'python-augeas<=0.5.0', + 'python-augeas', # For pkg_resources. >=1.0 so pip resolves it to a version cryptography # will tolerate; see #2599: 'setuptools>=1.0', diff --git a/letsencrypt-auto-source/build.py b/letsencrypt-auto-source/build.py index eebad61b7..a1e40fe44 100755 --- a/letsencrypt-auto-source/build.py +++ b/letsencrypt-auto-source/build.py @@ -21,14 +21,17 @@ def build(version=None, requirements=None): :arg version: The version to attach to the script. Default: the version of the certbot package :arg requirements: The contents of the requirements file to embed. Default: - contents of letsencrypt-auto-requirements.txt + contents of dependency-requirements.txt, letsencrypt-requirements.txt, + and certbot-requirements.txt """ special_replacements = { 'LE_AUTO_VERSION': version or certbot_version(DIR) } if requirements: - special_replacements['letsencrypt-auto-requirements.txt'] = requirements + special_replacements['dependency-requirements.txt'] = '' + special_replacements['letsencrypt-requirements.txt'] = '' + special_replacements['certbot-requirements.txt'] = requirements def replacer(match): token = match.group(1) diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index d4433d525..7e4a7554f 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -694,7 +694,7 @@ if [ "$1" = "--le-auto-phase2" ]; then # Hashin example: # pip install hashin -# hashin -r letsencrypt-auto-requirements.txt cryptography==1.5.2 +# hashin -r dependency-requirements.txt cryptography==1.5.2 # sets the new certbot-auto pinned version of cryptography to 1.5.2 argparse==1.4.0 \ @@ -754,9 +754,9 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 -funcsigs==0.4 \ - --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ - --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 +funcsigs==1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ + --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 idna==2.0 \ --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \ --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b @@ -851,15 +851,21 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==1.0.1 \ - --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ - --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc +mock==2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ + --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba + +# Contains the requirements for the letsencrypt package. +# +# Since the letsencrypt package depends on certbot and using pip with hashes +# requires that all installed packages have hashes listed, this allows +# dependency-requirements.txt to be used without requiring a hash for a +# (potentially unreleased) Certbot package. + letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. - acme==0.14.0 \ --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 diff --git a/letsencrypt-auto-source/letsencrypt-auto.template b/letsencrypt-auto-source/letsencrypt-auto.template index f6585a378..305435a9b 100755 --- a/letsencrypt-auto-source/letsencrypt-auto.template +++ b/letsencrypt-auto-source/letsencrypt-auto.template @@ -317,7 +317,9 @@ if [ "$1" = "--le-auto-phase2" ]; then # There is no $ interpolation due to quotes on starting heredoc delimiter. # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/letsencrypt-auto-requirements.txt" -{{ letsencrypt-auto-requirements.txt }} +{{ dependency-requirements.txt }} +{{ letsencrypt-requirements.txt }} +{{ certbot-requirements.txt }} UNLIKELY_EOF # ------------------------------------------------------------------------- cat << "UNLIKELY_EOF" > "$TEMP_DIR/pipstrap.py" diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt new file mode 100644 index 000000000..a49abeefd --- /dev/null +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -0,0 +1,12 @@ +acme==0.14.0 \ + --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ + --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 +certbot==0.14.0 \ + --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ + --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 +certbot-apache==0.14.0 \ + --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ + --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f +certbot-nginx==0.14.0 \ + --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ + --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db diff --git a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt b/letsencrypt-auto-source/pieces/dependency-requirements.txt similarity index 88% rename from letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt rename to letsencrypt-auto-source/pieces/dependency-requirements.txt index 5db66fe88..4cba83195 100644 --- a/letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt +++ b/letsencrypt-auto-source/pieces/dependency-requirements.txt @@ -5,7 +5,7 @@ # Hashin example: # pip install hashin -# hashin -r letsencrypt-auto-requirements.txt cryptography==1.5.2 +# hashin -r dependency-requirements.txt cryptography==1.5.2 # sets the new certbot-auto pinned version of cryptography to 1.5.2 argparse==1.4.0 \ @@ -65,9 +65,9 @@ cryptography==1.5.3 \ enum34==1.1.2 \ --hash=sha256:2475d7fcddf5951e92ff546972758802de5260bf409319a9f1934e6bbc8b1dc7 \ --hash=sha256:35907defb0f992b75ab7788f65fedc1cf20ffa22688e0e6f6f12afc06b3ea501 -funcsigs==0.4 \ - --hash=sha256:ff5ad9e2f8d9e5d1e8bbfbcf47722ab527cf0d51caeeed9da6d0f40799383fde \ - --hash=sha256:d83ce6df0b0ea6618700fe1db353526391a8a3ada1b7aba52fed7a61da772033 +funcsigs==1.0.2 \ + --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \ + --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 idna==2.0 \ --hash=sha256:9b2fc50bd3c4ba306b9651b69411ef22026d4d8335b93afc2214cef1246ce707 \ --hash=sha256:16199aad938b290f5be1057c0e1efc6546229391c23cea61ca940c115f7d3d3b @@ -162,24 +162,6 @@ zope.interface==4.1.3 \ --hash=sha256:928138365245a0e8869a5999fbcc2a45475a0a6ed52a494d60dbdc540335fedd \ --hash=sha256:0d841ba1bb840eea0e6489dc5ecafa6125554971f53b5acb87764441e61bceba \ --hash=sha256:b09c8c1d47b3531c400e0195697f1414a63221de6ef478598a4f1460f7d9a392 -mock==1.0.1 \ - --hash=sha256:b839dd2d9c117c701430c149956918a423a9863b48b09c90e30a6013e7d2f44f \ - --hash=sha256:8f83080daa249d036cbccfb8ae5cc6ff007b88d6d937521371afabe7b19badbc -letsencrypt==0.7.0 \ - --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ - --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 - -# THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. - -acme==0.14.0 \ - --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ - --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 -certbot==0.14.0 \ - --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ - --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 -certbot-apache==0.14.0 \ - --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ - --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f -certbot-nginx==0.14.0 \ - --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ - --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db +mock==2.0.0 \ + --hash=sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1 \ + --hash=sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba diff --git a/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt new file mode 100644 index 000000000..8e745c9cd --- /dev/null +++ b/letsencrypt-auto-source/pieces/letsencrypt-requirements.txt @@ -0,0 +1,10 @@ +# Contains the requirements for the letsencrypt package. +# +# Since the letsencrypt package depends on certbot and using pip with hashes +# requires that all installed packages have hashes listed, this allows +# dependency-requirements.txt to be used without requiring a hash for a +# (potentially unreleased) Certbot package. + +letsencrypt==0.7.0 \ + --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ + --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 diff --git a/tools/_venv_common.sh b/tools/_venv_common.sh index ddbb02c62..20ed4c034 100755 --- a/tools/_venv_common.sh +++ b/tools/_venv_common.sh @@ -19,7 +19,7 @@ virtualenv --no-site-packages --setuptools $VENV_NAME $VENV_ARGS # invocations use latest pip install -U pip pip install -U setuptools -pip install "$@" +./tools/pip_install.sh "$@" set +x echo "Please run the following command to activate developer environment:" diff --git a/tools/pip_install.sh b/tools/pip_install.sh new file mode 100755 index 000000000..8a58f9e48 --- /dev/null +++ b/tools/pip_install.sh @@ -0,0 +1,13 @@ +#!/bin/sh -e +# pip installs packages using Certbot's requirements file as constraints + +# get the root of the Certbot repo +repo_root=$(git rev-parse --show-toplevel) +requirements="$repo_root/letsencrypt-auto-source/pieces/dependency-requirements.txt" +constraints=$(mktemp) +trap "rm -f $constraints" EXIT +# extract pinned requirements without hashes +sed -n -e 's/^\([^[:space:]]*==[^[:space:]]*\).*$/\1/p' $requirements > $constraints + +# install the requested packages using the pinned requirements as constraints +pip install --constraint $constraints "$@" diff --git a/tools/release.sh b/tools/release.sh index 1da11fe2c..3b1d4d8a6 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -170,19 +170,14 @@ cd ~- for pkg in acme certbot certbot-apache certbot-nginx ; do echo $pkg==$version \\ pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' -done > /tmp/hashes.$$ +done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate -if ! wc -l /tmp/hashes.$$ | grep -qE "^\s*12 " ; then +if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*12 " ; then echo Unexpected pip hash output exit 1 fi -# perform hideous surgery on requirements.txt... -head -n -12 letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt > /tmp/req.$$ -cat /tmp/hashes.$$ >> /tmp/req.$$ -cp /tmp/req.$$ letsencrypt-auto-source/pieces/letsencrypt-auto-requirements.txt - # ensure we have the latest built version of leauto letsencrypt-auto-source/build.py diff --git a/tox.ini b/tox.ini index a956b19a9..771b92d11 100644 --- a/tox.ini +++ b/tox.ini @@ -10,20 +10,22 @@ envlist = modification,py{26,33,34,35,36},cover,lint # loops, especially on Travis [base] +# wraps pip install to use pinned versions of dependencies +pip_install = {toxinidir}/tools/pip_install.sh # packages installed separately to ensure that downstream deps problems # are detected, c.f. #1002 core_commands = - pip install -e acme[dev] + {[base]pip_install} -e acme[dev] nosetests -v acme --processes=-1 - pip install -e .[dev] + {[base]pip_install} -e .[dev] nosetests -v certbot --processes=-1 --process-timeout=100 core_install_args = -e acme[dev] -e .[dev] core_paths = acme/acme certbot plugin_commands = - pip install -e certbot-apache + {[base]pip_install} -e certbot-apache nosetests -v certbot_apache --processes=-1 --process-timeout=80 - pip install -e certbot-nginx + {[base]pip_install} -e certbot-nginx nosetests -v certbot_nginx --processes=-1 plugin_install_args = -e certbot-apache -e certbot-nginx plugin_paths = certbot-apache/certbot_apache certbot-nginx/certbot_nginx @@ -38,7 +40,7 @@ compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test other_commands = - pip install -e letshelp-certbot + {[base]pip_install} -e letshelp-certbot nosetests -v letshelp_certbot --processes=-1 python tests/lock_test.py other_install_args = -e letshelp-certbot @@ -69,12 +71,12 @@ deps = [testenv:py27_install] basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} [testenv:cover] basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} ./tox.cover.sh [testenv:lint] @@ -84,25 +86,25 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:mypy] basepython = python3.4 commands = - pip install mypy - pip install -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + {[base]pip_install} mypy + {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:apacheconftest] #basepython = python2.7 commands = - pip install {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} {toxinidir}/certbot-apache/certbot_apache/tests/apache-conf-files/apache-conf-test --debian-modules [testenv:nginxroundtrip] commands = - pip install {[base]core_install_args} -e certbot-nginx + {[base]pip_install} {[base]core_install_args} -e certbot-nginx python certbot-compatibility-test/nginx/roundtrip.py certbot-compatibility-test/nginx/nginx-roundtrip-testdata # This is a duplication of the command line in testenv:le_auto to From 74c7ffe25ecb51d276f0d5f33cdd1192321781a1 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 11 May 2017 12:11:04 -0700 Subject: [PATCH 65/96] Make it easier to add new packages to the release script --- tools/release.sh | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tools/release.sh b/tools/release.sh index 3b1d4d8a6..be8e56353 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -45,7 +45,8 @@ export GPG_TTY=$(tty) PORT=${PORT:-1234} # subpackages to be released -SUBPKGS=${SUBPKGS:-"acme certbot-apache certbot-nginx"} +SUBPKGS_NO_CERTBOT="acme certbot-apache certbot-nginx" +SUBPKGS="certbot $SUBPKGS_NO_CERTBOT" subpkgs_modules="$(echo $SUBPKGS | sed s/-/_/g)" # certbot_compatibility_test is not packaged because: # - it is not meant to be used by anyone else than Certbot devs @@ -83,21 +84,22 @@ git checkout "$RELEASE_BRANCH" SetVersion() { ver="$1" - for pkg_dir in $SUBPKGS certbot-compatibility-test + # bumping Certbot's version number is done differently + for pkg_dir in $SUBPKGS_NO_CERTBOT certbot-compatibility-test do sed -i "s/^version.*/version = '$ver'/" $pkg_dir/setup.py done sed -i "s/^__version.*/__version__ = '$ver'/" certbot/__init__.py # interactive user input - git add -p certbot $SUBPKGS certbot-compatibility-test + git add -p $SUBPKGS certbot-compatibility-test } SetVersion "$version" echo "Preparing sdists and wheels" -for pkg_dir in . $SUBPKGS +for pkg_dir in . $SUBPKGS_NO_CERTBOT do cd $pkg_dir @@ -118,7 +120,7 @@ done mkdir "dist.$version" mv dist "dist.$version/certbot" -for pkg_dir in $SUBPKGS +for pkg_dir in $SUBPKGS_NO_CERTBOT do mv $pkg_dir/dist "dist.$version/$pkg_dir/" done @@ -140,7 +142,7 @@ pip install -U pip pip install \ --no-cache-dir \ --extra-index-url http://localhost:$PORT \ - certbot $SUBPKGS + $SUBPKGS # stop local PyPI kill $! cd ~- @@ -160,20 +162,22 @@ mkdir kgs kgs="kgs/$version" pip freeze | tee $kgs pip install nose -for module in certbot $subpkgs_modules ; do +for module in $subpkgs_modules ; do echo testing $module nosetests $module done cd ~- # pin pip hashes of the things we just built -for pkg in acme certbot certbot-apache certbot-nginx ; do +for pkg in $SUBPKGS ; do echo $pkg==$version \\ pip hash dist."$version/$pkg"/*.{whl,gz} | grep "^--hash" | python2 -c 'from sys import stdin; input = stdin.read(); print " ", input.replace("\n--hash", " \\\n --hash"),' done > letsencrypt-auto-source/pieces/certbot-requirements.txt deactivate -if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*12 " ; then +# there should be one requirement specifier and two hashes for each subpackage +expected_count=$(expr $(echo $SUBPKGS | wc -w) \* 3) +if ! wc -l letsencrypt-auto-source/pieces/certbot-requirements.txt | grep -qE "^\s*$expected_count " ; then echo Unexpected pip hash output exit 1 fi From 71451dd54b6d2e7eab6ba8f075240a13b303e86e Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Thu, 11 May 2017 15:49:34 -0700 Subject: [PATCH 66/96] security: preserve permissions on renewal conf (#4430) Ensure that permissions are preserved when renewal data is written to conf files. This allows users to limit access to the file, if they wish. Testing done: * `tox -e py27` * `tox -e lint` * Manual Testing * Got a new certificate. Restricted the permissions on the renewal conf. Renewed the certificate. Verified that the new renewal conf permissions matched. --- certbot/storage.py | 11 +++++++++++ certbot/tests/storage_test.py | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/certbot/storage.py b/certbot/storage.py index c35a268e3..4f167d4ea 100644 --- a/certbot/storage.py +++ b/certbot/storage.py @@ -4,6 +4,7 @@ import glob import logging import os import re +import stat import configobj import parsedatetime @@ -117,10 +118,20 @@ def write_renewal_config(o_filename, n_filename, archive_dir, target, relevant_d # TODO: add human-readable comments explaining other available # parameters logger.debug("Writing new config %s.", n_filename) + + # Ensure that the file exists + open(n_filename, 'a').close() + + # Copy permissions from the old version of the file, if it exists. + if os.path.exists(o_filename): + current_permissions = stat.S_IMODE(os.lstat(o_filename).st_mode) + os.chmod(n_filename, current_permissions) + with open(n_filename, "wb") as f: config.write(outfile=f) return config + def rename_renewal_config(prev_name, new_name, cli_config): """Renames cli_config.certname's config to cli_config.new_certname. diff --git a/certbot/tests/storage_test.py b/certbot/tests/storage_test.py index d8fe98536..e6e2b25ff 100644 --- a/certbot/tests/storage_test.py +++ b/certbot/tests/storage_test.py @@ -3,6 +3,7 @@ import datetime import os import shutil +import stat import unittest import configobj @@ -758,13 +759,16 @@ class RenewableCertTests(BaseRenewableCertTest): with open(temp, "w") as f: f.write("[renewalparams]\nuseful = value # A useful value\n" "useless = value # Not needed\n") + os.chmod(temp, 0o640) target = {} for x in ALL_FOUR: target[x] = "somewhere" archive_dir = "the_archive" relevant_data = {"useful": "new_value"} + from certbot import storage storage.write_renewal_config(temp, temp2, archive_dir, target, relevant_data) + with open(temp2, "r") as f: content = f.read() # useful value was updated @@ -775,6 +779,9 @@ class RenewableCertTests(BaseRenewableCertTest): self.assertTrue("useless" not in content) # check version was stored self.assertTrue("version = {0}".format(certbot.__version__) in content) + # ensure permissions are copied + self.assertEqual(stat.S_IMODE(os.lstat(temp).st_mode), + stat.S_IMODE(os.lstat(temp2).st_mode)) def test_update_symlinks(self): from certbot import storage From 9e206f802418b699caa19c3d82a5eede6fa3e53d Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Fri, 28 Apr 2017 14:53:07 -0700 Subject: [PATCH 67/96] DigitalOcean DNS Authenticator Implement an Authenticator which can fulfill a dns-01 challenge using the DigitalOcean API. Applicable only for domains using DigitalOcean for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-digitalocean -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-digitalocean -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting digitalocean interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Credentials file missing token. * Credentials file with blank token. * Credentials file with incorrect token. * Domain name not registered to DigitalOcean account. --- certbot-dns-digitalocean/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-digitalocean/MANIFEST.in | 3 + certbot-dns-digitalocean/README.rst | 1 + .../certbot_dns_digitalocean/__init__.py | 1 + .../dns_digitalocean.py | 168 ++++++++++++++++ .../dns_digitalocean_test.py | 170 ++++++++++++++++ certbot-dns-digitalocean/docs/.gitignore | 1 + certbot-dns-digitalocean/docs/Makefile | 20 ++ certbot-dns-digitalocean/docs/api.rst | 8 + .../docs/api/dns_digitalocean.rst | 5 + certbot-dns-digitalocean/docs/conf.py | 180 +++++++++++++++++ certbot-dns-digitalocean/docs/index.rst | 28 +++ certbot-dns-digitalocean/docs/make.bat | 36 ++++ certbot-dns-digitalocean/setup.cfg | 2 + certbot-dns-digitalocean/setup.py | 69 +++++++ certbot/cli.py | 2 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 4 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 6 +- 22 files changed, 897 insertions(+), 4 deletions(-) create mode 100644 certbot-dns-digitalocean/LICENSE.txt create mode 100644 certbot-dns-digitalocean/MANIFEST.in create mode 100644 certbot-dns-digitalocean/README.rst create mode 100644 certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py create mode 100644 certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py create mode 100644 certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py create mode 100644 certbot-dns-digitalocean/docs/.gitignore create mode 100644 certbot-dns-digitalocean/docs/Makefile create mode 100644 certbot-dns-digitalocean/docs/api.rst create mode 100644 certbot-dns-digitalocean/docs/api/dns_digitalocean.rst create mode 100644 certbot-dns-digitalocean/docs/conf.py create mode 100644 certbot-dns-digitalocean/docs/index.rst create mode 100644 certbot-dns-digitalocean/docs/make.bat create mode 100644 certbot-dns-digitalocean/setup.cfg create mode 100644 certbot-dns-digitalocean/setup.py diff --git a/certbot-dns-digitalocean/LICENSE.txt b/certbot-dns-digitalocean/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-digitalocean/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-digitalocean/MANIFEST.in b/certbot-dns-digitalocean/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-digitalocean/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-digitalocean/README.rst b/certbot-dns-digitalocean/README.rst new file mode 100644 index 000000000..6cccdfeb7 --- /dev/null +++ b/certbot-dns-digitalocean/README.rst @@ -0,0 +1 @@ +DigitalOcean DNS Authenticator plugin for Certbot diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py new file mode 100644 index 000000000..40b2527f8 --- /dev/null +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/__init__.py @@ -0,0 +1 @@ +"""DigitalOcean DNS Authenticator""" diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py new file mode 100644 index 000000000..73e632faf --- /dev/null +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py @@ -0,0 +1,168 @@ +"""DNS Authenticator for DigitalOcean.""" +import logging + +import digitalocean +import zope.interface + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common + +logger = logging.getLogger(__name__) + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for DigitalOcean + + This Authenticator uses the DigitalOcean API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).' + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): + super(Authenticator, cls).add_parser_arguments(add) + add('credentials', help='DigitalOcean credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the DigitalOcean API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'DigitalOcean credentials INI file', + { + 'token': 'API token for DigitalOcean account' + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_digitalocean_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_digitalocean_client().del_txt_record(domain, validation_name, validation) + + def _get_digitalocean_client(self): + return _DigitalOceanClient(self.credentials.conf('token')) + + +class _DigitalOceanClient(object): + """ + Encapsulates all communication with the DigitalOcean API. + """ + + def __init__(self, token): + self.manager = digitalocean.Manager(token=token) + + def add_txt_record(self, domain_name, record_name, record_content): + """ + Add a TXT record using the supplied information. + + :param str domain_name: The domain to use to associate the record with. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :raises certbot.errors.PluginError: if an error occurs communicating with the DigitalOcean + API + """ + + try: + domain = self._find_domain(domain_name) + except digitalocean.Error as e: + hint = None + + if str(e).startswith("Unable to authenticate"): + hint = 'Did you provide a valid API token?' + + logger.debug('Error finding domain using the DigitalOcean API: %s', e) + raise errors.PluginError('Error finding domain using the DigitalOcean API: {0}{1}' + .format(e, ' ({0})'.format(hint) if hint else '')) + + try: + result = domain.create_new_domain_record( + type='TXT', + name=self._compute_record_name(domain, record_name), + data=record_content) + + record_id = result['domain_record']['id'] + + logger.debug('Successfully added TXT record with id: %d', record_id) + except digitalocean.Error as e: + logger.debug('Error adding TXT record using the DigitalOcean API: %s', e) + raise errors.PluginError('Error adding TXT record using the DigitalOcean API: {0}' + .format(e)) + + def del_txt_record(self, domain_name, record_name, record_content): + """ + Delete a TXT record using the supplied information. + + Note that both the record's name and content are used to ensure that similar records + created concurrently (e.g., due to concurrent invocations of this plugin) are not deleted. + + Failures are logged, but not raised. + + :param str domain_name: The domain to use to associate the record with. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + """ + + try: + domain = self._find_domain(domain_name) + except digitalocean.Error as e: + logger.debug('Error finding domain using the DigitalOcean API: %s', e) + return + + try: + domain_records = domain.get_records() + + matching_records = [record for record in domain_records + if record.type == 'TXT' + and record.name == self._compute_record_name(domain, record_name) + and record.data == record_content] + except digitalocean.Error as e: + logger.debug('Error getting DNS records using the DigitalOcean API: %s', e) + return + + for record in matching_records: + try: + logger.debug('Removing TXT record with id: %s', record.id) + record.destroy() + except digitalocean.Error as e: + logger.warn('Error deleting TXT record %s using the DigitalOcean API: %s', + record.id, e) + + def _find_domain(self, domain_name): + """ + Find the domain object for a given domain name. + + :param str domain_name: The domain name for which to find the corresponding Domain. + :returns: The Domain, if found. + :rtype: `~digitalocean.Domain` + :raises certbot.errors.PluginError: if no matching Domain is found. + """ + + domain_name_guesses = dns_common.base_domain_name_guesses(domain_name) + + domains = self.manager.get_all_domains() + + for guess in domain_name_guesses: + matches = [domain for domain in domains if domain.name == guess] + + if len(matches) > 0: + domain = matches[0] + logger.debug('Found base domain for %s using name %s', domain_name, guess) + return domain + + raise errors.PluginError('Unable to determine base domain for {0} using names: {1}.' + .format(domain_name, domain_name_guesses)) + + @staticmethod + def _compute_record_name(domain, full_record_name): + # The domain, from DigitalOcean's point of view, is automatically appended. + return full_record_name.rpartition("." + domain.name)[0] diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py new file mode 100644 index 000000000..7e97eed07 --- /dev/null +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean_test.py @@ -0,0 +1,170 @@ +"""Tests for certbot_dns_digitalocean.dns_digitalocean.""" + +import os +import unittest + +import digitalocean +import mock + +from certbot import errors +from certbot.plugins import dns_test_common +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +API_ERROR = digitalocean.DataReadError() +TOKEN = 'a-token' + + +class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): + + def setUp(self): + from certbot_dns_digitalocean.dns_digitalocean import Authenticator + + super(AuthenticatorTest, self).setUp() + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"digitalocean_token": TOKEN}, path) + + self.config = mock.MagicMock(digitalocean_credentials=path, + digitalocean_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "digitalocean") + + self.mock_client = mock.MagicMock() + # _get_digitalocean_client | pylint: disable=protected-access + self.auth._get_digitalocean_client = mock.MagicMock(return_value=self.mock_client) + + def test_perform(self): + self.auth.perform([self.achall]) + + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + def test_cleanup(self): + # _attempt_cleanup | pylint: disable=protected-access + self.auth._attempt_cleanup = True + self.auth.cleanup([self.achall]) + + expected = [mock.call.del_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + +class DigitalOceanClientTest(unittest.TestCase): + id = 1 + record_prefix = "_acme-challenge" + record_name = record_prefix + "." + DOMAIN + record_content = "bar" + + def setUp(self): + from certbot_dns_digitalocean.dns_digitalocean import _DigitalOceanClient + + self.digitalocean_client = _DigitalOceanClient(TOKEN) + + self.manager = mock.MagicMock() + self.digitalocean_client.manager = self.manager + + def test_add_txt_record(self): + wrong_domain_mock = mock.MagicMock() + wrong_domain_mock.name = "other.invalid" + wrong_domain_mock.create_new_domain_record.side_effect = AssertionError('Wrong Domain') + + domain_mock = mock.MagicMock() + domain_mock.name = DOMAIN + domain_mock.create_new_domain_record.return_value = {'domain_record': {'id': self.id}} + + self.manager.get_all_domains.return_value = [wrong_domain_mock, domain_mock] + + self.digitalocean_client.add_txt_record(DOMAIN, self.record_name, self.record_content) + + domain_mock.create_new_domain_record.assert_called_with(type='TXT', + name=self.record_prefix, + data=self.record_content) + + def test_add_txt_record_fail_to_find_domain(self): + self.manager.get_all_domains.return_value = [] + + self.assertRaises(errors.PluginError, + self.digitalocean_client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_add_txt_record_error_finding_domain(self): + self.manager.get_all_domains.side_effect = API_ERROR + + self.assertRaises(errors.PluginError, + self.digitalocean_client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_add_txt_record_error_creating_record(self): + domain_mock = mock.MagicMock() + domain_mock.name = DOMAIN + domain_mock.create_new_domain_record.side_effect = API_ERROR + + self.manager.get_all_domains.return_value = [domain_mock] + + self.assertRaises(errors.PluginError, + self.digitalocean_client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record(self): + first_record_mock = mock.MagicMock() + first_record_mock.type = 'TXT' + first_record_mock.name = "DIFFERENT" + first_record_mock.data = self.record_content + + correct_record_mock = mock.MagicMock() + correct_record_mock.type = 'TXT' + correct_record_mock.name = self.record_prefix + correct_record_mock.data = self.record_content + + last_record_mock = mock.MagicMock() + last_record_mock.type = 'TXT' + last_record_mock.name = self.record_prefix + last_record_mock.data = "DIFFERENT" + + domain_mock = mock.MagicMock() + domain_mock.name = DOMAIN + domain_mock.get_records.return_value = [first_record_mock, + correct_record_mock, + last_record_mock] + + self.manager.get_all_domains.return_value = [domain_mock] + + self.digitalocean_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + correct_record_mock.destroy.assert_called() + + self.assertItemsEqual(first_record_mock.destroy.call_args_list, []) + self.assertItemsEqual(last_record_mock.destroy.call_args_list, []) + + def test_del_txt_record_error_finding_domain(self): + self.manager.get_all_domains.side_effect = API_ERROR + + self.digitalocean_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_error_finding_record(self): + domain_mock = mock.MagicMock() + domain_mock.name = DOMAIN + domain_mock.get_records.side_effect = API_ERROR + + self.manager.get_all_domains.return_value = [domain_mock] + + self.digitalocean_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_error_deleting_record(self): + record_mock = mock.MagicMock() + record_mock.type = 'TXT' + record_mock.name = self.record_prefix + record_mock.data = self.record_content + record_mock.destroy.side_effect = API_ERROR + + domain_mock = mock.MagicMock() + domain_mock.name = DOMAIN + domain_mock.get_records.return_value = [record_mock] + + self.manager.get_all_domains.return_value = [domain_mock] + + self.digitalocean_client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-digitalocean/docs/.gitignore b/certbot-dns-digitalocean/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-digitalocean/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-digitalocean/docs/Makefile b/certbot-dns-digitalocean/docs/Makefile new file mode 100644 index 000000000..701a4fdf9 --- /dev/null +++ b/certbot-dns-digitalocean/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-digitalocean +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-digitalocean/docs/api.rst b/certbot-dns-digitalocean/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-digitalocean/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-digitalocean/docs/api/dns_digitalocean.rst b/certbot-dns-digitalocean/docs/api/dns_digitalocean.rst new file mode 100644 index 000000000..8a787987e --- /dev/null +++ b/certbot-dns-digitalocean/docs/api/dns_digitalocean.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_digitalocean.dns_digitalocean` +------------------------------------------------ + +.. automodule:: certbot_dns_digitalocean.dns_digitalocean + :members: diff --git a/certbot-dns-digitalocean/docs/conf.py b/certbot-dns-digitalocean/docs/conf.py new file mode 100644 index 000000000..e223b1535 --- /dev/null +++ b/certbot-dns-digitalocean/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-digitalocean documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 10:52:06 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-digitalocean' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-digitaloceandoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-digitalocean.tex', u'certbot-dns-digitalocean Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-digitalocean', u'certbot-dns-digitalocean Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-digitalocean', u'certbot-dns-digitalocean Documentation', + author, 'certbot-dns-digitalocean', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-digitalocean/docs/index.rst b/certbot-dns-digitalocean/docs/index.rst new file mode 100644 index 000000000..9f66382ee --- /dev/null +++ b/certbot-dns-digitalocean/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-digitalocean documentation master file, created by + sphinx-quickstart on Wed May 10 10:52:06 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-digitalocean's documentation! +==================================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_digitalocean + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-digitalocean/docs/make.bat b/certbot-dns-digitalocean/docs/make.bat new file mode 100644 index 000000000..e1bda5e27 --- /dev/null +++ b/certbot-dns-digitalocean/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-digitalocean + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-digitalocean/setup.cfg b/certbot-dns-digitalocean/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-digitalocean/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-digitalocean/setup.py b/certbot-dns-digitalocean/setup.py new file mode 100644 index 000000000..f76eaa7db --- /dev/null +++ b/certbot-dns-digitalocean/setup.py @@ -0,0 +1,69 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.15.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + 'mock', + 'python-digitalocean>=1.11', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-digitalocean', + version=version, + description="DigitalOcean DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-digitalocean = certbot_dns_digitalocean.dns_digitalocean:Authenticator', + ], + }, + test_suite='certbot_dns_digitalocean', +) diff --git a/certbot/cli.py b/certbot/cli.py index f802244ae..250c0d2af 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1223,6 +1223,8 @@ def _plugins_parsing(helpful, plugins): help='Obtain certs by placing files in a webroot directory.') helpful.add(["plugins", "certonly"], "--dns-cloudflare", action="store_true", help='Obtain certs using a DNS TXT record (if you are using Cloudflare for DNS).') + helpful.add(["plugins", "certonly"], "--dns-digitalocean", action="store_true", + help='Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).') # things should not be reorder past/pre this comment: # plugins_group should be displayed in --help before plugin diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 5d1f9cdd1..cff0ed157 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -29,6 +29,7 @@ class PluginEntryPoint(object): "certbot", "certbot-apache", "certbot-dns-cloudflare", + "certbot-dns-digitalocean", "certbot-nginx", ] """Distributions for which prefix will be omitted.""" diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 29ab97a8d..c243accca 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -133,7 +133,7 @@ def choose_plugin(prepared, question): else: return None -noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare"] +noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-digitalocean"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -239,6 +239,8 @@ def cli_plugin_requests(config): req_auth = set_configurator(req_auth, "manual") if config.dns_cloudflare: req_auth = set_configurator(req_auth, "dns-cloudflare") + if config.dns_digitalocean: + req_auth = set_configurator(req_auth, "dns-digitalocean") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) return req_auth, req_inst diff --git a/tools/venv.sh b/tools/venv.sh index 2b5f4272c..da0a07830 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -15,6 +15,7 @@ fi -e .[dev,docs] \ -e certbot-apache \ -e certbot-dns-cloudflare \ + -e certbot-dns-digitalocean \ -e certbot-nginx \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tools/venv3.sh b/tools/venv3.sh index 2063d99d8..caf6bd8ba 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -14,6 +14,7 @@ fi -e .[dev,docs] \ -e certbot-apache \ -e certbot-dns-cloudflare \ + -e certbot-dns-digitalocean \ -e certbot-nginx \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tox.cover.sh b/tox.cover.sh index 81bf19d5a..afe0e03ed 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_digitalocean certbot_nginx letshelp_certbot" else pkgs="$@" fi @@ -23,6 +23,8 @@ cover () { min=100 elif [ "$1" = "certbot_dns_cloudflare" ]; then min=98 + elif [ "$1" = "certbot_dns_digitalocean" ]; then + min=98 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "letshelp_certbot" ]; then diff --git a/tox.ini b/tox.ini index 771b92d11..656c9c67b 100644 --- a/tox.ini +++ b/tox.ini @@ -33,8 +33,10 @@ plugin_paths = certbot-apache/certbot_apache certbot-nginx/certbot_nginx dns_plugin_commands = pip install -e certbot-dns-cloudflare nosetests -v certbot_dns_cloudflare --processes=-1 -dns_plugin_install_args = -e certbot-dns-cloudflare -dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare + pip install -e certbot-dns-digitalocean + nosetests -v certbot_dns_digitalocean --processes=-1 +dns_plugin_install_args = -e certbot-dns-cloudflare -e certbot-dns-digitalocean +dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test From 42d07d756df0cf96c9d20b44e772858391d48384 Mon Sep 17 00:00:00 2001 From: Ryan Pineo Date: Fri, 12 May 2017 15:45:54 -0400 Subject: [PATCH 68/96] support version 0.12.0 of configargparse fixes #4648 --- certbot/tests/util_test.py | 17 +++++++++++++++++ certbot/util.py | 7 ++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 59a4f10b2..60b3089bb 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -6,6 +6,7 @@ import shutil import stat import unittest +import configargparse import mock import six from six.moves import reload_module # pylint: disable=import-error @@ -368,6 +369,22 @@ class AddDeprecatedArgumentTest(unittest.TestCase): pass self.assertTrue("--old-option" not in stdout.getvalue()) + def test_when_configargparse_set(self): + '''In configargparse versions < 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a set.''' + orig = configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = set() + self._call("--old-option", 1) + self.assertEqual(len(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE), 1) + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = orig + + def test_when_configargparse_tuple(self): + '''In configargparse versions >= 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a tuple.''' + orig = configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = tuple() + self._call("--old-option", 1) + self.assertEqual(len(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE), 1) + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = orig + class EnforceLeValidity(unittest.TestCase): """Test enforce_le_validity.""" diff --git a/certbot/util.py b/certbot/util.py index e343fdcb1..5eec90b50 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -477,7 +477,12 @@ def add_deprecated_argument(add_argument, argument_name, nargs): sys.stderr.write( "Use of {0} is deprecated.\n".format(option_string)) - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add(ShowWarning) + # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was changed from a set + # to a tuple. + if isinstance(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE, set): + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add(ShowWarning) + else: + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE += (ShowWarning,) add_argument(argument_name, action=ShowWarning, help=argparse.SUPPRESS, nargs=nargs) From 65f7f3e12b42673a87b8da69fab60c4ecfa218ad Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 15 May 2017 12:22:47 -0700 Subject: [PATCH 69/96] Modify special action types only once --- certbot-apache/certbot_apache/configurator.py | 24 +++++++++++++++++-- .../certbot_apache/tests/configurator_test.py | 15 ++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/certbot-apache/certbot_apache/configurator.py b/certbot-apache/certbot_apache/configurator.py index 2885cd0ef..13b325d7f 100644 --- a/certbot-apache/certbot_apache/configurator.py +++ b/certbot-apache/certbot_apache/configurator.py @@ -1065,8 +1065,28 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): span_end = span_val[6] with open(span_filep, 'r') as fh: fh.seek(span_start) - vh_contents = fh.read(span_end-span_start) - return vh_contents.split("\n") + vh_contents = fh.read(span_end-span_start).split("\n") + self._remove_closing_vhost_tag(vh_contents) + return vh_contents + + def _remove_closing_vhost_tag(self, vh_contents): + """Removes the closing VirtualHost tag if it exists. + + This method modifies vh_contents directly to remove the closing + tag. If the closing vhost tag is found, everything on the line + after it is also removed. Whether or not this tag is included + in the result of span depends on the Augeas version. + + :param list vh_contents: VirtualHost block contents to check + + """ + for offset, line in enumerate(reversed(vh_contents)): + if line: + line_index = line.lower().find("") + if line_index != -1: + content_index = len(vh_contents) - offset - 1 + vh_contents[content_index] = line[:line_index] + break def _update_ssl_vhosts_addrs(self, vh_path): ssl_addrs = set() diff --git a/certbot-apache/certbot_apache/tests/configurator_test.py b/certbot-apache/certbot_apache/tests/configurator_test.py index bffbeee6a..75589dce5 100644 --- a/certbot-apache/certbot_apache/tests/configurator_test.py +++ b/certbot-apache/certbot_apache/tests/configurator_test.py @@ -597,6 +597,21 @@ class MultipleVhostsTest(util.ApacheTest): # already listens to the correct port self.assertEqual(mock_add_dir.call_count, 0) + def test_make_vhost_ssl_with_mock_span(self): + # span excludes the closing tag in older versions + # of Augeas + return_value = [self.vh_truth[0].filep, 1, 12, 0, 0, 0, 1142] + with mock.patch.object(self.config.aug, 'span') as mock_span: + mock_span.return_value = return_value + self.test_make_vhost_ssl() + + def test_make_vhost_ssl_with_mock_span2(self): + # span includes the closing tag in newer versions + # of Augeas + return_value = [self.vh_truth[0].filep, 1, 12, 0, 0, 0, 1157] + with mock.patch.object(self.config.aug, 'span') as mock_span: + mock_span.return_value = return_value + self.test_make_vhost_ssl() def test_make_vhost_ssl(self): ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) From f5b61d56bde973f791b32421f8f33d0e95d5cc7d Mon Sep 17 00:00:00 2001 From: Joona Hoikkala Date: Mon, 15 May 2017 22:32:12 +0300 Subject: [PATCH 70/96] Force augeas file reload to recalculate span indicies --- certbot-apache/certbot_apache/augeas_configurator.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/certbot-apache/certbot_apache/augeas_configurator.py b/certbot-apache/certbot_apache/augeas_configurator.py index e053d0468..444fbb763 100644 --- a/certbot-apache/certbot_apache/augeas_configurator.py +++ b/certbot-apache/certbot_apache/augeas_configurator.py @@ -120,8 +120,8 @@ class AugeasConfigurator(common.Plugin): # If the augeas tree didn't change, no files were saved and a backup # should not be created + save_files = set() if save_paths: - save_files = set() for path in save_paths: save_files.add(self.aug.get(path)[6:]) @@ -140,6 +140,12 @@ class AugeasConfigurator(common.Plugin): self.save_notes = "" self.aug.save() + # Force reload if files were modified + # This is needed to recalculate augeas directive span + if save_files: + for sf in save_files: + self.aug.remove("/files/"+sf) + self.aug.load() if title and not temporary: try: self.reverter.finalize_checkpoint(title) From d467295d2a3f71148c4c04ace5212b4697f30168 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Mon, 15 May 2017 15:01:54 -0700 Subject: [PATCH 71/96] Make 42d07d7 more closely follow repo conventions --- certbot/tests/util_test.py | 36 +++++++++++++++++++++--------------- certbot/util.py | 5 +++-- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 60b3089bb..3d51a1d96 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -6,7 +6,6 @@ import shutil import stat import unittest -import configargparse import mock import six from six.moves import reload_module # pylint: disable=import-error @@ -369,21 +368,28 @@ class AddDeprecatedArgumentTest(unittest.TestCase): pass self.assertTrue("--old-option" not in stdout.getvalue()) - def test_when_configargparse_set(self): - '''In configargparse versions < 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a set.''' - orig = configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = set() - self._call("--old-option", 1) - self.assertEqual(len(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE), 1) - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = orig + def test_set_constant(self): + """Test when ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a set. - def test_when_configargparse_tuple(self): - '''In configargparse versions >= 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a tuple.''' - orig = configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = tuple() - self._call("--old-option", 1) - self.assertEqual(len(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE), 1) - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = orig + This variable is a set in configargparse versions < 0.12.0. + + """ + self._test_constant_common(set) + + def test_tuple_constant(self): + """Test when ACTION_TYPES_THAT_DONT_NEED_A_VALUE is a tuple. + + This variable is a tuple in configargparse versions >= 0.12.0. + + """ + self._test_constant_common(tuple) + + def _test_constant_common(self, typ): + with mock.patch("certbot.util.configargparse") as mock_configargparse: + mock_configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = typ() + self._call("--old-option", 1) + self.assertEqual( + len(mock_configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE), 1) class EnforceLeValidity(unittest.TestCase): diff --git a/certbot/util.py b/certbot/util.py index 5eec90b50..bece91f01 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -477,9 +477,10 @@ def add_deprecated_argument(add_argument, argument_name, nargs): sys.stderr.write( "Use of {0} is deprecated.\n".format(option_string)) - # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was changed from a set - # to a tuple. + # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was changed from a + # set to a tuple. if isinstance(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE, set): + # pylint: disable=no-member configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add(ShowWarning) else: configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE += (ShowWarning,) From 23e6c28d802648944638d8b0c054543f72d1d5e6 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Mon, 15 May 2017 15:30:50 -0700 Subject: [PATCH 72/96] Allow Nginx to insert include files with comments inside (#4666) * add failing test case * allow include files to insert comments * lint --- certbot-nginx/certbot_nginx/parser.py | 8 +++-- .../certbot_nginx/tests/parser_test.py | 29 ++++++++++++++++++- .../testdata/etc_nginx/comment_in_file.conf | 1 + 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/comment_in_file.conf diff --git a/certbot-nginx/certbot_nginx/parser.py b/certbot-nginx/certbot_nginx/parser.py index 558275b0d..4e4aa36ca 100644 --- a/certbot-nginx/certbot_nginx/parser.py +++ b/certbot-nginx/certbot_nginx/parser.py @@ -529,7 +529,10 @@ def _add_directive(block, directive, replace): """ directive = nginxparser.UnspacedList(directive) - if len(directive) == 0 or directive[0] == '#': + def is_whitespace_or_comment(directive): + """Is this directive either a whitespace or comment directive?""" + return len(directive) == 0 or directive[0] == '#' + if is_whitespace_or_comment(directive): # whitespace or comment block.append(directive) return @@ -574,7 +577,8 @@ def _add_directive(block, directive, replace): for included_directive in included_directives: included_dir_loc = find_location(included_directive) included_dir_name = included_directive[0] - if not can_append(included_dir_loc, included_dir_name): + if not is_whitespace_or_comment(included_directive) \ + and not can_append(included_dir_loc, included_dir_name): if block[included_dir_loc] != included_directive: raise errors.MisconfigurationError(err_fmt.format(included_directive, block[included_dir_loc])) diff --git a/certbot-nginx/certbot_nginx/tests/parser_test.py b/certbot-nginx/certbot_nginx/tests/parser_test.py index 7b33b1075..3877bf5d4 100644 --- a/certbot-nginx/certbot_nginx/tests/parser_test.py +++ b/certbot-nginx/certbot_nginx/tests/parser_test.py @@ -13,7 +13,7 @@ from certbot_nginx import parser from certbot_nginx.tests import util -class NginxParserTest(util.NginxTest): +class NginxParserTest(util.NginxTest): #pylint: disable=too-many-public-methods """Nginx Parser Test.""" def setUp(self): @@ -230,6 +230,33 @@ class NginxParserTest(util.NginxTest): ['ssl_certificate', '/etc/ssl/cert2.pem']], replace=False) + def test_comment_is_repeatable(self): + nparser = parser.NginxParser(self.config_path) + example_com = nparser.abs_path('sites-enabled/example.com') + mock_vhost = obj.VirtualHost(example_com, + None, None, None, + set(['.example.com', 'example.*']), + None, [0]) + nparser.add_server_directives(mock_vhost, + [['\n ', '#', ' ', 'what a nice comment']], + replace=False) + nparser.add_server_directives(mock_vhost, + [['\n ', 'include', ' ', + nparser.abs_path('comment_in_file.conf')]], + replace=False) + from certbot_nginx.parser import COMMENT + self.assertEqual(nparser.parsed[example_com], + [[['server'], [['listen', '69.50.225.155:9000'], + ['listen', '127.0.0.1'], + ['server_name', '.example.com'], + ['server_name', 'example.*'], + ['#', ' ', 'what a nice comment'], + [], + ['include', nparser.abs_path('comment_in_file.conf')], + ['#', COMMENT], + []]]] +) + def test_replace_server_directives(self): nparser = parser.NginxParser(self.config_path) target = set(['.example.com', 'example.*']) diff --git a/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/comment_in_file.conf b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/comment_in_file.conf new file mode 100644 index 000000000..f761079fa --- /dev/null +++ b/certbot-nginx/certbot_nginx/tests/testdata/etc_nginx/comment_in_file.conf @@ -0,0 +1 @@ +# a comment inside a file \ No newline at end of file From 8c29cb0810adbb5f08b4f8317ff522899738e197 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 May 2017 12:19:07 -0700 Subject: [PATCH 73/96] Force nginx tests to run during CI (#4558) * force nginx tests to run during CI * offer default value --- tests/boulder-integration.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/boulder-integration.sh b/tests/boulder-integration.sh index 08c482676..d86a6fb8c 100755 --- a/tests/boulder-integration.sh +++ b/tests/boulder-integration.sh @@ -203,7 +203,9 @@ common revoke --cert-path "$root/conf/live/le2.wtf/cert.pem" \ common unregister -if type nginx; +# Most CI systems set this variable to true. +# If the tests are running as part of CI, Nginx should be available. +if ${CI:-false} || type nginx; then . ./certbot-nginx/tests/boulder-integration.sh fi From 28f7c03f3ac9201d1cf3b0e1e5920397355736e8 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 May 2017 12:49:42 -0700 Subject: [PATCH 74/96] Add 0.14.1 notes to the CHANGELOG --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b8d5c0c7..eb9b4efde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). +## 0.14.1 - 2017-05-16 + +### Fixed + +* Certbot now works with configargparse 0.12.0. +* Issues with the Apache plugin and Augeas 1.7+ have been resolved. +* A problem where the Nginx plugin would fail to install certificates on + systems that had the plugin's SSL/TLS options file from 7+ months ago has + been fixed. + ## 0.14.0 - 2017-05-04 ### Added From 0a3d06cfd1db4993a48e46fff65aa5d03c2cea46 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 May 2017 12:53:08 -0700 Subject: [PATCH 75/96] fix spacing --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb9b4efde..dad6e96b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,8 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). * Certbot now works with configargparse 0.12.0. * Issues with the Apache plugin and Augeas 1.7+ have been resolved. * A problem where the Nginx plugin would fail to install certificates on - systems that had the plugin's SSL/TLS options file from 7+ months ago has - been fixed. +systems that had the plugin's SSL/TLS options file from 7+ months ago has been +fixed. ## 0.14.0 - 2017-05-04 From 42d5b15d552a95d94de3b2b75b53683ba0c628df Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 May 2017 12:54:15 -0700 Subject: [PATCH 76/96] add GH link --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dad6e96b6..2f66582d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ Certbot adheres to [Semantic Versioning](http://semver.org/). systems that had the plugin's SSL/TLS options file from 7+ months ago has been fixed. +More details about these changes can be found on our GitHub repo: +https://github.com/certbot/certbot/issues?q=is%3Aissue+milestone%3A0.14.1+is%3Aclosed + ## 0.14.0 - 2017-05-04 ### Added From 05c31a47cbc9e400d577355ac394db20d1864d79 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Tue, 16 May 2017 14:34:01 -0700 Subject: [PATCH 77/96] Make 0.14.1 release changes in master (#4675) * Release 0.14.1 (cherry picked from commit 78e3bd6e8ccbb3e01aa0fe1543fe1b35ff1bb319) * Bump version to 0.15.0 --- certbot-auto | 26 +++++++++--------- docs/cli-help.txt | 2 +- letsencrypt-auto | 26 +++++++++--------- letsencrypt-auto-source/certbot-auto.asc | 14 +++++----- letsencrypt-auto-source/letsencrypt-auto | 24 ++++++++-------- letsencrypt-auto-source/letsencrypt-auto.sig | Bin 256 -> 256 bytes .../pieces/certbot-requirements.txt | 24 ++++++++-------- 7 files changed, 58 insertions(+), 58 deletions(-) diff --git a/certbot-auto b/certbot-auto index 0db142a78..39edbb3c5 100755 --- a/certbot-auto +++ b/certbot-auto @@ -28,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.14.0" +LE_AUTO_VERSION="0.14.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -860,18 +860,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.14.0 \ - --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ - --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 -certbot==0.14.0 \ - --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ - --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 -certbot-apache==0.14.0 \ - --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ - --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f -certbot-nginx==0.14.0 \ - --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ - --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db +acme==0.14.1 \ + --hash=sha256:f535d6459dcafa436749a8d2fdfafed21b792efa05b8bd3263fcd739c2e1497c \ + --hash=sha256:0e6d9d1bbb71d80c61c8d10ab9a40bcf38e25f0fa016b9769e96ebf5a79b552b +certbot==0.14.1 \ + --hash=sha256:f950a058d4f657160de4ad163d9f781fe7adeec0c0a44556841adb03ad135d13 \ + --hash=sha256:519b28124869d97116cb1f2f04ccc2937c0b2fd32fce43576eb80c0e4ff1ab65 +certbot-apache==0.14.1 \ + --hash=sha256:1dda9b4dcf66f6dfba37c787d849e69ad25a344572f74a76fc4447bb1a5417b2 \ + --hash=sha256:da84996e345fc5789da3575225536b27fa3b35f89b2db2d8f494a34bced14f9b +certbot-nginx==0.14.1 \ + --hash=sha256:bd3d4a1dcd6fa9e8ead19a9da88693f08b63464c86be2442e42cd60565c3f05f \ + --hash=sha256:f0c19f667072e4cfa6b92abf8312b6bee3ed1d2432676b211593034e7d1abb7e UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/docs/cli-help.txt b/docs/cli-help.txt index 2318a7255..f1fcba132 100644 --- a/docs/cli-help.txt +++ b/docs/cli-help.txt @@ -89,7 +89,7 @@ optional arguments: case, and to know when to deprecate support for past Python versions and flags. If you wish to hide this information from the Let's Encrypt server, set this to - "". (default: CertbotACMEClient/0.14.0 (certbot; + "". (default: CertbotACMEClient/0.14.1 (certbot; Ubuntu 16.04.2 LTS) Authenticator/XXX Installer/YYY (SUBCOMMAND; flags: FLAGS) Py/2.7.12). The flags encoded in the user agent are: --duplicate, --force- diff --git a/letsencrypt-auto b/letsencrypt-auto index 0db142a78..39edbb3c5 100755 --- a/letsencrypt-auto +++ b/letsencrypt-auto @@ -28,7 +28,7 @@ if [ -z "$VENV_PATH" ]; then VENV_PATH="$XDG_DATA_HOME/$VENV_NAME" fi VENV_BIN="$VENV_PATH/bin" -LE_AUTO_VERSION="0.14.0" +LE_AUTO_VERSION="0.14.1" BASENAME=$(basename $0) USAGE="Usage: $BASENAME [OPTIONS] A self-updating wrapper script for the Certbot ACME client. When run, updates @@ -860,18 +860,18 @@ letsencrypt==0.7.0 \ # THE LINES BELOW ARE EDITED BY THE RELEASE SCRIPT; ADD ALL DEPENDENCIES ABOVE. -acme==0.14.0 \ - --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ - --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 -certbot==0.14.0 \ - --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ - --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 -certbot-apache==0.14.0 \ - --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ - --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f -certbot-nginx==0.14.0 \ - --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ - --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db +acme==0.14.1 \ + --hash=sha256:f535d6459dcafa436749a8d2fdfafed21b792efa05b8bd3263fcd739c2e1497c \ + --hash=sha256:0e6d9d1bbb71d80c61c8d10ab9a40bcf38e25f0fa016b9769e96ebf5a79b552b +certbot==0.14.1 \ + --hash=sha256:f950a058d4f657160de4ad163d9f781fe7adeec0c0a44556841adb03ad135d13 \ + --hash=sha256:519b28124869d97116cb1f2f04ccc2937c0b2fd32fce43576eb80c0e4ff1ab65 +certbot-apache==0.14.1 \ + --hash=sha256:1dda9b4dcf66f6dfba37c787d849e69ad25a344572f74a76fc4447bb1a5417b2 \ + --hash=sha256:da84996e345fc5789da3575225536b27fa3b35f89b2db2d8f494a34bced14f9b +certbot-nginx==0.14.1 \ + --hash=sha256:bd3d4a1dcd6fa9e8ead19a9da88693f08b63464c86be2442e42cd60565c3f05f \ + --hash=sha256:f0c19f667072e4cfa6b92abf8312b6bee3ed1d2432676b211593034e7d1abb7e UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/certbot-auto.asc b/letsencrypt-auto-source/certbot-auto.asc index 236e4dcdc..cdc9ef58e 100644 --- a/letsencrypt-auto-source/certbot-auto.asc +++ b/letsencrypt-auto-source/certbot-auto.asc @@ -1,11 +1,11 @@ -----BEGIN PGP SIGNATURE----- Version: GnuPG v2 -iQEcBAABCAAGBQJZC76WAAoJEE0XyZXNl3XyXhcIAJ1+gPoWZmXjFcC4by2tDBoM -Lkxf5BNxq8aq7qSohU8SqSo6ShDkWh9ci390n+jbOX1R503uQL1egGbEAJbziFYq -vym6j0AmqM+2/YcWmcj3J7RYtDOV1sUPKD2pgUxWtvQrd9iZ1235WMzBF/uBprzm -qAtFwF04V2H3kkC4e7+jAEkFzs1TJ8fYumqqqw0NgSwM6bikfurpRyf8qR2RVYWt -e3GOTxyBVbjhp2UPy/O8Xx7iBD3m+t9mJgsCJ9l8s7xKot6LF7+WrJkn0A3cfKcR -LSTataKedsP3u1jOgP3y2ujumBlDlDRuXn6vK/YKNYNnHte5B9mstSzoDGgRvHE= -=3Jgs +iQEcBAABCAAGBQJZGzDgAAoJEE0XyZXNl3XyBXYIAIYBMJKzAbLYsHrP/KF3aLLh +S9AWK5IP/tftHWgxS0mQ0JqQvWsRLGoQo7xaeKKIBD8QQsHA9hsdxPwy++rQcaZY +AzvpUBPIfiCDCa1XPiRy7YduAvsAoPB7jncP8rYdoFZL3lcUpbmI/9Sk1nlsm81n +5EcNJ9T8RRAkkH0i6DTLine48DgI7MlLhce/mAr3wDrcKAmENZksZW7vgAlI69ri +cTb+qIlwgFRLAF0Q41klTiFdHi6+vj+mFHHNFyuERpf7VT3ngBZmAmiRybxo/m8g +p9/54LGw3bQ25uAZXKVtIX5CqOoJL1GHe13MEyDOgBSDp+KqNGWJ8PEPA9XGwqw= +=H8UX -----END PGP SIGNATURE----- diff --git a/letsencrypt-auto-source/letsencrypt-auto b/letsencrypt-auto-source/letsencrypt-auto index 7e4a7554f..983c7e33d 100755 --- a/letsencrypt-auto-source/letsencrypt-auto +++ b/letsencrypt-auto-source/letsencrypt-auto @@ -866,18 +866,18 @@ letsencrypt==0.7.0 \ --hash=sha256:105a5fb107e45bcd0722eb89696986dcf5f08a86a321d6aef25a0c7c63375ade \ --hash=sha256:c36e532c486a7e92155ee09da54b436a3c420813ec1c590b98f635d924720de9 -acme==0.14.0 \ - --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ - --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 -certbot==0.14.0 \ - --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ - --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 -certbot-apache==0.14.0 \ - --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ - --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f -certbot-nginx==0.14.0 \ - --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ - --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db +acme==0.14.1 \ + --hash=sha256:f535d6459dcafa436749a8d2fdfafed21b792efa05b8bd3263fcd739c2e1497c \ + --hash=sha256:0e6d9d1bbb71d80c61c8d10ab9a40bcf38e25f0fa016b9769e96ebf5a79b552b +certbot==0.14.1 \ + --hash=sha256:f950a058d4f657160de4ad163d9f781fe7adeec0c0a44556841adb03ad135d13 \ + --hash=sha256:519b28124869d97116cb1f2f04ccc2937c0b2fd32fce43576eb80c0e4ff1ab65 +certbot-apache==0.14.1 \ + --hash=sha256:1dda9b4dcf66f6dfba37c787d849e69ad25a344572f74a76fc4447bb1a5417b2 \ + --hash=sha256:da84996e345fc5789da3575225536b27fa3b35f89b2db2d8f494a34bced14f9b +certbot-nginx==0.14.1 \ + --hash=sha256:bd3d4a1dcd6fa9e8ead19a9da88693f08b63464c86be2442e42cd60565c3f05f \ + --hash=sha256:f0c19f667072e4cfa6b92abf8312b6bee3ed1d2432676b211593034e7d1abb7e UNLIKELY_EOF # ------------------------------------------------------------------------- diff --git a/letsencrypt-auto-source/letsencrypt-auto.sig b/letsencrypt-auto-source/letsencrypt-auto.sig index cb07e9573ee0c8b116c5b9a8d758cc0e2b10961d..9a95dea5712d616c76d5dd2a7432d0770ece9ebc 100644 GIT binary patch literal 256 zcmV+b0ssDF#^-kRK?i;CYYk)E?B%4(9BZFNY9;)9>rZSC*MZ);2R0K2OtVVt%UXxg z{f*^*IipLXZ${39P3|d*qJH_aQkL@Al?@2hxEUD|Ax%ocn5?qvtb4+xO2{iR0?nfd zZd-=i>=AnX^nGt#?d1#NFYpOIlfk?p(JuoL`(z zU9jP=Wbkoa1)-+@OW7w$v!*~FBW%s!syPeLcRD;_*sG9?u~k2aPo0VJpo#e-Jn{AS zie(k@SgJ_j(a`#|1bX$2g$54+A;%0tX&&(jbV&P)Q6k%yKL%eTHq9#Why;pY5bgwI GZYFf1+knyl literal 256 zcmV+b0ssD}*cqby5KtZP(Mxv3J)FkHt>eYFn0e(bA5G*B{HO!8qxTDY5xsI2*#Mgb z5U%;nxm4m=scfQB7aLexo@l4Q2a?^K3zSQFK=7TboMlyK zu)t&mKKnf*3aVCnqzu@VE>eW$VD5r6{8T*}hy-5gBVxQ^RM!@X_)XDdF46=vEGDm1 zt$DKaS%!b_${M$E{07#~CZwuL&&vru10`97U;Qrc0@V1C=`Dfr+8M{H_isyB`Ki9b G^;U^#=z%T( diff --git a/letsencrypt-auto-source/pieces/certbot-requirements.txt b/letsencrypt-auto-source/pieces/certbot-requirements.txt index a49abeefd..90b47a3cb 100644 --- a/letsencrypt-auto-source/pieces/certbot-requirements.txt +++ b/letsencrypt-auto-source/pieces/certbot-requirements.txt @@ -1,12 +1,12 @@ -acme==0.14.0 \ - --hash=sha256:fca8766a2596833e8886f7ef72cf82d1f6c6cffa895781a5676861c251b24b70 \ - --hash=sha256:ce7d2bca31e85adac1030c944e0a9d96e8b0f85cdc616b78d40eb09c91803543 -certbot==0.14.0 \ - --hash=sha256:071790b1ec4e5b94aa1688f8a62a10905c28438cd55d990cdb8c9f733d3a4a41 \ - --hash=sha256:98add3721e1edaedb404879a9d39bd49020e94fc8eedbc46032a00ada51d7741 -certbot-apache==0.14.0 \ - --hash=sha256:ab837efce7aa4c4e47a724a60dcbeacadb9dfe64bd1d32a4e854678c4fcd82a3 \ - --hash=sha256:bbcd21d9f3fd8cdc4453ef94d0cb6033c3a19f879dcd314231501ebb7180168f -certbot-nginx==0.14.0 \ - --hash=sha256:608b2f6f2b04ce93c503a95ffba4f0e0ca2e0cb9ea587a8376368fa621b388e4 \ - --hash=sha256:86e964b2a7818cc165d913e27e504f2ef2f60750ab0db6d39bfb3465d54c30db +acme==0.14.1 \ + --hash=sha256:f535d6459dcafa436749a8d2fdfafed21b792efa05b8bd3263fcd739c2e1497c \ + --hash=sha256:0e6d9d1bbb71d80c61c8d10ab9a40bcf38e25f0fa016b9769e96ebf5a79b552b +certbot==0.14.1 \ + --hash=sha256:f950a058d4f657160de4ad163d9f781fe7adeec0c0a44556841adb03ad135d13 \ + --hash=sha256:519b28124869d97116cb1f2f04ccc2937c0b2fd32fce43576eb80c0e4ff1ab65 +certbot-apache==0.14.1 \ + --hash=sha256:1dda9b4dcf66f6dfba37c787d849e69ad25a344572f74a76fc4447bb1a5417b2 \ + --hash=sha256:da84996e345fc5789da3575225536b27fa3b35f89b2db2d8f494a34bced14f9b +certbot-nginx==0.14.1 \ + --hash=sha256:bd3d4a1dcd6fa9e8ead19a9da88693f08b63464c86be2442e42cd60565c3f05f \ + --hash=sha256:f0c19f667072e4cfa6b92abf8312b6bee3ed1d2432676b211593034e7d1abb7e From 4caff11371fe58a6211488affe29d61da9c80cac Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Wed, 17 May 2017 11:26:26 -0700 Subject: [PATCH 78/96] Google Cloud DNS Authenticator (#4581) Implement an Authenticator which can fulfill a dns-01 challenge using the Google Cloud DNS API. Applicable only for domains using Google Cloud DNS for DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-google -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-google -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting google interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Domain name not registered to Google Cloud Platform account. --- .../certbot_dns_cloudflare/dns_cloudflare.py | 2 +- .../dns_digitalocean.py | 2 +- certbot-dns-google/LICENSE.txt | 190 ++++++++++++++++ certbot-dns-google/MANIFEST.in | 3 + certbot-dns-google/README.rst | 1 + .../certbot_dns_google/__init__.py | 1 + .../certbot_dns_google/dns_google.py | 184 ++++++++++++++++ .../certbot_dns_google/dns_google_test.py | 202 ++++++++++++++++++ certbot-dns-google/docs/.gitignore | 1 + certbot-dns-google/docs/Makefile | 20 ++ certbot-dns-google/docs/api.rst | 8 + certbot-dns-google/docs/api/dns_google.rst | 5 + certbot-dns-google/docs/conf.py | 180 ++++++++++++++++ certbot-dns-google/docs/index.rst | 28 +++ certbot-dns-google/docs/make.bat | 36 ++++ certbot-dns-google/setup.cfg | 2 + certbot-dns-google/setup.py | 70 ++++++ certbot/cli.py | 2 + certbot/plugins/disco.py | 1 + certbot/plugins/dns_common.py | 4 +- certbot/plugins/selection.py | 5 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 6 +- 25 files changed, 951 insertions(+), 8 deletions(-) create mode 100644 certbot-dns-google/LICENSE.txt create mode 100644 certbot-dns-google/MANIFEST.in create mode 100644 certbot-dns-google/README.rst create mode 100644 certbot-dns-google/certbot_dns_google/__init__.py create mode 100644 certbot-dns-google/certbot_dns_google/dns_google.py create mode 100644 certbot-dns-google/certbot_dns_google/dns_google_test.py create mode 100644 certbot-dns-google/docs/.gitignore create mode 100644 certbot-dns-google/docs/Makefile create mode 100644 certbot-dns-google/docs/api.rst create mode 100644 certbot-dns-google/docs/api/dns_google.rst create mode 100644 certbot-dns-google/docs/conf.py create mode 100644 certbot-dns-google/docs/index.rst create mode 100644 certbot-dns-google/docs/make.bat create mode 100644 certbot-dns-google/setup.cfg create mode 100644 certbot-dns-google/setup.py diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py index f8f674693..50040b916 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py @@ -29,7 +29,7 @@ class Authenticator(dns_common.DNSAuthenticator): self.credentials = None @classmethod - def add_parser_arguments(cls, add): + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ super(Authenticator, cls).add_parser_arguments(add) add('credentials', help='Cloudflare credentials INI file.') diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py index 73e632faf..30e0f2525 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py @@ -26,7 +26,7 @@ class Authenticator(dns_common.DNSAuthenticator): self.credentials = None @classmethod - def add_parser_arguments(cls, add): + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ super(Authenticator, cls).add_parser_arguments(add) add('credentials', help='DigitalOcean credentials INI file.') diff --git a/certbot-dns-google/LICENSE.txt b/certbot-dns-google/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-google/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-google/MANIFEST.in b/certbot-dns-google/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-google/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-google/README.rst b/certbot-dns-google/README.rst new file mode 100644 index 000000000..37586abbf --- /dev/null +++ b/certbot-dns-google/README.rst @@ -0,0 +1 @@ +Google Cloud DNS Authenticator plugin for Certbot diff --git a/certbot-dns-google/certbot_dns_google/__init__.py b/certbot-dns-google/certbot_dns_google/__init__.py new file mode 100644 index 000000000..9e9096d83 --- /dev/null +++ b/certbot-dns-google/certbot_dns_google/__init__.py @@ -0,0 +1 @@ +"""Google Cloud DNS Authenticator""" diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py new file mode 100644 index 000000000..3f7a8d9f2 --- /dev/null +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -0,0 +1,184 @@ +"""DNS Authenticator for Google Cloud DNS.""" +import json +import logging + +import zope.interface +from googleapiclient import discovery +from googleapiclient import errors as googleapiclient_errors +from oauth2client.service_account import ServiceAccountCredentials + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common + +logger = logging.getLogger(__name__) + +ACCT_URL = 'https://developers.google.com/identity/protocols/OAuth2ServiceAccount#creatinganaccount' +PERMISSIONS_URL = 'https://cloud.google.com/dns/access-control#permissions_and_roles' + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for Google Cloud DNS + + This Authenticator uses the Google Cloud DNS API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using Google Cloud DNS for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=60) + add('credentials', + help=('Path to Google Cloud DNS service account JSON file. (See {0} for' + + 'information about creating a service account and {1} for information about the' + + 'required permissions.)').format(ACCT_URL, PERMISSIONS_URL)) + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the Google Cloud DNS API.' + + def _setup_credentials(self): + self._configure_file('credentials', 'path to Google Cloud DNS service account JSON file') + + dns_common.validate_file_permissions(self.conf('credentials')) + + def _perform(self, domain, validation_name, validation): + self._get_google_client().add_txt_record(domain, validation_name, validation, self.ttl) + + def _cleanup(self, domain, validation_name, validation): + self._get_google_client().del_txt_record(domain, validation_name, validation, self.ttl) + + def _get_google_client(self): + return _GoogleClient(self.conf('credentials')) + + +class _GoogleClient(object): + """ + Encapsulates all communication with the Google Cloud DNS API. + """ + + def __init__(self, account_json): + + scopes = ['https://www.googleapis.com/auth/ndev.clouddns.readwrite'] + credentials = ServiceAccountCredentials.from_json_keyfile_name(account_json, scopes) + self.dns = discovery.build('dns', 'v1', credentials=credentials, cache_discovery=False) + with open(account_json) as account: + self.project_id = json.load(account)['project_id'] + + def add_txt_record(self, domain, record_name, record_content, record_ttl): + """ + Add a TXT record using the supplied information. + + :param str domain: The domain to use to look up the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL (number of seconds that the record may be cached). + :raises certbot.errors.PluginError: if an error occurs communicating with the Google API + """ + + zone_id = self._find_managed_zone_id(domain) + + data = { + "kind": "dns#change", + "additions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": [record_content, ], + "ttl": record_ttl, + }, + ], + } + + changes = self.dns.changes() # changes | pylint: disable=no-member + + try: + request = changes.create(project=self.project_id, managedZone=zone_id, body=data) + response = request.execute() + + status = response['status'] + change = response['id'] + while status == 'pending': + request = changes.get(project=self.project_id, managedZone=zone_id, changeId=change) + response = request.execute() + status = response['status'] + except googleapiclient_errors.Error as e: + logger.error('Encountered error adding TXT record: %s', e) + raise errors.PluginError('Error communicating with the Google Cloud DNS API: {0}' + .format(e)) + + def del_txt_record(self, domain, record_name, record_content, record_ttl): + """ + Delete a TXT record using the supplied information. + + :param str domain: The domain to use to look up the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :param int record_ttl: The record TTL (number of seconds that the record may be cached). + :raises certbot.errors.PluginError: if an error occurs communicating with the Google API + """ + + try: + zone_id = self._find_managed_zone_id(domain) + except errors.PluginError as e: + logger.warn('Error finding zone. Skipping cleanup.') + return + + data = { + "kind": "dns#change", + "deletions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": record_name + ".", + "rrdatas": [record_content, ], + "ttl": record_ttl, + }, + ], + } + + changes = self.dns.changes() # changes | pylint: disable=no-member + + try: + request = changes.create(project=self.project_id, managedZone=zone_id, body=data) + request.execute() + except googleapiclient_errors.Error as e: + logger.warn('Encountered error deleting TXT record: %s', e) + + def _find_managed_zone_id(self, domain): + """ + Find the managed zone for a given domain. + + :param str domain: The domain for which to find the managed zone. + :returns: The ID of the managed zone, if found. + :rtype: str + :raises certbot.errors.PluginError: if the managed zone cannot be found. + """ + + zone_dns_name_guesses = dns_common.base_domain_name_guesses(domain) + + mz = self.dns.managedZones() # managedZones | pylint: disable=no-member + for zone_name in zone_dns_name_guesses: + try: + request = mz.list(project=self.project_id, dnsName=zone_name + '.') + response = request.execute() + zones = response['managedZones'] + except googleapiclient_errors.Error as e: + raise errors.PluginError('Encountered error finding managed zone: {0}' + .format(e)) + + if len(zones) > 0: + zone_id = zones[0]['id'] + logger.debug('Found id of %s for %s using name %s', zone_id, domain, zone_name) + return zone_id + + raise errors.PluginError('Unable to determine managed zone for {0} using zone names: {1}.' + .format(domain, zone_dns_name_guesses)) diff --git a/certbot-dns-google/certbot_dns_google/dns_google_test.py b/certbot-dns-google/certbot_dns_google/dns_google_test.py new file mode 100644 index 000000000..eb41fa4ee --- /dev/null +++ b/certbot-dns-google/certbot_dns_google/dns_google_test.py @@ -0,0 +1,202 @@ +"""Tests for certbot_dns_google.dns_google.""" + +import os +import unittest + +import mock +from googleapiclient.errors import Error + +from certbot import errors +from certbot.plugins import dns_test_common +from certbot.plugins.dns_test_common import DOMAIN +from certbot.tests import util as test_util + +ACCOUNT_JSON_PATH = '/not/a/real/path.json' +API_ERROR = Error() +PROJECT_ID = "test-test-1" + + +class AuthenticatorTest(test_util.TempDirTestCase, dns_test_common.BaseAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_google.dns_google import Authenticator + + path = os.path.join(self.tempdir, 'file.json') + open(path, "wb").close() + + super(AuthenticatorTest, self).setUp() + self.config = mock.MagicMock(google_credentials=path, + google_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "google") + + self.mock_client = mock.MagicMock() + # _get_google_client | pylint: disable=protected-access + self.auth._get_google_client = mock.MagicMock(return_value=self.mock_client) + + def test_perform(self): + self.auth.perform([self.achall]) + + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + def test_cleanup(self): + # _attempt_cleanup | pylint: disable=protected-access + self.auth._attempt_cleanup = True + self.auth.cleanup([self.achall]) + + expected = [mock.call.del_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + +class GoogleClientTest(unittest.TestCase): + record_name = "foo" + record_content = "bar" + record_ttl = 42 + zone = "ZONE_ID" + change = "an-id" + + def _setUp_client_with_mock(self, zone_request_side_effect): + from certbot_dns_google.dns_google import _GoogleClient + + client = _GoogleClient(ACCOUNT_JSON_PATH) + + # Setup + mock_mz = mock.MagicMock() + mock_mz.list.return_value.execute.side_effect = zone_request_side_effect + + mock_changes = mock.MagicMock() + + client.dns.managedZones = mock.MagicMock(return_value=mock_mz) + client.dns.changes = mock.MagicMock(return_value=mock_changes) + + return client, mock_changes + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_add_txt_record(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + expected_body = { + "kind": "dns#change", + "additions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": self.record_name + ".", + "rrdatas": [self.record_content, ], + "ttl": self.record_ttl, + }, + ], + } + + changes.create.assert_called_with(body=expected_body, + managedZone=self.zone, + project=PROJECT_ID) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_add_txt_record_and_poll(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + changes.create.return_value.execute.return_value = {'status': 'pending', 'id': self.change} + changes.get.return_value.execute.return_value = {'status': 'done'} + + client.add_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + changes.create.assert_called_with(body=mock.ANY, + managedZone=self.zone, + project=PROJECT_ID) + + changes.get.assert_called_with(changeId=self.change, + managedZone=self.zone, + project=PROJECT_ID) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_add_txt_record_error_during_zone_lookup(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock(API_ERROR) + + self.assertRaises(errors.PluginError, client.add_txt_record, + DOMAIN, self.record_name, self.record_content, self.record_ttl) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_add_txt_record_zone_not_found(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock([{'managedZones': []}, + {'managedZones': []}]) + + self.assertRaises(errors.PluginError, client.add_txt_record, + DOMAIN, self.record_name, self.record_content, self.record_ttl) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_add_txt_record_error_during_add(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + changes.create.side_effect = API_ERROR + + self.assertRaises(errors.PluginError, client.add_txt_record, + DOMAIN, self.record_name, self.record_content, self.record_ttl) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_del_txt_record(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + + client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + expected_body = { + "kind": "dns#change", + "deletions": [ + { + "kind": "dns#resourceRecordSet", + "type": "TXT", + "name": self.record_name + ".", + "rrdatas": [self.record_content, ], + "ttl": self.record_ttl, + }, + ], + } + + changes.create.assert_called_with(body=expected_body, + managedZone=self.zone, + project=PROJECT_ID) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_del_txt_record_error_during_zone_lookup(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock(API_ERROR) + + client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_del_txt_record_zone_not_found(self, unused_credential_mock): + client, unused_changes = self._setUp_client_with_mock([{'managedZones': []}, + {'managedZones': []}]) + + client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + @mock.patch('oauth2client.service_account.ServiceAccountCredentials.from_json_keyfile_name') + @mock.patch('certbot_dns_google.dns_google.open', + mock.mock_open(read_data='{"project_id": "' + PROJECT_ID + '"}')) + def test_del_txt_record_error_during_delete(self, unused_credential_mock): + client, changes = self._setUp_client_with_mock([{'managedZones': [{'id': self.zone}]}]) + changes.create.side_effect = API_ERROR + + client.del_txt_record(DOMAIN, self.record_name, self.record_content, self.record_ttl) + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-google/docs/.gitignore b/certbot-dns-google/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-google/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-google/docs/Makefile b/certbot-dns-google/docs/Makefile new file mode 100644 index 000000000..ea465031b --- /dev/null +++ b/certbot-dns-google/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-google +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-google/docs/api.rst b/certbot-dns-google/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-google/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-google/docs/api/dns_google.rst b/certbot-dns-google/docs/api/dns_google.rst new file mode 100644 index 000000000..6f5459934 --- /dev/null +++ b/certbot-dns-google/docs/api/dns_google.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_google.dns_google` +------------------------------------ + +.. automodule:: certbot_dns_google.dns_google + :members: diff --git a/certbot-dns-google/docs/conf.py b/certbot-dns-google/docs/conf.py new file mode 100644 index 000000000..4ff1af1d1 --- /dev/null +++ b/certbot-dns-google/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-google documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 15:47:49 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-google' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-googledoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-google.tex', u'certbot-dns-google Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-google', u'certbot-dns-google Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-google', u'certbot-dns-google Documentation', + author, 'certbot-dns-google', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-google/docs/index.rst b/certbot-dns-google/docs/index.rst new file mode 100644 index 000000000..a8a322f97 --- /dev/null +++ b/certbot-dns-google/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-google documentation master file, created by + sphinx-quickstart on Wed May 10 15:47:49 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-google's documentation! +============================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_google + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-google/docs/make.bat b/certbot-dns-google/docs/make.bat new file mode 100644 index 000000000..181c12699 --- /dev/null +++ b/certbot-dns-google/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-google + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-google/setup.cfg b/certbot-dns-google/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-google/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-google/setup.py b/certbot-dns-google/setup.py new file mode 100644 index 000000000..043a9ded1 --- /dev/null +++ b/certbot-dns-google/setup.py @@ -0,0 +1,70 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.15.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + 'google-api-python-client', + 'mock', + 'oauth2client', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-google', + version=version, + description="Google Cloud DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-google = certbot_dns_google.dns_google:Authenticator', + ], + }, + test_suite='certbot_dns_google', +) diff --git a/certbot/cli.py b/certbot/cli.py index 250c0d2af..b0356db23 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1225,6 +1225,8 @@ def _plugins_parsing(helpful, plugins): help='Obtain certs using a DNS TXT record (if you are using Cloudflare for DNS).') helpful.add(["plugins", "certonly"], "--dns-digitalocean", action="store_true", help='Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).') + helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", + help='Obtain certs using a DNS TXT record (if you are using Google Cloud DNS).') # things should not be reorder past/pre this comment: # plugins_group should be displayed in --help before plugin diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index cff0ed157..f048fa8c4 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -30,6 +30,7 @@ class PluginEntryPoint(object): "certbot-apache", "certbot-dns-cloudflare", "certbot-dns-digitalocean", + "certbot-dns-google", "certbot-nginx", ] """Distributions for which prefix will be omitted.""" diff --git a/certbot/plugins/dns_common.py b/certbot/plugins/dns_common.py index de8f695c7..f71905d79 100644 --- a/certbot/plugins/dns_common.py +++ b/certbot/plugins/dns_common.py @@ -30,9 +30,9 @@ class DNSAuthenticator(common.Plugin): self._attempt_cleanup = False @classmethod - def add_parser_arguments(cls, add): + def add_parser_arguments(cls, add, default_propagation_seconds=10): # pylint: disable=arguments-differ add('propagation-seconds', - default=10, + default=default_propagation_seconds, type=int, help='The number of seconds to wait for DNS to propagate before asking the ACME server ' 'to verify the DNS record.') diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index c243accca..ef8d70009 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -133,7 +133,8 @@ def choose_plugin(prepared, question): else: return None -noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-digitalocean"] +noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-digitalocean", + "dns-google"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -241,6 +242,8 @@ def cli_plugin_requests(config): req_auth = set_configurator(req_auth, "dns-cloudflare") if config.dns_digitalocean: req_auth = set_configurator(req_auth, "dns-digitalocean") + if config.dns_google: + req_auth = set_configurator(req_auth, "dns-google") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) return req_auth, req_inst diff --git a/tools/venv.sh b/tools/venv.sh index da0a07830..ca34d8db8 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -16,6 +16,7 @@ fi -e certbot-apache \ -e certbot-dns-cloudflare \ -e certbot-dns-digitalocean \ + -e certbot-dns-google \ -e certbot-nginx \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tools/venv3.sh b/tools/venv3.sh index caf6bd8ba..931852165 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -15,6 +15,7 @@ fi -e certbot-apache \ -e certbot-dns-cloudflare \ -e certbot-dns-digitalocean \ + -e certbot-dns-google \ -e certbot-nginx \ -e letshelp-certbot \ -e certbot-compatibility-test diff --git a/tox.cover.sh b/tox.cover.sh index afe0e03ed..1ac248796 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_digitalocean certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_digitalocean certbot_dns_google certbot_nginx letshelp_certbot" else pkgs="$@" fi @@ -25,6 +25,8 @@ cover () { min=98 elif [ "$1" = "certbot_dns_digitalocean" ]; then min=98 + elif [ "$1" = "certbot_dns_google" ]; then + min=99 elif [ "$1" = "certbot_nginx" ]; then min=97 elif [ "$1" = "letshelp_certbot" ]; then diff --git a/tox.ini b/tox.ini index 656c9c67b..640bb0359 100644 --- a/tox.ini +++ b/tox.ini @@ -35,8 +35,10 @@ dns_plugin_commands = nosetests -v certbot_dns_cloudflare --processes=-1 pip install -e certbot-dns-digitalocean nosetests -v certbot_dns_digitalocean --processes=-1 -dns_plugin_install_args = -e certbot-dns-cloudflare -e certbot-dns-digitalocean -dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean + pip install -e certbot-dns-google + nosetests -v certbot_dns_google --processes=-1 +dns_plugin_install_args = -e certbot-dns-cloudflare -e certbot-dns-digitalocean -e certbot-dns-google +dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-google/certbot_dns_google compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test From 686f5d6c81c3797e30c54c2bb34aa407121e8ca8 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 17 May 2017 13:46:52 -0700 Subject: [PATCH 79/96] Move 'jwk' and 'alg' fields to protected header. (#4677) * Move 'jwk' and 'alg' fields to protected header. Previously, these were in the unprotected JWS header, which Boulder currently allows. However, the next version of the spec doesn't allow anything in the unprotected header. Moving these fields now allows server implementers who are implementing the Certbot/Boulder version of ACME (https://github.com/letsencrypt/boulder/blob/master/docs/acme-divergences.md) to use JOSE libraries that don't support unprotected headers. Fixes #4417. * Only protect existing headers. --- acme/acme/jose/jws.py | 3 ++- acme/acme/jws.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acme/acme/jose/jws.py b/acme/acme/jose/jws.py index 8fa8d7670..5f446e4b1 100644 --- a/acme/acme/jose/jws.py +++ b/acme/acme/jose/jws.py @@ -222,7 +222,8 @@ class Signature(json_util.JSONObjectWithFields): protected_params = {} for header in protect: - protected_params[header] = header_params.pop(header) + if header in header_params: + protected_params[header] = header_params.pop(header) if protected_params: # pylint: disable=star-args protected = cls.header_cls(**protected_params).json_dumps() diff --git a/acme/acme/jws.py b/acme/acme/jws.py index 79e96edcb..f9b81749a 100644 --- a/acme/acme/jws.py +++ b/acme/acme/jws.py @@ -49,6 +49,6 @@ class JWS(jose.JWS): # jwk field if kid is not provided. include_jwk = kid is None return super(JWS, cls).sign(payload, key=key, alg=alg, - protect=frozenset(['nonce', 'url', 'kid']), + protect=frozenset(['nonce', 'url', 'kid', 'jwk', 'alg']), nonce=nonce, url=url, kid=kid, include_jwk=include_jwk) From 10bac107ee6e7a565ce8bb60c1c6b37661012ed2 Mon Sep 17 00:00:00 2001 From: Jacob Hoffman-Andrews Date: Wed, 17 May 2017 14:24:59 -0700 Subject: [PATCH 80/96] Add an account deactivate utility script. (#4254) * Add an account deactivate utility script. This is handy if you created an account with a tool other than Certbot, and want to deactivate the account. * Move deactivate.py to tools. * Add test for ConflictError. * Fix lint error. * Document how to set server. --- acme/acme/client.py | 3 +++ acme/acme/client_test.py | 6 +++++ acme/acme/errors.py | 11 +++++++++ tools/deactivate.py | 49 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 tools/deactivate.py diff --git a/acme/acme/client.py b/acme/acme/client.py index a069876d5..9455159de 100644 --- a/acme/acme/client.py +++ b/acme/acme/client.py @@ -564,6 +564,9 @@ class ClientNetwork(object): # pylint: disable=too-many-instance-attributes except ValueError: jobj = None + if response.status_code == 409: + raise errors.ConflictError(response.headers.get('Location')) + if not response.ok: if jobj is not None: if response_ct != cls.JSON_ERROR_CONTENT_TYPE: diff --git a/acme/acme/client_test.py b/acme/acme/client_test.py index 09bb38c00..cd1a90645 100644 --- a/acme/acme/client_test.py +++ b/acme/acme/client_test.py @@ -513,6 +513,12 @@ class ClientNetworkTest(unittest.TestCase): self.assertEqual( self.response, self.net._check_response(self.response)) + def test_check_response_conflict(self): + self.response.ok = False + self.response.status_code = 409 + # pylint: disable=protected-access + self.assertRaises(errors.ConflictError, self.net._check_response, self.response) + def test_check_response_jobj(self): self.response.json.return_value = {} for response_ct in [self.net.JSON_CONTENT_TYPE, 'foo']: diff --git a/acme/acme/errors.py b/acme/acme/errors.py index 7446b60fc..9d991fd75 100644 --- a/acme/acme/errors.py +++ b/acme/acme/errors.py @@ -82,3 +82,14 @@ class PollError(ClientError): def __repr__(self): return '{0}(exhausted={1!r}, updated={2!r})'.format( self.__class__.__name__, self.exhausted, self.updated) + +class ConflictError(ClientError): + """Error for when the server returns a 409 (Conflict) HTTP status. + + In the version of ACME implemented by Boulder, this is used to find an + account if you only have the private key, but don't know the account URL. + """ + def __init__(self, location): + self.location = location + super(ConflictError, self).__init__() + diff --git a/tools/deactivate.py b/tools/deactivate.py new file mode 100644 index 000000000..5facc8436 --- /dev/null +++ b/tools/deactivate.py @@ -0,0 +1,49 @@ +""" +Given an ACME account key as input, deactivate the account. + +This can be useful if you created an account with a non-Certbot client and now +want to deactivate it. + +Private key should be in PKCS#8 PEM form. + +To provide the URL for the ACME server you want to use, set it in the $DIRECTORY +environment variable, e.g.: + +DIRECTORY=https://acme-staging.api.letsencrypt.org/directory python \ + deactivate.py private_key.pem +""" +import os +import sys + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization + +from acme import client as acme_client +from acme import errors as acme_errors +from acme import jose +from acme import messages + +DIRECTORY = os.getenv('DIRECTORY', 'http://localhost:4000/directory') + +if len(sys.argv) != 2: + print("Usage: python deactivate.py private_key.pem") + sys.exit(1) + +data = open(sys.argv[1], "r").read() +key = jose.JWKRSA(key=serialization.load_pem_private_key( + data, None, default_backend())) + +net = acme_client.ClientNetwork(key, verify_ssl=False, + user_agent="acme account deactivator") + +client = acme_client.Client(DIRECTORY, key=key, net=net) +try: + # We expect this to fail and give us a Conflict response with a Location + # header pointing at the account's URL. + client.register() +except acme_errors.ConflictError as e: + location = e.location +if location is None: + raise "Key was not previously registered (but now is)." +client.deactivate_registration(messages.RegistrationResource(uri=location)) From 462c0aba629766efc6d41945e61a858ed3de8734 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Wed, 17 May 2017 14:25:50 -0700 Subject: [PATCH 81/96] Modify special action types only once (#4656) --- certbot/tests/util_test.py | 1 + certbot/util.py | 32 ++++++++++++++++++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/certbot/tests/util_test.py b/certbot/tests/util_test.py index 3d51a1d96..3f6bd2a39 100644 --- a/certbot/tests/util_test.py +++ b/certbot/tests/util_test.py @@ -388,6 +388,7 @@ class AddDeprecatedArgumentTest(unittest.TestCase): with mock.patch("certbot.util.configargparse") as mock_configargparse: mock_configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE = typ() self._call("--old-option", 1) + self._call("--old-option2", 2) self.assertEqual( len(mock_configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE), 1) diff --git a/certbot/util.py b/certbot/util.py index bece91f01..9db9d7bf9 100644 --- a/certbot/util.py +++ b/certbot/util.py @@ -458,6 +458,13 @@ def safe_email(email): return False +class _ShowWarning(argparse.Action): + """Action to log a warning when an argument is used.""" + def __call__(self, unused1, unused2, unused3, option_string=None): + sys.stderr.write( + "Use of {0} is deprecated.\n".format(option_string)) + + def add_deprecated_argument(add_argument, argument_name, nargs): """Adds a deprecated argument with the name argument_name. @@ -471,20 +478,17 @@ def add_deprecated_argument(add_argument, argument_name, nargs): :param nargs: Value for nargs when adding the argument to argparse. """ - class ShowWarning(argparse.Action): - """Action to log a warning when an argument is used.""" - def __call__(self, unused1, unused2, unused3, option_string=None): - sys.stderr.write( - "Use of {0} is deprecated.\n".format(option_string)) - - # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was changed from a - # set to a tuple. - if isinstance(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE, set): - # pylint: disable=no-member - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add(ShowWarning) - else: - configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE += (ShowWarning,) - add_argument(argument_name, action=ShowWarning, + if _ShowWarning not in configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE: + # In version 0.12.0 ACTION_TYPES_THAT_DONT_NEED_A_VALUE was + # changed from a set to a tuple. + if isinstance(configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE, set): + # pylint: disable=no-member + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE.add( + _ShowWarning) + else: + configargparse.ACTION_TYPES_THAT_DONT_NEED_A_VALUE += ( + _ShowWarning,) + add_argument(argument_name, action=_ShowWarning, help=argparse.SUPPRESS, nargs=nargs) From 04759095c29f5667c6722eed592e0a6db57f4c84 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Thu, 18 May 2017 08:15:00 -0700 Subject: [PATCH 82/96] Fix example links (#4678) * fix example links * use single backticks not double --- docs/challenges.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/challenges.rst b/docs/challenges.rst index 0c923c45b..e45b9d852 100644 --- a/docs/challenges.rst +++ b/docs/challenges.rst @@ -58,7 +58,7 @@ HTTP-01 challenge: files in order to have them served by your existing web server. If you said your webroot for example.com was /var/www/example.com, then a file placed in /var/www/example.com/.well-known/acme-challenge/testfile should appear on - your web site at http://example.com/.well-known/acme-challenge/testfile (which you can test using a web browser). (A redirection to HTTPS + your web site at `http://example.com/.well-known/acme-challenge/testfile` (which you can test using a web browser). (A redirection to HTTPS is OK here and should not stop the challenge from working.) Note that you should *not* specify the .well-known/acme-challenge directory itself. Instead, you should specify the top level directory that web content is served from. @@ -70,7 +70,7 @@ HTTP-01 challenge: * (With manual plugin) You updated the webroot directory incorrectly If you used `--manual`, you need to know where you can put files in order to have them served by your existing web server. If you think your webroot for example.com is /var/www/example.com, then a file placed in /var/www/example.com/.well-known/acme-challenge/testfile should appear on - your web site at http://example.com/.well-known/acme-challenge/testfile. (A redirection to HTTPS + your web site at `http://example.com/.well-known/acme-challenge/testfile`. (A redirection to HTTPS is OK here and should not stop the challenge from working.) You should also make sure that you don't make a typo in the name of the file when creating it. * Your existing web server's configuration refuses to serve files From 7da53819682fe1b320e0e2890d4cc4be0a637303 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Thu, 18 May 2017 14:05:47 -0700 Subject: [PATCH 83/96] Common code for Lexicon-based DNS authenticators (#4583) Introduce abstract classes to provide base functionality for Lexicon-based DNS Authenticator plugins and corresponding test cases. --- certbot/plugins/dns_common_lexicon.py | 97 ++++++++++++++++ certbot/plugins/dns_common_lexicon_test.py | 27 +++++ certbot/plugins/dns_test_common_lexicon.py | 128 +++++++++++++++++++++ docs/api/plugins/dns_common_lexicon.rst | 5 + 4 files changed, 257 insertions(+) create mode 100644 certbot/plugins/dns_common_lexicon.py create mode 100644 certbot/plugins/dns_common_lexicon_test.py create mode 100644 certbot/plugins/dns_test_common_lexicon.py create mode 100644 docs/api/plugins/dns_common_lexicon.rst diff --git a/certbot/plugins/dns_common_lexicon.py b/certbot/plugins/dns_common_lexicon.py new file mode 100644 index 000000000..7a97fc950 --- /dev/null +++ b/certbot/plugins/dns_common_lexicon.py @@ -0,0 +1,97 @@ +"""Common code for DNS Authenticator Plugins built on Lexicon.""" + +import logging + +from requests.exceptions import HTTPError, RequestException + +from certbot import errors +from certbot.plugins import dns_common + +logger = logging.getLogger(__name__) + + +class LexiconClient(object): + """ + Encapsulates all communication with a DNS provider via Lexicon. + """ + + def __init__(self): + self.provider = None + + def add_txt_record(self, domain, record_name, record_content): + """ + Add a TXT record using the supplied information. + + :param str domain: The domain to use to look up the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :raises errors.PluginError: if an error occurs communicating with the DNS Provider API + """ + self._find_domain_id(domain) + + try: + self.provider.create_record(type='TXT', name=record_name, content=record_content) + except RequestException as e: + logger.debug('Encountered error adding TXT record: %s', e, exc_info=True) + raise errors.PluginError('Error adding TXT record: {0}'.format(e)) + + def del_txt_record(self, domain, record_name, record_content): + """ + Delete a TXT record using the supplied information. + + :param str domain: The domain to use to look up the managed zone. + :param str record_name: The record name (typically beginning with '_acme-challenge.'). + :param str record_content: The record content (typically the challenge validation). + :raises errors.PluginError: if an error occurs communicating with the DNS Provider API + """ + try: + self._find_domain_id(domain) + except errors.PluginError as e: + logger.debug('Encountered error finding domain_id during deletion: %s', e, + exc_info=True) + return + + try: + self.provider.delete_record(type='TXT', name=record_name, content=record_content) + except RequestException as e: + logger.debug('Encountered error deleting TXT record: %s', e, exc_info=True) + + def _find_domain_id(self, domain): + """ + Find the domain_id for a given domain. + + :param str domain: The domain for which to find the domain_id. + :raises errors.PluginError: if the domain_id cannot be found. + """ + + domain_name_guesses = dns_common.base_domain_name_guesses(domain) + + for domain_name in domain_name_guesses: + try: + self.provider.options['domain'] = domain_name + + self.provider.authenticate() + + return # If `authenticate` doesn't throw an exception, we've found the right name + except HTTPError as e: + result = self._handle_http_error(e, domain_name) + + if result: + raise result + except Exception as e: # pylint: disable=broad-except + result = self._handle_general_error(e, domain_name) + + if result: + raise result + + raise errors.PluginError('Unable to determine zone identifier for {0} using zone names: {1}' + .format(domain, domain_name_guesses)) + + def _handle_http_error(self, e, domain_name): + return errors.PluginError('Error determining zone identifier for {0}: {1}.' + .format(domain_name, e)) + + def _handle_general_error(self, e, domain_name): + if not str(e).startswith('No domain found'): + return errors.PluginError('Unexpected error determining zone identifier for {0}: {1}' + .format(domain_name, e)) diff --git a/certbot/plugins/dns_common_lexicon_test.py b/certbot/plugins/dns_common_lexicon_test.py new file mode 100644 index 000000000..986362ca9 --- /dev/null +++ b/certbot/plugins/dns_common_lexicon_test.py @@ -0,0 +1,27 @@ +"""Tests for certbot.plugins.dns_common_lexicon.""" + +import unittest + +import mock + +from certbot.plugins import dns_common_lexicon +from certbot.plugins import dns_test_common_lexicon + + +class LexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + + class _FakeLexiconClient(dns_common_lexicon.LexiconClient): + pass + + def setUp(self): + super(LexiconClientTest, self).setUp() + + self.client = LexiconClientTest._FakeLexiconClient() + self.provider_mock = mock.MagicMock() + + self.client.provider = self.provider_mock + + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot/plugins/dns_test_common_lexicon.py b/certbot/plugins/dns_test_common_lexicon.py new file mode 100644 index 000000000..f9c5735e8 --- /dev/null +++ b/certbot/plugins/dns_test_common_lexicon.py @@ -0,0 +1,128 @@ +"""Base test class for DNS authenticators built on Lexicon.""" + +import mock +from acme import jose +from requests.exceptions import HTTPError, RequestException + +from certbot import errors +from certbot.plugins import dns_test_common +from certbot.tests import util as test_util + +DOMAIN = 'example.com' +KEY = jose.JWKRSA.load(test_util.load_vector("rsa512_key.pem")) + +# These classes are intended to be subclassed/mixed in, so not all members are defined. +# pylint: disable=no-member + +class BaseLexiconAuthenticatorTest(dns_test_common.BaseAuthenticatorTest): + + def test_perform(self): + self.auth.perform([self.achall]) + + expected = [mock.call.add_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + def test_cleanup(self): + self.auth._attempt_cleanup = True # _attempt_cleanup | pylint: disable=protected-access + self.auth.cleanup([self.achall]) + + expected = [mock.call.del_txt_record(DOMAIN, '_acme-challenge.'+DOMAIN, mock.ANY)] + self.assertEqual(expected, self.mock_client.mock_calls) + + +class BaseLexiconClientTest(object): + DOMAIN_NOT_FOUND = Exception('No domain found') + GENERIC_ERROR = RequestException + LOGIN_ERROR = HTTPError('400 Client Error: ...') + UNKNOWN_LOGIN_ERROR = HTTPError('500 Surprise! Error: ...') + + record_prefix = "_acme-challenge" + record_name = record_prefix + "." + DOMAIN + record_content = "bar" + + def test_add_txt_record(self): + self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) + + self.provider_mock.create_record.assert_called_with(type='TXT', + name=self.record_name, + content=self.record_content) + + def test_add_txt_record_try_twice_to_find_domain(self): + self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, ''] + + self.client.add_txt_record(DOMAIN, self.record_name, self.record_content) + + self.provider_mock.create_record.assert_called_with(type='TXT', + name=self.record_name, + content=self.record_content) + + def test_add_txt_record_fail_to_find_domain(self): + self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, + self.DOMAIN_NOT_FOUND, + self.DOMAIN_NOT_FOUND,] + + self.assertRaises(errors.PluginError, + self.client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_add_txt_record_fail_to_authenticate(self): + self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR + + self.assertRaises(errors.PluginError, + self.client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_add_txt_record_fail_to_authenticate_with_unknown_error(self): + self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR + + self.assertRaises(errors.PluginError, + self.client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_add_txt_record_error_finding_domain(self): + self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR + + self.assertRaises(errors.PluginError, + self.client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_add_txt_record_error_adding_record(self): + self.provider_mock.create_record.side_effect = self.GENERIC_ERROR + + self.assertRaises(errors.PluginError, + self.client.add_txt_record, + DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record(self): + self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + self.provider_mock.delete_record.assert_called_with(type='TXT', + name=self.record_name, + content=self.record_content) + + def test_del_txt_record_fail_to_find_domain(self): + self.provider_mock.authenticate.side_effect = [self.DOMAIN_NOT_FOUND, + self.DOMAIN_NOT_FOUND, + self.DOMAIN_NOT_FOUND, ] + + self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_fail_to_authenticate(self): + self.provider_mock.authenticate.side_effect = self.LOGIN_ERROR + + self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_fail_to_authenticate_with_unknown_error(self): + self.provider_mock.authenticate.side_effect = self.UNKNOWN_LOGIN_ERROR + + self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_error_finding_domain(self): + self.provider_mock.authenticate.side_effect = self.GENERIC_ERROR + + self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) + + def test_del_txt_record_error_deleting_record(self): + self.provider_mock.delete_record.side_effect = self.GENERIC_ERROR + + self.client.del_txt_record(DOMAIN, self.record_name, self.record_content) diff --git a/docs/api/plugins/dns_common_lexicon.rst b/docs/api/plugins/dns_common_lexicon.rst new file mode 100644 index 000000000..a48166828 --- /dev/null +++ b/docs/api/plugins/dns_common_lexicon.rst @@ -0,0 +1,5 @@ +:mod:`certbot.plugins.dns_common_lexicon` +----------------------------------------- + +.. automodule:: certbot.plugins.dns_common_lexicon + :members: From 16d9537c418ebc3d0d1ce946bb61e04e9bf4dac8 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Thu, 18 May 2017 16:44:05 -0700 Subject: [PATCH 84/96] Moved files to 'certbot-route53' --- .gitignore => certbot-route53/.gitignore | 0 LICENSE => certbot-route53/LICENSE | 0 MANIFEST.in => certbot-route53/MANIFEST.in | 0 README.md => certbot-route53/README.md | 0 {certbot_route53 => certbot-route53/certbot_route53}/__init__.py | 0 .../certbot_route53}/authenticator.py | 0 sample-aws-policy.json => certbot-route53/sample-aws-policy.json | 0 setup.cfg => certbot-route53/setup.cfg | 0 setup.py => certbot-route53/setup.py | 0 .../tester.pkoch-macos_sierra.sh | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename .gitignore => certbot-route53/.gitignore (100%) rename LICENSE => certbot-route53/LICENSE (100%) rename MANIFEST.in => certbot-route53/MANIFEST.in (100%) rename README.md => certbot-route53/README.md (100%) rename {certbot_route53 => certbot-route53/certbot_route53}/__init__.py (100%) rename {certbot_route53 => certbot-route53/certbot_route53}/authenticator.py (100%) rename sample-aws-policy.json => certbot-route53/sample-aws-policy.json (100%) rename setup.cfg => certbot-route53/setup.cfg (100%) rename setup.py => certbot-route53/setup.py (100%) rename tester.pkoch-macos_sierra.sh => certbot-route53/tester.pkoch-macos_sierra.sh (100%) diff --git a/.gitignore b/certbot-route53/.gitignore similarity index 100% rename from .gitignore rename to certbot-route53/.gitignore diff --git a/LICENSE b/certbot-route53/LICENSE similarity index 100% rename from LICENSE rename to certbot-route53/LICENSE diff --git a/MANIFEST.in b/certbot-route53/MANIFEST.in similarity index 100% rename from MANIFEST.in rename to certbot-route53/MANIFEST.in diff --git a/README.md b/certbot-route53/README.md similarity index 100% rename from README.md rename to certbot-route53/README.md diff --git a/certbot_route53/__init__.py b/certbot-route53/certbot_route53/__init__.py similarity index 100% rename from certbot_route53/__init__.py rename to certbot-route53/certbot_route53/__init__.py diff --git a/certbot_route53/authenticator.py b/certbot-route53/certbot_route53/authenticator.py similarity index 100% rename from certbot_route53/authenticator.py rename to certbot-route53/certbot_route53/authenticator.py diff --git a/sample-aws-policy.json b/certbot-route53/sample-aws-policy.json similarity index 100% rename from sample-aws-policy.json rename to certbot-route53/sample-aws-policy.json diff --git a/setup.cfg b/certbot-route53/setup.cfg similarity index 100% rename from setup.cfg rename to certbot-route53/setup.cfg diff --git a/setup.py b/certbot-route53/setup.py similarity index 100% rename from setup.py rename to certbot-route53/setup.py diff --git a/tester.pkoch-macos_sierra.sh b/certbot-route53/tester.pkoch-macos_sierra.sh similarity index 100% rename from tester.pkoch-macos_sierra.sh rename to certbot-route53/tester.pkoch-macos_sierra.sh From 1ceefa794ef483ffc8d202c45d7bb86cc1e2cd36 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 19 May 2017 11:15:35 -0700 Subject: [PATCH 85/96] pin Sphinx<=1.5.6 (#4687) --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b7697dbd3..80050a2c9 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,8 @@ dev_extras = [ docs_extras = [ 'repoze.sphinx.autointerface', - 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + # autodoc_member_order = 'bysource', autodoc_default_flags, and #4686 + 'Sphinx >=1.0,<=1.5.6', 'sphinx_rtd_theme', ] From bbbfc473d32963c2a7b52a67bf23573c06d04cdf Mon Sep 17 00:00:00 2001 From: "Jeff R. Allen" Date: Fri, 19 May 2017 22:54:00 +0200 Subject: [PATCH 86/96] Handle mixed case domains in CSRs (#4685) Lowercase domains from CSR, just like the domains from the command line are. Fixes #4684 --- certbot/cli.py | 4 +++- certbot/tests/crypto_util_test.py | 4 ++-- certbot/tests/testdata/csr.der | Bin 353 -> 281 bytes certbot/tests/testdata/csr.pem | 14 ++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/certbot/cli.py b/certbot/cli.py index b0356db23..5053b77fa 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -620,7 +620,9 @@ class HelpfulArgumentParser(object): % parsed_args.csr[0]) parsed_args.actual_csr = (csr, typ) - csr_domains, config_domains = set(domains), set(parsed_args.domains) + + csr_domains = set([d.lower() for d in domains]) + config_domains = set(parsed_args.domains) if csr_domains != config_domains: raise errors.ConfigurationError( "Inconsistent domain requests:\nFrom the CSR: {0}\nFrom command line/config: {1}" diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index c83ad96b1..8adf753d6 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -163,7 +163,7 @@ class ImportCSRFileTest(unittest.TestCase): util.CSR(file=csrfile, data=data_pem, form="pem"), - ["example.com"],), + ["Example.com"],), self._call(csrfile, data)) def test_pem_csr(self): @@ -175,7 +175,7 @@ class ImportCSRFileTest(unittest.TestCase): util.CSR(file=csrfile, data=data, form="pem"), - ["example.com"],), + ["Example.com"],), self._call(csrfile, data)) def test_bad_csr(self): diff --git a/certbot/tests/testdata/csr.der b/certbot/tests/testdata/csr.der index 22900a6125aed31c9cfd1b6fe315c40dbd2fd99f..5c03f3a1114a9d0ad0899670c744efcbf0126685 100644 GIT binary patch delta 185 zcmaFJG?OXTpovk`pz#0`BZEP-A-4f18*?ZNn=q4OsG+cdAc(`k!xfyLo2naJl30>z zsAwP$66E5M@XRYoEy_zRQ3%gWNzW`PRtPAmRPZTDF%&Tn0x4nU;dY%^lNrolz{|#| z)#lOmotKf3o0Wmtk%3(UN-ar<_)FR?K>|cC!e8|frNn=P>HM{3%>zx6IfY6PO4sV zey#y8Tp=SjD+9A5Ly`NRca2;0qofT*IS%%v@A=!&;vT8<`n&vXYs=p)dvtP(4>Y_D gIO4 Date: Fri, 19 May 2017 16:23:53 -0700 Subject: [PATCH 87/96] CloudXNS DNS Authenticator (#4585) Implement an Authenticator which can fulfill a dns-01 challenge using the CloudXNS DNS API. Applicable only for domains using CloudXNS DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-cloudxns -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-cloudxns -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting cloudxns interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Domain name not registered to CloudXNS account. --- certbot-dns-cloudxns/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-cloudxns/MANIFEST.in | 3 + certbot-dns-cloudxns/README.rst | 1 + .../certbot_dns_cloudxns/__init__.py | 1 + .../certbot_dns_cloudxns/dns_cloudxns.py | 84 ++++++++ .../certbot_dns_cloudxns/dns_cloudxns_test.py | 54 +++++ certbot-dns-cloudxns/docs/.gitignore | 1 + certbot-dns-cloudxns/docs/Makefile | 20 ++ certbot-dns-cloudxns/docs/api.rst | 8 + .../docs/api/dns_cloudxns.rst | 5 + certbot-dns-cloudxns/docs/conf.py | 180 +++++++++++++++++ certbot-dns-cloudxns/docs/index.rst | 28 +++ certbot-dns-cloudxns/docs/make.bat | 36 ++++ certbot-dns-cloudxns/setup.cfg | 2 + certbot-dns-cloudxns/setup.py | 68 +++++++ certbot/cli.py | 2 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 6 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 31 ++- 22 files changed, 718 insertions(+), 9 deletions(-) create mode 100644 certbot-dns-cloudxns/LICENSE.txt create mode 100644 certbot-dns-cloudxns/MANIFEST.in create mode 100644 certbot-dns-cloudxns/README.rst create mode 100644 certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py create mode 100644 certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py create mode 100644 certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns_test.py create mode 100644 certbot-dns-cloudxns/docs/.gitignore create mode 100644 certbot-dns-cloudxns/docs/Makefile create mode 100644 certbot-dns-cloudxns/docs/api.rst create mode 100644 certbot-dns-cloudxns/docs/api/dns_cloudxns.rst create mode 100644 certbot-dns-cloudxns/docs/conf.py create mode 100644 certbot-dns-cloudxns/docs/index.rst create mode 100644 certbot-dns-cloudxns/docs/make.bat create mode 100644 certbot-dns-cloudxns/setup.cfg create mode 100644 certbot-dns-cloudxns/setup.py diff --git a/certbot-dns-cloudxns/LICENSE.txt b/certbot-dns-cloudxns/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-cloudxns/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-cloudxns/MANIFEST.in b/certbot-dns-cloudxns/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-cloudxns/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-cloudxns/README.rst b/certbot-dns-cloudxns/README.rst new file mode 100644 index 000000000..b127770df --- /dev/null +++ b/certbot-dns-cloudxns/README.rst @@ -0,0 +1 @@ +CloudXNS DNS Authenticator plugin for Certbot diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py new file mode 100644 index 000000000..8df02d0fa --- /dev/null +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/__init__.py @@ -0,0 +1 @@ +"""CloudXNS DNS Authenticator""" diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py new file mode 100644 index 000000000..2e9d23a88 --- /dev/null +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns.py @@ -0,0 +1,84 @@ +"""DNS Authenticator for CloudXNS DNS.""" +import logging + +import zope.interface +from lexicon.providers import cloudxns + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +ACCOUNT_URL = 'https://www.cloudxns.net/en/AccountManage/apimanage.html' + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for CloudXNS DNS + + This Authenticator uses the CloudXNS DNS API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using CloudXNS for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + add('credentials', help='CloudXNS credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the CloudXNS API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'CloudXNS credentials INI file', + { + 'api-key': 'API key for CloudXNS account, obtained from {0}'.format(ACCOUNT_URL), + 'secret-key': 'Secret key for CloudXNS account, obtained from {0}' + .format(ACCOUNT_URL) + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_cloudxns_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_cloudxns_client().del_txt_record(domain, validation_name, validation) + + def _get_cloudxns_client(self): + return _CloudXNSLexiconClient(self.credentials.conf('api-key'), + self.credentials.conf('secret-key'), + self.ttl) + + +class _CloudXNSLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the CloudXNS via Lexicon. + """ + + def __init__(self, api_key, secret_key, ttl): + super(_CloudXNSLexiconClient, self).__init__() + + self.provider = cloudxns.Provider({ + 'auth_username': api_key, + 'auth_token': secret_key, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + hint = None + if str(e).startswith('400 Client Error:'): + hint = 'Are your API key and Secret key values correct?' + + return errors.PluginError('Error determining zone identifier for {0}: {1}.{2}' + .format(domain_name, e, ' ({0})'.format(hint) if hint else '')) diff --git a/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns_test.py b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns_test.py new file mode 100644 index 000000000..c9bad23ab --- /dev/null +++ b/certbot-dns-cloudxns/certbot_dns_cloudxns/dns_cloudxns_test.py @@ -0,0 +1,54 @@ +"""Tests for certbot_dns_cloudxns.dns_cloudxns.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError, RequestException + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.tests import util as test_util + +DOMAIN_NOT_FOUND = Exception('No domain found') +GENERIC_ERROR = RequestException +LOGIN_ERROR = HTTPError('400 Client Error: ...') + +API_KEY = 'foo' +SECRET = 'bar' + + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_cloudxns.dns_cloudxns import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"cloudxns_api_key": API_KEY, "cloudxns_secret_key": SECRET}, path) + + self.config = mock.MagicMock(cloudxns_credentials=path, + cloudxns_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "cloudxns") + + self.mock_client = mock.MagicMock() + # _get_cloudxns_client | pylint: disable=protected-access + self.auth._get_cloudxns_client = mock.MagicMock(return_value=self.mock_client) + + +class CloudXNSLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + + def setUp(self): + from certbot_dns_cloudxns.dns_cloudxns import _CloudXNSLexiconClient + + self.client = _CloudXNSLexiconClient(API_KEY, SECRET, 0) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-cloudxns/docs/.gitignore b/certbot-dns-cloudxns/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-cloudxns/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-cloudxns/docs/Makefile b/certbot-dns-cloudxns/docs/Makefile new file mode 100644 index 000000000..ecda13dfe --- /dev/null +++ b/certbot-dns-cloudxns/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-cloudxns +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-cloudxns/docs/api.rst b/certbot-dns-cloudxns/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-cloudxns/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-cloudxns/docs/api/dns_cloudxns.rst b/certbot-dns-cloudxns/docs/api/dns_cloudxns.rst new file mode 100644 index 000000000..be794d1a0 --- /dev/null +++ b/certbot-dns-cloudxns/docs/api/dns_cloudxns.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_cloudxns.dns_cloudxns` +---------------------------------------- + +.. automodule:: certbot_dns_cloudxns.dns_cloudxns + :members: diff --git a/certbot-dns-cloudxns/docs/conf.py b/certbot-dns-cloudxns/docs/conf.py new file mode 100644 index 000000000..9e2f4c0e6 --- /dev/null +++ b/certbot-dns-cloudxns/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-cloudxns documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 16:05:50 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-cloudxns' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-cloudxnsdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-cloudxns.tex', u'certbot-dns-cloudxns Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-cloudxns', u'certbot-dns-cloudxns Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-cloudxns', u'certbot-dns-cloudxns Documentation', + author, 'certbot-dns-cloudxns', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-cloudxns/docs/index.rst b/certbot-dns-cloudxns/docs/index.rst new file mode 100644 index 000000000..41ea250cd --- /dev/null +++ b/certbot-dns-cloudxns/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-cloudxns documentation master file, created by + sphinx-quickstart on Wed May 10 16:05:50 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-cloudxns's documentation! +================================================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_cloudxns + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-cloudxns/docs/make.bat b/certbot-dns-cloudxns/docs/make.bat new file mode 100644 index 000000000..12f4f0de6 --- /dev/null +++ b/certbot-dns-cloudxns/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-cloudxns + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-cloudxns/setup.cfg b/certbot-dns-cloudxns/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-cloudxns/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-cloudxns/setup.py b/certbot-dns-cloudxns/setup.py new file mode 100644 index 000000000..4849007bc --- /dev/null +++ b/certbot-dns-cloudxns/setup.py @@ -0,0 +1,68 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.15.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + 'dns-lexicon', + 'mock', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-cloudxns', + version=version, + description="CloudXNS DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-cloudxns = certbot_dns_cloudxns.dns_cloudxns:Authenticator', + ], + }, + test_suite='certbot_dns_cloudxns', +) diff --git a/certbot/cli.py b/certbot/cli.py index 5053b77fa..95d61f7c2 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1225,6 +1225,8 @@ def _plugins_parsing(helpful, plugins): help='Obtain certs by placing files in a webroot directory.') helpful.add(["plugins", "certonly"], "--dns-cloudflare", action="store_true", help='Obtain certs using a DNS TXT record (if you are using Cloudflare for DNS).') + helpful.add(["plugins", "certonly"], "--dns-cloudxns", action="store_true", + help='Obtain certs using a DNS TXT record (if you are using CloudXNS for DNS).') helpful.add(["plugins", "certonly"], "--dns-digitalocean", action="store_true", help='Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).') helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index f048fa8c4..066941bf8 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -29,6 +29,7 @@ class PluginEntryPoint(object): "certbot", "certbot-apache", "certbot-dns-cloudflare", + "certbot-dns-cloudxns", "certbot-dns-digitalocean", "certbot-dns-google", "certbot-nginx", diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index ef8d70009..4f67782dc 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -133,8 +133,8 @@ def choose_plugin(prepared, question): else: return None -noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-digitalocean", - "dns-google"] +noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", + "dns-digitalocean", "dns-google"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -240,6 +240,8 @@ def cli_plugin_requests(config): req_auth = set_configurator(req_auth, "manual") if config.dns_cloudflare: req_auth = set_configurator(req_auth, "dns-cloudflare") + if config.dns_cloudxns: + req_auth = set_configurator(req_auth, "dns-cloudxns") if config.dns_digitalocean: req_auth = set_configurator(req_auth, "dns-digitalocean") if config.dns_google: diff --git a/tools/venv.sh b/tools/venv.sh index ca34d8db8..fbaf3d89e 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -15,6 +15,7 @@ fi -e .[dev,docs] \ -e certbot-apache \ -e certbot-dns-cloudflare \ + -e certbot-dns-cloudxns \ -e certbot-dns-digitalocean \ -e certbot-dns-google \ -e certbot-nginx \ diff --git a/tools/venv3.sh b/tools/venv3.sh index 931852165..4b8b1a5bf 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -14,6 +14,7 @@ fi -e .[dev,docs] \ -e certbot-apache \ -e certbot-dns-cloudflare \ + -e certbot-dns-cloudxns \ -e certbot-dns-digitalocean \ -e certbot-dns-google \ -e certbot-nginx \ diff --git a/tox.cover.sh b/tox.cover.sh index 1ac248796..4226b3cd4 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_digitalocean certbot_dns_google certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_google certbot_nginx letshelp_certbot" else pkgs="$@" fi @@ -23,6 +23,8 @@ cover () { min=100 elif [ "$1" = "certbot_dns_cloudflare" ]; then min=98 + elif [ "$1" = "certbot_dns_cloudxns" ]; then + min=99 elif [ "$1" = "certbot_dns_digitalocean" ]; then min=98 elif [ "$1" = "certbot_dns_google" ]; then diff --git a/tox.ini b/tox.ini index 640bb0359..84d2b7501 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,12 @@ dns_plugin_commands = dns_plugin_install_args = -e certbot-dns-cloudflare -e certbot-dns-digitalocean -e certbot-dns-google dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-digitalocean/certbot_dns_digitalocean certbot-dns-google/certbot_dns_google +lexicon_dns_plugin_commands = + pip install -e certbot-dns-cloudxns + nosetests -v certbot_dns_cloudxns --processes=-1 +lexicon_dns_plugin_install_args = -e certbot-dns-cloudxns +lexicon_dns_plugin_paths = certbot-dns-cloudxns/certbot_dns_cloudxns + compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test @@ -54,6 +60,8 @@ other_paths = letshelp-certbot/letshelp_certbot tests/lock_test.py commands = {[base]core_commands} {[base]plugin_commands} + {[base]dns_plugin_commands} + {[base]lexicon_dns_plugin_commands} {[base]other_commands} setenv = @@ -72,15 +80,26 @@ deps = py{26,27}-oldest: PyOpenSSL==0.13 py{26,27}-oldest: requests<=2.11.1 +[testenv:py26] +commands = + {[base]core_commands} + {[base]plugin_commands} + {[base]dns_plugin_commands} + {[base]other_commands} + +[testenv:py26-oldest] +commands = + {[testenv:py26]commands} + [testenv:py27_install] basepython = python2.7 commands = - {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]lexicon_dns_plugin_install_args} {[base]other_install_args} [testenv:cover] basepython = python2.7 commands = - {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]other_install_args} + {[base]pip_install} {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]lexicon_dns_plugin_install_args} {[base]other_install_args} ./tox.cover.sh [testenv:lint] @@ -90,15 +109,15 @@ basepython = python2.7 # duplicate code checking; if one of the commands fails, others will # continue, but tox return code will reflect previous error commands = - {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} - pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} + {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]lexicon_dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + pylint --reports=n --rcfile=.pylintrc {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]lexicon_dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:mypy] basepython = python3.4 commands = {[base]pip_install} mypy - {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} - mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} + {[base]pip_install} -q {[base]core_install_args} {[base]plugin_install_args} {[base]dns_plugin_install_args} {[base]lexicon_dns_plugin_install_args} {[base]compatibility_install_args} {[base]other_install_args} + mypy --py2 --ignore-missing-imports {[base]core_paths} {[base]plugin_paths} {[base]dns_plugin_paths} {[base]lexicon_dns_plugin_paths} {[base]compatibility_paths} {[base]other_paths} [testenv:apacheconftest] #basepython = python2.7 From c2b24702b75ca47198265619c5b94abfdb750013 Mon Sep 17 00:00:00 2001 From: Brad Warren Date: Fri, 19 May 2017 16:26:15 -0700 Subject: [PATCH 88/96] Fix defaults on older systems (#4691) * Creates SupportedChallengesAction This fixes #3987 as the call to set_by_default can be removed entirely. Additionally, logger.warning can be used rather than writing to stderr directly because #3184 has been resolved and we're guaranteed to having logging setup. * Move validator to SupportedChallengesAction supported_challenges_validator was moved to SupportedChallengesAction so argparse.ArgumentError can be easily used to provide nice error output. Tests in standalone_test.py were also updated so the module still has 100% test coverage. * Better document ArgumentError usage --- certbot/plugins/standalone.py | 76 ++++++++++++++++++------------ certbot/plugins/standalone_test.py | 40 ++++++++-------- 2 files changed, 65 insertions(+), 51 deletions(-) diff --git a/certbot/plugins/standalone.py b/certbot/plugins/standalone.py index 0c15930a3..68200666c 100644 --- a/certbot/plugins/standalone.py +++ b/certbot/plugins/standalone.py @@ -3,7 +3,6 @@ import argparse import collections import logging import socket -import sys import threading import OpenSSL @@ -13,7 +12,6 @@ import zope.interface from acme import challenges from acme import standalone as acme_standalone -from certbot import cli from certbot import errors from certbot import interfaces @@ -114,39 +112,57 @@ class ServerManager(object): SUPPORTED_CHALLENGES = [challenges.TLSSNI01, challenges.HTTP01] -def supported_challenges_validator(data): - """Supported challenges validator for the `argparse`. +class SupportedChallengesAction(argparse.Action): + """Action class for parsing standalone_supported_challenges.""" - It should be passed as `type` argument to `add_argument`. + def __call__(self, parser, namespace, values, option_string=None): + logger.warning( + "The standalone specific supported challenges flag is " + "deprecated. Please use the --preferred-challenges flag " + "instead.") + converted_values = self._convert_and_validate(values) + namespace.standalone_supported_challenges = converted_values - """ - if cli.set_by_cli("standalone_supported_challenges"): - sys.stderr.write( - "WARNING: The standalone specific " - "supported challenges flag is deprecated.\n" - "Please use the --preferred-challenges flag instead.\n") - challs = data.split(",") + def _convert_and_validate(self, data): + """Validate the value of supported challenges provided by the user. - # tls-sni-01 was dvsni during private beta - if "dvsni" in challs: - logger.info("Updating legacy standalone_supported_challenges value") - challs = [challenges.TLSSNI01.typ if chall == "dvsni" else chall - for chall in challs] - data = ",".join(challs) + References to "dvsni" are automatically converted to "tls-sni-01". - unrecognized = [name for name in challs - if name not in challenges.Challenge.TYPES] - if unrecognized: - raise argparse.ArgumentTypeError( - "Unrecognized challenges: {0}".format(", ".join(unrecognized))) + :param str data: comma delimited list of challenge types - choices = set(chall.typ for chall in SUPPORTED_CHALLENGES) - if not set(challs).issubset(choices): - raise argparse.ArgumentTypeError( - "Plugin does not support the following (valid) " - "challenges: {0}".format(", ".join(set(challs) - choices))) + :returns: validated and converted list of challenge types + :rtype: str - return data + """ + challs = data.split(",") + + # tls-sni-01 was dvsni during private beta + if "dvsni" in challs: + logger.info( + "Updating legacy standalone_supported_challenges value") + challs = [challenges.TLSSNI01.typ if chall == "dvsni" else chall + for chall in challs] + data = ",".join(challs) + + unrecognized = [name for name in challs + if name not in challenges.Challenge.TYPES] + + # argparse.ArgumentErrors raised out of argparse.Action objects + # are caught by argparse which prints usage information and the + # error that occurred before calling sys.exit. + if unrecognized: + raise argparse.ArgumentError( + self, + "Unrecognized challenges: {0}".format(", ".join(unrecognized))) + + choices = set(chall.typ for chall in SUPPORTED_CHALLENGES) + if not set(challs).issubset(choices): + raise argparse.ArgumentError( + self, + "Plugin does not support the following (valid) " + "challenges: {0}".format(", ".join(set(challs) - choices))) + + return data @zope.interface.implementer(interfaces.IAuthenticator) @@ -184,7 +200,7 @@ class Authenticator(common.Plugin): def add_parser_arguments(cls, add): add("supported-challenges", help=argparse.SUPPRESS, - type=supported_challenges_validator, + action=SupportedChallengesAction, default=",".join(chall.typ for chall in SUPPORTED_CHALLENGES)) @property diff --git a/certbot/plugins/standalone_test.py b/certbot/plugins/standalone_test.py index 83e0fcf7f..65d16c2f2 100644 --- a/certbot/plugins/standalone_test.py +++ b/certbot/plugins/standalone_test.py @@ -62,28 +62,26 @@ class ServerManagerTest(unittest.TestCase): self.assertEqual(self.mgr.running(), {}) -class SupportedChallengesValidatorTest(unittest.TestCase): - """Tests for plugins.standalone.supported_challenges_validator.""" +class SupportedChallengesActionTest(unittest.TestCase): + """Tests for plugins.standalone.SupportedChallengesAction.""" + + def _call(self, value): + with mock.patch("certbot.plugins.standalone.logger") as mock_logger: + # stderr is mocked to prevent potential argparse error + # output from cluttering test output + with mock.patch("sys.stderr"): + config = self.parser.parse_args([self.flag, value]) + + self.assertTrue(mock_logger.warning.called) + return getattr(config, self.dest) def setUp(self): - self.set_by_cli_patch = mock.patch( - "certbot.plugins.standalone.cli.set_by_cli") - self.stderr_patch = mock.patch("certbot.plugins.standalone.sys.stderr") + self.flag = "--standalone-supported-challenges" + self.dest = self.flag[2:].replace("-", "_") + self.parser = argparse.ArgumentParser() - self.set_by_cli_patch.start().return_value = True - self.stderr = self.stderr_patch.start() - - def tearDown(self): - self.set_by_cli_patch.stop() - self.stderr_patch.stop() - - def _call(self, data): - from certbot.plugins.standalone import ( - supported_challenges_validator) - return_value = supported_challenges_validator(data) - self.assertTrue(self.stderr.write.called) # pylint: disable=no-member - self.stderr.write.reset_mock() # pylint: disable=no-member - return return_value + from certbot.plugins.standalone import SupportedChallengesAction + self.parser.add_argument(self.flag, action=SupportedChallengesAction) def test_correct(self): self.assertEqual("tls-sni-01", self._call("tls-sni-01")) @@ -93,10 +91,10 @@ class SupportedChallengesValidatorTest(unittest.TestCase): def test_unrecognized(self): assert "foo" not in challenges.Challenge.TYPES - self.assertRaises(argparse.ArgumentTypeError, self._call, "foo") + self.assertRaises(SystemExit, self._call, "foo") def test_not_subset(self): - self.assertRaises(argparse.ArgumentTypeError, self._call, "dns") + self.assertRaises(SystemExit, self._call, "dns") def test_dvsni(self): self.assertEqual("tls-sni-01", self._call("dvsni")) From c3434bac26592585d12feb781a87f3e2be846e42 Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Fri, 19 May 2017 16:39:25 -0700 Subject: [PATCH 89/96] DNS plugins: fix whitespace issue in authenticator pydoc (#4699) --- certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py | 2 +- .../certbot_dns_digitalocean/dns_digitalocean.py | 2 +- certbot-dns-google/certbot_dns_google/dns_google.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py index 50040b916..6979581ee 100644 --- a/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py +++ b/certbot-dns-cloudflare/certbot_dns_cloudflare/dns_cloudflare.py @@ -16,7 +16,7 @@ ACCOUNT_URL = 'https://www.cloudflare.com/a/account/my-account' @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) class Authenticator(dns_common.DNSAuthenticator): - """DNS Authenticator for Cloudflare + """DNS Authenticator for Cloudflare This Authenticator uses the Cloudflare API to fulfill a dns-01 challenge. """ diff --git a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py index 30e0f2525..4bf279279 100644 --- a/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py +++ b/certbot-dns-digitalocean/certbot_dns_digitalocean/dns_digitalocean.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) class Authenticator(dns_common.DNSAuthenticator): - """DNS Authenticator for DigitalOcean + """DNS Authenticator for DigitalOcean This Authenticator uses the DigitalOcean API to fulfill a dns-01 challenge. """ diff --git a/certbot-dns-google/certbot_dns_google/dns_google.py b/certbot-dns-google/certbot_dns_google/dns_google.py index 3f7a8d9f2..908c020e1 100644 --- a/certbot-dns-google/certbot_dns_google/dns_google.py +++ b/certbot-dns-google/certbot_dns_google/dns_google.py @@ -20,7 +20,7 @@ PERMISSIONS_URL = 'https://cloud.google.com/dns/access-control#permissions_and_r @zope.interface.implementer(interfaces.IAuthenticator) @zope.interface.provider(interfaces.IPluginFactory) class Authenticator(dns_common.DNSAuthenticator): - """DNS Authenticator for Google Cloud DNS + """DNS Authenticator for Google Cloud DNS This Authenticator uses the Google Cloud DNS API to fulfill a dns-01 challenge. """ From 75c91940af353a1c9c4255716363d9b45928ab72 Mon Sep 17 00:00:00 2001 From: Kevin Le Date: Mon, 22 May 2017 11:26:02 -0700 Subject: [PATCH 90/96] [#4382] Install git into Docker development file (#4703) * install git into Docker development file * moved git install command in Dockerfile-dev to same RUN --- Dockerfile-dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile-dev b/Dockerfile-dev index 607aa3441..581b58f11 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -23,7 +23,7 @@ WORKDIR /opt/certbot/src # TODO: Install Apache/Nginx for plugin development. COPY letsencrypt-auto-source/letsencrypt-auto /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto RUN /opt/certbot/src/letsencrypt-auto-source/letsencrypt-auto --os-packages-only && \ - apt-get install python3-dev -y && \ + apt-get install python3-dev git -y && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* \ /tmp/* \ From 26808790680c96b7a6aff48982a15b28730d7e41 Mon Sep 17 00:00:00 2001 From: Fabian Date: Mon, 22 May 2017 22:00:44 +0200 Subject: [PATCH 91/96] Print stdout when running a hook (#4167, #4487) (#4702) --- certbot/hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/certbot/hooks.py b/certbot/hooks.py index 75d7a3b20..b3c1fc3e2 100644 --- a/certbot/hooks.py +++ b/certbot/hooks.py @@ -126,11 +126,13 @@ def execute(shell_cmd): cmd = Popen(shell_cmd, shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True) out, err = cmd.communicate() + base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) + if out: + logger.info('Output from %s:\n%s', base_cmd, out) if cmd.returncode != 0: logger.error('Hook command "%s" returned error code %d', shell_cmd, cmd.returncode) if err: - base_cmd = os.path.basename(shell_cmd.split(None, 1)[0]) logger.error('Error output from %s:\n%s', base_cmd, err) return (err, out) From 42c0117c16aa5145db41d14c0f1d1e59e6c12505 Mon Sep 17 00:00:00 2001 From: Aaron Cohen Date: Mon, 22 May 2017 14:43:08 -0700 Subject: [PATCH 92/96] Domain change wording (#4709) * Change wording of renew with new domains msg to allow clearer display. * Further improve domain change message formatting. * Fix text formatting tests --- certbot/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/certbot/main.py b/certbot/main.py index eafcf49dd..50dad8d1e 100644 --- a/certbot/main.py +++ b/certbot/main.py @@ -254,12 +254,14 @@ def _ask_user_to_confirm_new_names(config, new_domains, certname, old_domains): """ if config.renew_with_new_domains: return - msg = ("Confirm that you intend to update certificate {0} " - "to include domains {1}. Note that it previously " - "contained domains {2}.".format( + + msg = ("You are updating certificate {0} to include domains: {1}{br}{br}" + "It previously included domains: {2}{br}{br}" + "Did you intend to make this change?".format( certname, - new_domains, - old_domains)) + ", ".join(new_domains), + ", ".join(old_domains), + br=os.linesep)) obj = zope.component.getUtility(interfaces.IDisplay) if not obj.yesno(msg, "Update cert", "Cancel", default=True): raise errors.ConfigurationError("Specified mismatched cert name and domains.") From fb0287726827181859ed792b04bbcae9e5ca5f8f Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Mon, 22 May 2017 17:06:04 -0700 Subject: [PATCH 93/96] DNSimple DNS Authenticator (#4587) Implement an Authenticator which can fulfill a dns-01 challenge using the DNSimple DNS API. Applicable only for domains using DNSimple DNS. Testing Done: * `tox -e py27` * `tox -e lint` * Manual testing: * Used `certbot certonly --dns-dnsimple -d`, specifying a credentials file as a command line argument. Verified that a certificate was successfully obtained without user interaction. * Used `certbot certonly --dns-dnsimple -d`, without specifying a credentials file as a command line argument. Verified that the user was prompted and that a certificate was successfully obtained. * Used `certbot certonly -d`. Verified that the user was prompted for a credentials file after selecting dnsimple interactively and that a certificate was successfully obtained. * Used `certbot renew --force-renewal`. Verified that certificates were renewed without user interaction. * Negative testing: * Path to non-existent credentials file. * Credentials file with unsafe permissions (644). * Path to credentials file with an invalid token. * Path to credentials file without a token. * Domain name not registered to DNSimple account. --- certbot-dns-dnsimple/LICENSE.txt | 190 ++++++++++++++++++ certbot-dns-dnsimple/MANIFEST.in | 3 + certbot-dns-dnsimple/README.rst | 1 + .../certbot_dns_dnsimple/__init__.py | 1 + .../certbot_dns_dnsimple/dns_dnsimple.py | 79 ++++++++ .../certbot_dns_dnsimple/dns_dnsimple_test.py | 51 +++++ certbot-dns-dnsimple/docs/.gitignore | 1 + certbot-dns-dnsimple/docs/Makefile | 20 ++ certbot-dns-dnsimple/docs/api.rst | 8 + .../docs/api/dns_dnsimple.rst | 5 + certbot-dns-dnsimple/docs/conf.py | 180 +++++++++++++++++ certbot-dns-dnsimple/docs/index.rst | 28 +++ certbot-dns-dnsimple/docs/make.bat | 36 ++++ certbot-dns-dnsimple/setup.cfg | 2 + certbot-dns-dnsimple/setup.py | 68 +++++++ certbot/cli.py | 2 + certbot/plugins/disco.py | 1 + certbot/plugins/selection.py | 4 +- tools/venv.sh | 1 + tools/venv3.sh | 1 + tox.cover.sh | 4 +- tox.ini | 6 +- 22 files changed, 688 insertions(+), 4 deletions(-) create mode 100644 certbot-dns-dnsimple/LICENSE.txt create mode 100644 certbot-dns-dnsimple/MANIFEST.in create mode 100644 certbot-dns-dnsimple/README.rst create mode 100644 certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py create mode 100644 certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py create mode 100644 certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple_test.py create mode 100644 certbot-dns-dnsimple/docs/.gitignore create mode 100644 certbot-dns-dnsimple/docs/Makefile create mode 100644 certbot-dns-dnsimple/docs/api.rst create mode 100644 certbot-dns-dnsimple/docs/api/dns_dnsimple.rst create mode 100644 certbot-dns-dnsimple/docs/conf.py create mode 100644 certbot-dns-dnsimple/docs/index.rst create mode 100644 certbot-dns-dnsimple/docs/make.bat create mode 100644 certbot-dns-dnsimple/setup.cfg create mode 100644 certbot-dns-dnsimple/setup.py diff --git a/certbot-dns-dnsimple/LICENSE.txt b/certbot-dns-dnsimple/LICENSE.txt new file mode 100644 index 000000000..981c46c9f --- /dev/null +++ b/certbot-dns-dnsimple/LICENSE.txt @@ -0,0 +1,190 @@ + Copyright 2015 Electronic Frontier Foundation and others + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/certbot-dns-dnsimple/MANIFEST.in b/certbot-dns-dnsimple/MANIFEST.in new file mode 100644 index 000000000..18f018c08 --- /dev/null +++ b/certbot-dns-dnsimple/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE.txt +include README.rst +recursive-include docs * diff --git a/certbot-dns-dnsimple/README.rst b/certbot-dns-dnsimple/README.rst new file mode 100644 index 000000000..5edb6bf4b --- /dev/null +++ b/certbot-dns-dnsimple/README.rst @@ -0,0 +1 @@ +DNSimple DNS Authenticator plugin for Certbot diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py new file mode 100644 index 000000000..1d6747249 --- /dev/null +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/__init__.py @@ -0,0 +1 @@ +"""DNSimple DNS Authenticator""" diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py new file mode 100644 index 000000000..f489f889a --- /dev/null +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple.py @@ -0,0 +1,79 @@ +"""DNS Authenticator for DNSimple DNS.""" +import logging + +import zope.interface +from lexicon.providers import dnsimple + +from certbot import errors +from certbot import interfaces +from certbot.plugins import dns_common +from certbot.plugins import dns_common_lexicon + +logger = logging.getLogger(__name__) + +ACCOUNT_URL = 'https://dnsimple.com/user' + + +@zope.interface.implementer(interfaces.IAuthenticator) +@zope.interface.provider(interfaces.IPluginFactory) +class Authenticator(dns_common.DNSAuthenticator): + """DNS Authenticator for DNSimple + + This Authenticator uses the DNSimple v2 API to fulfill a dns-01 challenge. + """ + + description = 'Obtain certs using a DNS TXT record (if you are using DNSimple for DNS).' + ttl = 60 + + def __init__(self, *args, **kwargs): + super(Authenticator, self).__init__(*args, **kwargs) + self.credentials = None + + @classmethod + def add_parser_arguments(cls, add): # pylint: disable=arguments-differ + super(Authenticator, cls).add_parser_arguments(add, default_propagation_seconds=30) + add('credentials', help='DNSimple credentials INI file.') + + def more_info(self): # pylint: disable=missing-docstring,no-self-use + return 'This plugin configures a DNS TXT record to respond to a dns-01 challenge using ' + \ + 'the DNSimple API.' + + def _setup_credentials(self): + self.credentials = self._configure_credentials( + 'credentials', + 'DNSimple credentials INI file', + { + 'token': 'User access token for DNSimple v2 API. (See {0}.)'.format(ACCOUNT_URL) + } + ) + + def _perform(self, domain, validation_name, validation): + self._get_dnsimple_client().add_txt_record(domain, validation_name, validation) + + def _cleanup(self, domain, validation_name, validation): + self._get_dnsimple_client().del_txt_record(domain, validation_name, validation) + + def _get_dnsimple_client(self): + return _DNSimpleLexiconClient(self.credentials.conf('token'), self.ttl) + + +class _DNSimpleLexiconClient(dns_common_lexicon.LexiconClient): + """ + Encapsulates all communication with the DNSimple via Lexicon. + """ + + def __init__(self, token, ttl): + super(_DNSimpleLexiconClient, self).__init__() + + self.provider = dnsimple.Provider({ + 'auth_token': token, + 'ttl': ttl, + }) + + def _handle_http_error(self, e, domain_name): + hint = None + if str(e).startswith('401 Client Error: Unauthorized for url:'): + hint = 'Is your API token value correct?' + + return errors.PluginError('Error determining zone identifier for {0}: {1}.{2}' + .format(domain_name, e, ' ({0})'.format(hint) if hint else '')) diff --git a/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple_test.py b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple_test.py new file mode 100644 index 000000000..d8f3a23ea --- /dev/null +++ b/certbot-dns-dnsimple/certbot_dns_dnsimple/dns_dnsimple_test.py @@ -0,0 +1,51 @@ +"""Tests for certbot_dns_dnsimple.dns_dnsimple.""" + +import os +import unittest + +import mock +from requests.exceptions import HTTPError + +from certbot.plugins import dns_test_common +from certbot.plugins import dns_test_common_lexicon +from certbot.tests import util as test_util + +TOKEN = 'foo' + + +class AuthenticatorTest(test_util.TempDirTestCase, + dns_test_common_lexicon.BaseLexiconAuthenticatorTest): + + def setUp(self): + super(AuthenticatorTest, self).setUp() + + from certbot_dns_dnsimple.dns_dnsimple import Authenticator + + path = os.path.join(self.tempdir, 'file.ini') + dns_test_common.write({"dnsimple_token": TOKEN}, path) + + self.config = mock.MagicMock(dnsimple_credentials=path, + dnsimple_propagation_seconds=0) # don't wait during tests + + self.auth = Authenticator(self.config, "dnsimple") + + self.mock_client = mock.MagicMock() + # _get_dnsimple_client | pylint: disable=protected-access + self.auth._get_dnsimple_client = mock.MagicMock(return_value=self.mock_client) + + +class DNSimpleLexiconClientTest(unittest.TestCase, dns_test_common_lexicon.BaseLexiconClientTest): + + LOGIN_ERROR = HTTPError('401 Client Error: Unauthorized for url: ...') + + def setUp(self): + from certbot_dns_dnsimple.dns_dnsimple import _DNSimpleLexiconClient + + self.client = _DNSimpleLexiconClient(TOKEN, 0) + + self.provider_mock = mock.MagicMock() + self.client.provider = self.provider_mock + + +if __name__ == "__main__": + unittest.main() # pragma: no cover diff --git a/certbot-dns-dnsimple/docs/.gitignore b/certbot-dns-dnsimple/docs/.gitignore new file mode 100644 index 000000000..ba65b13af --- /dev/null +++ b/certbot-dns-dnsimple/docs/.gitignore @@ -0,0 +1 @@ +/_build/ diff --git a/certbot-dns-dnsimple/docs/Makefile b/certbot-dns-dnsimple/docs/Makefile new file mode 100644 index 000000000..13a07c00d --- /dev/null +++ b/certbot-dns-dnsimple/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = certbot-dns-dnsimple +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/certbot-dns-dnsimple/docs/api.rst b/certbot-dns-dnsimple/docs/api.rst new file mode 100644 index 000000000..8668ec5d8 --- /dev/null +++ b/certbot-dns-dnsimple/docs/api.rst @@ -0,0 +1,8 @@ +================= +API Documentation +================= + +.. toctree:: + :glob: + + api/** diff --git a/certbot-dns-dnsimple/docs/api/dns_dnsimple.rst b/certbot-dns-dnsimple/docs/api/dns_dnsimple.rst new file mode 100644 index 000000000..b0544107b --- /dev/null +++ b/certbot-dns-dnsimple/docs/api/dns_dnsimple.rst @@ -0,0 +1,5 @@ +:mod:`certbot_dns_dnsimple.dns_dnsimple` +---------------------------------------- + +.. automodule:: certbot_dns_dnsimple.dns_dnsimple + :members: diff --git a/certbot-dns-dnsimple/docs/conf.py b/certbot-dns-dnsimple/docs/conf.py new file mode 100644 index 000000000..da692fb9e --- /dev/null +++ b/certbot-dns-dnsimple/docs/conf.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# +# certbot-dns-dnsimple documentation build configuration file, created by +# sphinx-quickstart on Wed May 10 18:23:41 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ['sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode'] + +autodoc_member_order = 'bysource' +autodoc_default_flags = ['show-inheritance', 'private-members'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'certbot-dns-dnsimple' +copyright = u'2017, Certbot Project' +author = u'Certbot Project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = u'0' +# The full version, including alpha/beta/rc tags. +release = u'0' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +default_role = 'py:obj' + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# + +# http://docs.readthedocs.org/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs +# on_rtd is whether we are on readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +if not on_rtd: # only import and set the theme if we're building docs locally + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'certbot-dns-dnsimpledoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'certbot-dns-dnsimple.tex', u'certbot-dns-dnsimple Documentation', + u'Certbot Project', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'certbot-dns-dnsimple', u'certbot-dns-dnsimple Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'certbot-dns-dnsimple', u'certbot-dns-dnsimple Documentation', + author, 'certbot-dns-dnsimple', 'One line description of project.', + 'Miscellaneous'), +] + + + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/', None), + 'acme': ('https://acme-python.readthedocs.org/en/latest/', None), + 'certbot': ('https://certbot.eff.org/docs/', None), +} diff --git a/certbot-dns-dnsimple/docs/index.rst b/certbot-dns-dnsimple/docs/index.rst new file mode 100644 index 000000000..4ff1e59eb --- /dev/null +++ b/certbot-dns-dnsimple/docs/index.rst @@ -0,0 +1,28 @@ +.. certbot-dns-dnsimple documentation master file, created by + sphinx-quickstart on Wed May 10 18:23:41 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to certbot-dns-dnsimple's documentation! +================================================ + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +.. toctree:: + :maxdepth: 1 + + api + +.. automodule:: certbot_dns_dnsimple + :members: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/certbot-dns-dnsimple/docs/make.bat b/certbot-dns-dnsimple/docs/make.bat new file mode 100644 index 000000000..78e867256 --- /dev/null +++ b/certbot-dns-dnsimple/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=certbot-dns-dnsimple + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/certbot-dns-dnsimple/setup.cfg b/certbot-dns-dnsimple/setup.cfg new file mode 100644 index 000000000..2a9acf13d --- /dev/null +++ b/certbot-dns-dnsimple/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = 1 diff --git a/certbot-dns-dnsimple/setup.py b/certbot-dns-dnsimple/setup.py new file mode 100644 index 000000000..bfa89ea25 --- /dev/null +++ b/certbot-dns-dnsimple/setup.py @@ -0,0 +1,68 @@ +import sys + +from setuptools import setup +from setuptools import find_packages + + +version = '0.15.0.dev0' + +# Please update tox.ini when modifying dependency version requirements +install_requires = [ + 'acme=={0}'.format(version), + 'certbot=={0}'.format(version), + 'dns-lexicon', + 'mock', + # For pkg_resources. >=1.0 so pip resolves it to a version cryptography + # will tolerate; see #2599: + 'setuptools>=1.0', + 'zope.interface', +] + +docs_extras = [ + 'Sphinx>=1.0', # autodoc_member_order = 'bysource', autodoc_default_flags + 'sphinx_rtd_theme', +] + +setup( + name='certbot-dns-dnsimple', + version=version, + description="DNSimple DNS Authenticator plugin for Certbot", + url='https://github.com/certbot/certbot', + author="Certbot Project", + author_email='client-dev@letsencrypt.org', + license='Apache License 2.0', + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Plugins', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: Apache Software License', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Security', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Networking', + 'Topic :: System :: Systems Administration', + 'Topic :: Utilities', + ], + + packages=find_packages(), + include_package_data=True, + install_requires=install_requires, + extras_require={ + 'docs': docs_extras, + }, + entry_points={ + 'certbot.plugins': [ + 'dns-dnsimple = certbot_dns_dnsimple.dns_dnsimple:Authenticator', + ], + }, + test_suite='certbot_dns_dnsimple', +) diff --git a/certbot/cli.py b/certbot/cli.py index 95d61f7c2..ac3193773 100644 --- a/certbot/cli.py +++ b/certbot/cli.py @@ -1229,6 +1229,8 @@ def _plugins_parsing(helpful, plugins): help='Obtain certs using a DNS TXT record (if you are using CloudXNS for DNS).') helpful.add(["plugins", "certonly"], "--dns-digitalocean", action="store_true", help='Obtain certs using a DNS TXT record (if you are using DigitalOcean for DNS).') + helpful.add(["plugins", "certonly"], "--dns-dnsimple", action="store_true", + help='Obtain certs using a DNS TXT record (if you are using DNSimple for DNS).') helpful.add(["plugins", "certonly"], "--dns-google", action="store_true", help='Obtain certs using a DNS TXT record (if you are using Google Cloud DNS).') diff --git a/certbot/plugins/disco.py b/certbot/plugins/disco.py index 066941bf8..5347ab050 100644 --- a/certbot/plugins/disco.py +++ b/certbot/plugins/disco.py @@ -31,6 +31,7 @@ class PluginEntryPoint(object): "certbot-dns-cloudflare", "certbot-dns-cloudxns", "certbot-dns-digitalocean", + "certbot-dns-dnsimple", "certbot-dns-google", "certbot-nginx", ] diff --git a/certbot/plugins/selection.py b/certbot/plugins/selection.py index 4f67782dc..89fa0ab7b 100644 --- a/certbot/plugins/selection.py +++ b/certbot/plugins/selection.py @@ -134,7 +134,7 @@ def choose_plugin(prepared, question): return None noninstaller_plugins = ["webroot", "manual", "standalone", "dns-cloudflare", "dns-cloudxns", - "dns-digitalocean", "dns-google"] + "dns-digitalocean", "dns-dnsimple", "dns-google"] def record_chosen_plugins(config, plugins, auth, inst): "Update the config entries to reflect the plugins we actually selected." @@ -244,6 +244,8 @@ def cli_plugin_requests(config): req_auth = set_configurator(req_auth, "dns-cloudxns") if config.dns_digitalocean: req_auth = set_configurator(req_auth, "dns-digitalocean") + if config.dns_dnsimple: + req_auth = set_configurator(req_auth, "dns-dnsimple") if config.dns_google: req_auth = set_configurator(req_auth, "dns-google") logger.debug("Requested authenticator %s and installer %s", req_auth, req_inst) diff --git a/tools/venv.sh b/tools/venv.sh index fbaf3d89e..2a59737a7 100755 --- a/tools/venv.sh +++ b/tools/venv.sh @@ -17,6 +17,7 @@ fi -e certbot-dns-cloudflare \ -e certbot-dns-cloudxns \ -e certbot-dns-digitalocean \ + -e certbot-dns-dnsimple \ -e certbot-dns-google \ -e certbot-nginx \ -e letshelp-certbot \ diff --git a/tools/venv3.sh b/tools/venv3.sh index 4b8b1a5bf..a0c98126e 100755 --- a/tools/venv3.sh +++ b/tools/venv3.sh @@ -16,6 +16,7 @@ fi -e certbot-dns-cloudflare \ -e certbot-dns-cloudxns \ -e certbot-dns-digitalocean \ + -e certbot-dns-dnsimple \ -e certbot-dns-google \ -e certbot-nginx \ -e letshelp-certbot \ diff --git a/tox.cover.sh b/tox.cover.sh index 4226b3cd4..f7064f918 100755 --- a/tox.cover.sh +++ b/tox.cover.sh @@ -9,7 +9,7 @@ # -e makes sure we fail fast and don't submit coveralls submit if [ "xxx$1" = "xxx" ]; then - pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_google certbot_nginx letshelp_certbot" + pkgs="certbot acme certbot_apache certbot_dns_cloudflare certbot_dns_cloudxns certbot_dns_digitalocean certbot_dns_dnsimple certbot_dns_google certbot_nginx letshelp_certbot" else pkgs="$@" fi @@ -27,6 +27,8 @@ cover () { min=99 elif [ "$1" = "certbot_dns_digitalocean" ]; then min=98 + elif [ "$1" = "certbot_dns_dnsimple" ]; then + min=98 elif [ "$1" = "certbot_dns_google" ]; then min=99 elif [ "$1" = "certbot_nginx" ]; then diff --git a/tox.ini b/tox.ini index 84d2b7501..89eef5c76 100644 --- a/tox.ini +++ b/tox.ini @@ -43,8 +43,10 @@ dns_plugin_paths = certbot-dns-cloudflare/certbot_dns_cloudflare certbot-dns-dig lexicon_dns_plugin_commands = pip install -e certbot-dns-cloudxns nosetests -v certbot_dns_cloudxns --processes=-1 -lexicon_dns_plugin_install_args = -e certbot-dns-cloudxns -lexicon_dns_plugin_paths = certbot-dns-cloudxns/certbot_dns_cloudxns + pip install -e certbot-dns-dnsimple + nosetests -v certbot_dns_dnsimple --processes=-1 +lexicon_dns_plugin_install_args = -e certbot-dns-cloudxns -e certbot-dns-dnsimple +lexicon_dns_plugin_paths = certbot-dns-cloudxns/certbot_dns_cloudxns certbot-dns-dnsimple/certbot_dns_dnsimple compatibility_install_args = -e certbot-compatibility-test compatibility_paths = certbot-compatibility-test/certbot_compatibility_test From 033c995bd288fe42f36df4ddae1aa31327c47df7 Mon Sep 17 00:00:00 2001 From: ohemorange Date: Tue, 23 May 2017 13:18:50 -0700 Subject: [PATCH 94/96] Update options-ssl-nginx.conf in`prepare` if it hasn't been manually modified (#4689) Fixes #4559. * Update options-ssl-nginx.conf in prepare, if it hasn't been modified. * add previous options-ssl-nginx.conf hashes * InstallSslOptionsConfTest * remove .new file and only print warning once * save digest to /etc/letsencrypt * add comment reminding devs to update hashes * add comment and test for sha256sum * treat hash file as text file because python3 * move constants and rename hidden digest file --- certbot-nginx/certbot_nginx/configurator.py | 41 +++++++++- certbot-nginx/certbot_nginx/constants.py | 15 ++++ .../certbot_nginx/options-ssl-nginx.conf | 5 ++ .../certbot_nginx/tests/configurator_test.py | 75 +++++++++++++++++++ certbot-nginx/certbot_nginx/tests/util.py | 5 -- certbot/crypto_util.py | 15 ++++ certbot/tests/crypto_util_test.py | 9 +++ 7 files changed, 157 insertions(+), 8 deletions(-) diff --git a/certbot-nginx/certbot_nginx/configurator.py b/certbot-nginx/certbot_nginx/configurator.py index dd61575d4..752ccc133 100644 --- a/certbot-nginx/certbot_nginx/configurator.py +++ b/certbot-nginx/certbot_nginx/configurator.py @@ -139,6 +139,11 @@ class NginxConfigurator(common.Plugin): """Full absolute path to SSL configuration file.""" return os.path.join(self.config.config_dir, constants.MOD_SSL_CONF_DEST) + @property + def updated_mod_ssl_conf_digest(self): + """Full absolute path to digest of updated SSL configuration file.""" + return os.path.join(self.config.config_dir, constants.UPDATED_MOD_SSL_CONF_DIGEST) + # This is called in determine_authenticator and determine_installer def prepare(self): """Prepare the authenticator/installer. @@ -156,7 +161,7 @@ class NginxConfigurator(common.Plugin): self.parser = parser.NginxParser(self.conf('server-root')) - install_ssl_options_conf(self.mod_ssl_conf) + install_ssl_options_conf(self.mod_ssl_conf, self.updated_mod_ssl_conf_digest) # Set Version if self.version is None: @@ -862,8 +867,38 @@ def nginx_restart(nginx_ctl, nginx_conf): time.sleep(1) -def install_ssl_options_conf(options_ssl): +def install_ssl_options_conf(options_ssl, options_ssl_digest): """Copy Certbot's SSL options file into the system's config dir if required.""" + def _write_current_hash(): + with open(options_ssl_digest, "w") as f: + f.write(constants.CURRENT_SSL_OPTIONS_HASH) + + def _install_current_file(): + shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl) + _write_current_hash() + # Check to make sure options-ssl.conf is installed if not os.path.isfile(options_ssl): - shutil.copyfile(constants.MOD_SSL_CONF_SRC, options_ssl) + _install_current_file() + return + # there's already a file there. if it exactly matches a previous file hash, + # we can update it. otherwise, print a warning once per new version. + active_file_digest = crypto_util.sha256sum(options_ssl) + if active_file_digest in constants.PREVIOUS_SSL_OPTIONS_HASHES: # safe to update + _install_current_file() + elif active_file_digest == constants.CURRENT_SSL_OPTIONS_HASH: # already up to date + return + else: # has been manually modified, not safe to update + # did they modify the current version or an old version? + if os.path.isfile(options_ssl_digest): + with open(options_ssl_digest, "r") as f: + saved_digest = f.read() + # they modified it after we either installed or told them about this version, so return + if saved_digest == constants.CURRENT_SSL_OPTIONS_HASH: + return + # there's a new version but we couldn't update the file, or they deleted the digest. + # save the current digest so we only print this once, and print a warning + _write_current_hash() + logger.warning("%s has been manually modified; updated ssl configuration options " + "saved to %s. We recommend updating %s for security purposes.", + options_ssl, constants.MOD_SSL_CONF_SRC, options_ssl) diff --git a/certbot-nginx/certbot_nginx/constants.py b/certbot-nginx/certbot_nginx/constants.py index 8cf1f6bc9..765bdd7a8 100644 --- a/certbot-nginx/certbot_nginx/constants.py +++ b/certbot-nginx/certbot_nginx/constants.py @@ -17,6 +17,21 @@ MOD_SSL_CONF_SRC = pkg_resources.resource_filename( """Path to the nginx mod_ssl config file found in the Certbot distribution.""" +UPDATED_MOD_SSL_CONF_DIGEST = ".updated-options-ssl-nginx-conf-digest.txt" +"""Name of the hash of the updated or informed mod_ssl_conf as saved in `IConfig.config_dir`.""" + + +PREVIOUS_SSL_OPTIONS_HASHES = [ + '0f81093a1465e3d4eaa8b0c14e77b2a2e93568b0fc1351c2b87893a95f0de87c', + '9a7b32c49001fed4cff8ad24353329472a50e86ade1ef9b2b9e43566a619612e', + 'a6d9f1c7d6b36749b52ba061fff1421f9a0a3d2cfdafbd63c05d06f65b990937', + '7f95624dd95cf5afc708b9f967ee83a24b8025dc7c8d9df2b556bbc64256b3ff', +] +"""SHA256 hashes of the contents of previous versions of MOD_SSL_CONF_SRC""" + +CURRENT_SSL_OPTIONS_HASH = '394732f2bbe3e5e637c3fb5c6e980a1f1b90b01e2e8d6b7cff41dde16e2a756d' +"""SHA256 hash of the current contents of MOD_SSL_CONF_SRC""" + def os_constant(key): # XXX TODO: In the future, this could return different constants # based on what OS we are running under. To see an diff --git a/certbot-nginx/certbot_nginx/options-ssl-nginx.conf b/certbot-nginx/certbot_nginx/options-ssl-nginx.conf index e1839909d..7303f9bc6 100644 --- a/certbot-nginx/certbot_nginx/options-ssl-nginx.conf +++ b/certbot-nginx/certbot_nginx/options-ssl-nginx.conf @@ -1,3 +1,8 @@ +# This file contains important security parameters. If you modify this file manually, +# Certbot will be unable to automatically provide future security updates. +# Instead, you will need to manually update this file by referencing the contents of +# options-ssl-nginx.conf.new. + ssl_session_cache shared:le_nginx_SSL:1m; ssl_session_timeout 1440m; diff --git a/certbot-nginx/certbot_nginx/tests/configurator_test.py b/certbot-nginx/certbot_nginx/tests/configurator_test.py index 906d276b1..215fe3165 100644 --- a/certbot-nginx/certbot_nginx/tests/configurator_test.py +++ b/certbot-nginx/certbot_nginx/tests/configurator_test.py @@ -11,9 +11,11 @@ from acme import challenges from acme import messages from certbot import achallenges +from certbot import crypto_util from certbot import errors from certbot.tests import util as certbot_test_util +from certbot_nginx import constants from certbot_nginx import obj from certbot_nginx import parser from certbot_nginx.tests import util @@ -537,6 +539,79 @@ class NginxConfiguratorTest(util.NginxTest): self.assertTrue(util.contains_at_depth( generated_conf, ['ssl_stapling_verify', 'on'], 2)) +class InstallSslOptionsConfTest(util.NginxTest): + """Test that the options-ssl-nginx.conf file is installed and updated properly.""" + + def setUp(self): + super(InstallSslOptionsConfTest, self).setUp() + + self.config = util.get_nginx_configurator( + self.config_path, self.config_dir, self.work_dir, self.logs_dir) + + def _call(self): + from certbot_nginx.configurator import install_ssl_options_conf + install_ssl_options_conf(self.config.mod_ssl_conf, self.config.updated_mod_ssl_conf_digest) + + def _assert_current_file(self): + """If this is failing, remember that constants.PREVIOUS_SSL_OPTIONS_HASHES and + constants.CURRENT_SSL_OPTIONS_HASH must be updated when self.config.mod_ssl_conf + is updated. Add CURRENT_SSL_OPTIONS_HASH to PREVIOUS_SSL_OPTIONS_HASHES and set + CURRENT_SSL_OPTIONS_HASH to the hash of the updated self.config.mod_ssl_conf.""" + self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) + from certbot_nginx.constants import CURRENT_SSL_OPTIONS_HASH + self.assertEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), CURRENT_SSL_OPTIONS_HASH) + + def test_no_file(self): + # prepare should have placed a file there + self._assert_current_file() + os.remove(self.config.mod_ssl_conf) + self.assertFalse(os.path.isfile(self.config.mod_ssl_conf)) + self._call() + self._assert_current_file() + + def test_current_file(self): + self._assert_current_file() + self._call() + self._assert_current_file() + + def test_prev_file_updates_to_current(self): + from certbot_nginx.constants import PREVIOUS_SSL_OPTIONS_HASHES + with mock.patch('certbot.crypto_util.sha256sum') as mock_sha256: + mock_sha256.return_value = PREVIOUS_SSL_OPTIONS_HASHES[0] + self._call() + self._assert_current_file() + + def test_manually_modified_current_file_does_not_update(self): + with open(self.config.mod_ssl_conf, "a") as mod_ssl_conf: + mod_ssl_conf.write("a new line for the wrong hash\n") + with mock.patch("certbot_nginx.configurator.logger") as mock_logger: + self._call() + self.assertFalse(mock_logger.warning.called) + self.assertTrue(os.path.isfile(self.config.mod_ssl_conf)) + from certbot_nginx.constants import CURRENT_SSL_OPTIONS_HASH + self.assertEqual(crypto_util.sha256sum(constants.MOD_SSL_CONF_SRC), + CURRENT_SSL_OPTIONS_HASH) + self.assertNotEqual(crypto_util.sha256sum(self.config.mod_ssl_conf), + CURRENT_SSL_OPTIONS_HASH) + + def test_manually_modified_past_file_warns(self): + with open(self.config.mod_ssl_conf, "a") as mod_ssl_conf: + mod_ssl_conf.write("a new line for the wrong hash\n") + with open(self.config.updated_mod_ssl_conf_digest, "w") as f: + f.write("hashofanoldversion") + with mock.patch("certbot_nginx.configurator.logger") as mock_logger: + self._call() + self.assertEqual(mock_logger.warning.call_args[0][0], + "%s has been manually modified; updated ssl configuration options " + "saved to %s. We recommend updating %s for security purposes.") + from certbot_nginx.constants import CURRENT_SSL_OPTIONS_HASH + self.assertEqual(crypto_util.sha256sum(constants.MOD_SSL_CONF_SRC), + CURRENT_SSL_OPTIONS_HASH) + # only print warning once + with mock.patch("certbot_nginx.configurator.logger") as mock_logger: + self._call() + self.assertFalse(mock_logger.warning.called) + if __name__ == "__main__": unittest.main() # pragma: no cover diff --git a/certbot-nginx/certbot_nginx/tests/util.py b/certbot-nginx/certbot_nginx/tests/util.py index 4ab95374e..2ee38ec38 100644 --- a/certbot-nginx/certbot_nginx/tests/util.py +++ b/certbot-nginx/certbot_nginx/tests/util.py @@ -16,7 +16,6 @@ from certbot.tests import util as test_util from certbot.plugins import common -from certbot_nginx import constants from certbot_nginx import configurator from certbot_nginx import nginxparser @@ -30,10 +29,6 @@ class NginxTest(unittest.TestCase): # pylint: disable=too-few-public-methods "etc_nginx", "certbot_nginx.tests") self.logs_dir = tempfile.mkdtemp('logs') - self.ssl_options = common.setup_ssl_options( - self.config_dir, constants.MOD_SSL_CONF_SRC, - constants.MOD_SSL_CONF_DEST) - self.config_path = os.path.join(self.temp_dir, "etc_nginx") self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector( diff --git a/certbot/crypto_util.py b/certbot/crypto_util.py index 7de173568..2b2e7d0d8 100644 --- a/certbot/crypto_util.py +++ b/certbot/crypto_util.py @@ -4,6 +4,7 @@ is capable of handling the signatures. """ +import hashlib import logging import os @@ -353,3 +354,17 @@ def _notAfterBefore(cert_path, method): if six.PY3: timestamp_str = timestamp_str.decode('ascii') return pyrfc3339.parse(timestamp_str) + + +def sha256sum(filename): + """Compute a sha256sum of a file. + + :param str filename: path to the file whose hash will be computed + + :returns: sha256 digest of the file in hexadecimal + :rtype: str + """ + sha256 = hashlib.sha256() + with open(filename, 'rb') as f: + sha256.update(f.read()) + return sha256.hexdigest() diff --git a/certbot/tests/crypto_util_test.py b/certbot/tests/crypto_util_test.py index 8adf753d6..c678dc501 100644 --- a/certbot/tests/crypto_util_test.py +++ b/certbot/tests/crypto_util_test.py @@ -293,5 +293,14 @@ class NotAfterTest(unittest.TestCase): '2014-12-18T22:34:45+00:00') +class Sha256sumTest(unittest.TestCase): + """Tests for certbot.crypto_util.notAfter""" + + def test_sha256sum(self): + from certbot.crypto_util import sha256sum + self.assertEqual(sha256sum(CERT_PATH), + '914ffed8daf9e2c99d90ac95c77d54f32cbd556672facac380f0c063498df84e') + + if __name__ == '__main__': unittest.main() # pragma: no cover From ddd10548c8e7619786ec9faeab47fe046a2d53bf Mon Sep 17 00:00:00 2001 From: Zach Shepherd Date: Wed, 24 May 2017 10:37:08 -0700 Subject: [PATCH 95/96] route53: re-use boto3 client in wait (#4724) This change re-uses the boto3 client in the wait method of the route53 authenticator in order to make it more mockable for testing purposes. --- certbot-route53/certbot_route53/authenticator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/certbot-route53/certbot_route53/authenticator.py b/certbot-route53/certbot_route53/authenticator.py index 855165455..1f1f78bfc 100644 --- a/certbot-route53/certbot_route53/authenticator.py +++ b/certbot-route53/certbot_route53/authenticator.py @@ -137,9 +137,8 @@ class Authenticator(common.Plugin): """Wait for a change to be propagated to all Route53 DNS servers. https://docs.aws.amazon.com/Route53/latest/APIReference/API_GetChange.html """ - client = boto3.client("route53") for n in range(0, 120): - response = client.get_change(Id=change_id) + response = self.r53.get_change(Id=change_id) if response["ChangeInfo"]["Status"] == "INSYNC": return time.sleep(5) From 8ae3a9082da25e426ff7fcfb60db26d46b9d5286 Mon Sep 17 00:00:00 2001 From: Anna Liao Date: Wed, 24 May 2017 11:50:37 -0700 Subject: [PATCH 96/96] updated manual challenge prompt so last sentence is complete (#4704) Fixes #4641. --- certbot/plugins/manual.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/certbot/plugins/manual.py b/certbot/plugins/manual.py index 1163e7e7e..25e2564fa 100644 --- a/certbot/plugins/manual.py +++ b/certbot/plugins/manual.py @@ -42,7 +42,7 @@ Please deploy a DNS TXT record under the name {validation} -Once this is deployed,""" +Before continuing, verify the record is deployed.""" _HTTP_INSTRUCTIONS = """\ Make sure your web server displays the following content at {uri} before continuing: