Merge branch 'master' into plugin_tests

.travis.yml

@@ -21,6 +21,15 @@ env:
     - TOXENV=lint
     - TOXENV=cover
 
+
+# Only build pushes to the master branch, PRs, and branches beginning with
+# `test-`. This reduces the number of simultaneous Travis runs, which speeds
+# turnaround time on review since there is a cap of 5 simultaneous runs.
+branches:
+  only:
+    - master
+    - /^test-.*$/
+
 sudo: false  # containers
 addons:
   # make sure simplehttp simple verification works (custom /etc/hosts)

acme/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.0.0.dev20151008'
+version = '0.1.0.dev0'
 
 install_requires = [
     # load_pem_private/public_key (>=0.6)

docs/contributing.rst

@@ -267,6 +267,22 @@ Please:
 .. _PEP 8 - Style Guide for Python Code:
   https://www.python.org/dev/peps/pep-0008
 
+Submitting a pull request
+=========================
+
+Steps:
+
+1. Write your code!
+2. Make sure your environment is set up properly and that you're in your
+   virtualenv. You can do this by running ``./bootstrap/dev/venv.sh``.
+   (this is a **very important** step)
+3. Run ``./pep8.travis.sh`` to do a cursory check of your code style.
+   Fix any errors.
+4. Run ``tox -e lint`` to check for pylint errors. Fix any errors.
+5. Run ``tox`` to run the entire test suite including coverage. Fix any errors.
+6. If your code touches communication with an ACME server/Boulder, you
+   should run the integration tests, see `integration`_.
+7. Submit the PR.
 
 Updating the documentation
 ==========================

letsencrypt-apache/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.0.0.dev20151008'
+version = '0.1.0.dev0'
 
 install_requires = [
     'acme=={0}'.format(version),

letsencrypt-nginx/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.0.0.dev20151008'
+version = '0.1.0.dev0'
 
 install_requires = [
     'acme=={0}'.format(version),

letsencrypt/__init__.py

@@ -1,4 +1,4 @@
 """Let's Encrypt client."""
 
 # version number like 1.2.3a0, must have at least 2 parts, like 1.2
-__version__ = '0.0.0.dev20151008'
+__version__ = '0.1.0.dev0'

letsencrypt/cli.py

@@ -839,9 +839,23 @@ def _plugins_parsing(helpful, plugins):
     helpful.add_plugin_args(plugins)
 
 
-def _setup_logging(args):
-    level = -args.verbose_count * 10
-    fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
+def setup_log_file_handler(args, logfile, fmt):
+    """Setup file debug logging."""
+    log_file_path = os.path.join(args.logs_dir, logfile)
+    handler = logging.handlers.RotatingFileHandler(
+        log_file_path, maxBytes=2 ** 20, backupCount=10)
+    # rotate on each invocation, rollover only possible when maxBytes
+    # is nonzero and backupCount is nonzero, so we set maxBytes as big
+    # as possible not to overrun in single CLI invocation (1MB).
+    handler.doRollover()  # TODO: creates empty letsencrypt.log.1 file
+    handler.setLevel(logging.DEBUG)
+    handler_formatter = logging.Formatter(fmt=fmt)
+    handler_formatter.converter = time.gmtime  # don't use localtime
+    handler.setFormatter(handler_formatter)
+    return handler, log_file_path
+
+
+def _cli_log_handler(args, level, fmt):
     if args.text_mode:
         handler = colored_logging.StreamHandler()
         handler.setFormatter(logging.Formatter(fmt))
@@ -850,30 +864,26 @@ def _setup_logging(args):
         # dialog box is small, display as less as possible
         handler.setFormatter(logging.Formatter("%(message)s"))
     handler.setLevel(level)
+    return handler
+
+
+def setup_logging(args, cli_handler_factory, logfile):
+    """Setup logging."""
+    fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s"
+    level = -args.verbose_count * 10
+    file_handler, log_file_path = setup_log_file_handler(
+        args, logfile=logfile, fmt=fmt)
+    cli_handler = cli_handler_factory(args, level, fmt)
 
     # TODO: use fileConfig?
 
-    # unconditionally log to file for debugging purposes
-    # TODO: change before release?
-    log_file_name = os.path.join(args.logs_dir, 'letsencrypt.log')
-    file_handler = logging.handlers.RotatingFileHandler(
-        log_file_name, maxBytes=2 ** 20, backupCount=10)
-    # rotate on each invocation, rollover only possible when maxBytes
-    # is nonzero and backupCount is nonzero, so we set maxBytes as big
-    # as possible not to overrun in single CLI invocation (1MB).
-    file_handler.doRollover()  # TODO: creates empty letsencrypt.log.1 file
-    file_handler.setLevel(logging.DEBUG)
-    file_handler_formatter = logging.Formatter(fmt=fmt)
-    file_handler_formatter.converter = time.gmtime  # don't use localtime
-    file_handler.setFormatter(file_handler_formatter)
-
     root_logger = logging.getLogger()
     root_logger.setLevel(logging.DEBUG)  # send all records to handlers
-    root_logger.addHandler(handler)
+    root_logger.addHandler(cli_handler)
     root_logger.addHandler(file_handler)
 
     logger.debug("Root logging level set at %d", level)
-    logger.info("Saving debug log to %s", log_file_name)
+    logger.info("Saving debug log to %s", log_file_path)
 
 
 def _handle_exception(exc_type, exc_value, trace, args):
@@ -942,7 +952,7 @@ def main(cli_args=sys.argv[1:]):
     # private key! #525
     le_util.make_or_verify_dir(
         args.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args)
-    _setup_logging(args)
+    setup_logging(args, _cli_log_handler, logfile='letsencrypt.log')
 
     # do not log `args`, as it contains sensitive data (e.g. revoke --key)!
     logger.debug("Arguments: %r", cli_args)

letsencrypt/renewer.py

@@ -8,6 +8,7 @@ within lineages of successor certificates, according to configuration.
 
 """
 import argparse
+import logging
 import os
 import sys
 
@@ -17,10 +18,12 @@ import zope.component
 
 from letsencrypt import account
 from letsencrypt import configuration
+from letsencrypt import colored_logging
 from letsencrypt import cli
 from letsencrypt import client
 from letsencrypt import crypto_util
 from letsencrypt import errors
+from letsencrypt import le_util
 from letsencrypt import notify
 from letsencrypt import storage
 
@@ -28,6 +31,9 @@ from letsencrypt.display import util as display_util
 from letsencrypt.plugins import disco as plugins_disco
 
 
+logger = logging.getLogger(__name__)
+
+
 class _AttrDict(dict):
     """Attribute dictionary.
 
@@ -104,6 +110,12 @@ def renew(cert, old_version):
     #       (where fewer than all names were renewed)
 
 
+def _cli_log_handler(args, level, fmt):  # pylint: disable=unused-argument
+    handler = colored_logging.StreamHandler()
+    handler.setFormatter(logging.Formatter(fmt))
+    return handler
+
+
 def _paths_parser(parser):
     add = parser.add_argument_group("paths").add_argument
     add("--config-dir", default=cli.flag_default("config_dir"),
@@ -119,11 +131,16 @@ def _paths_parser(parser):
 def _create_parser():
     parser = argparse.ArgumentParser()
     #parser.add_argument("--cron", action="store_true", help="Run as cronjob.")
-    # pylint: disable=protected-access
+    parser.add_argument(
+        "-v", "--verbose", dest="verbose_count", action="count",
+        default=cli.flag_default("verbose_count"), help="This flag can be used "
+        "multiple times to incrementally increase the verbosity of output, "
+        "e.g. -vvv.")
+
     return _paths_parser(parser)
 
 
-def main(config=None, args=sys.argv[1:]):
+def main(config=None, cli_args=sys.argv[1:]):
     """Main function for autorenewer script."""
     # TODO: Distinguish automated invocation from manual invocation,
     #       perhaps by looking at sys.argv[0] and inhibiting automated
@@ -133,8 +150,12 @@ def main(config=None, args=sys.argv[1:]):
 
     zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
 
-    cli_config = configuration.RenewerConfiguration(
-        _create_parser().parse_args(args))
+    args = _create_parser().parse_args(cli_args)
+
+    le_util.make_or_verify_dir(args.logs_dir, 0o700, os.geteuid())
+    cli.setup_logging(args, _cli_log_handler, logfile='renewer.log')
+
+    cli_config = configuration.RenewerConfiguration(args)
 
     config = storage.config_with_defaults(config)
     # Now attempt to read the renewer config file and augment or replace

letsencrypt/tests/renewer_test.py

@@ -44,8 +44,15 @@ class BaseRenewableCertTest(unittest.TestCase):
         self.tempdir = tempfile.mkdtemp()
 
         self.cli_config = configuration.RenewerConfiguration(
-            namespace=mock.MagicMock(config_dir=self.tempdir))
+            namespace=mock.MagicMock(
+                config_dir=self.tempdir,
+                work_dir=self.tempdir,
+                logs_dir=self.tempdir,
+            )
+        )
+
         # TODO: maybe provide RenewerConfiguration.make_dirs?
+        # TODO: main() should create those dirs, c.f. #902
         os.makedirs(os.path.join(self.tempdir, "live", "example.org"))
         os.makedirs(os.path.join(self.tempdir, "archive", "example.org"))
         os.makedirs(os.path.join(self.tempdir, "renewal"))
@@ -62,6 +69,9 @@ class BaseRenewableCertTest(unittest.TestCase):
         self.test_rc = storage.RenewableCert(
             self.config, self.defaults, self.cli_config)
 
+    def tearDown(self):
+        shutil.rmtree(self.tempdir)
+
     def _write_out_ex_kinds(self):
         for kind in ALL_FOUR:
             where = getattr(self.test_rc, kind)
@@ -79,11 +89,6 @@ class BaseRenewableCertTest(unittest.TestCase):
 class RenewableCertTests(BaseRenewableCertTest):
     # pylint: disable=too-many-public-methods
     """Tests for letsencrypt.renewer.*."""
-    def setUp(self):
-        super(RenewableCertTests, self).setUp()
-
-    def tearDown(self):
-        shutil.rmtree(self.tempdir)
 
     def test_initialization(self):
         self.assertEqual(self.test_rc.lineagename, "example.org")
@@ -665,11 +670,17 @@ class RenewableCertTests(BaseRenewableCertTest):
         # This should fail because the renewal itself appears to fail
         self.assertFalse(renewer.renew(self.test_rc, 1))
 
+    def _common_cli_args(self):
+        return [
+            "--config-dir", self.cli_config.config_dir,
+            "--work-dir", self.cli_config.work_dir,
+            "--logs-dir", self.cli_config.logs_dir,
+        ]
+
     @mock.patch("letsencrypt.renewer.notify")
     @mock.patch("letsencrypt.storage.RenewableCert")
     @mock.patch("letsencrypt.renewer.renew")
     def test_main(self, mock_renew, mock_rc, mock_notify):
-        """Test for main() function."""
         from letsencrypt import renewer
         mock_rc_instance = mock.MagicMock()
         mock_rc_instance.should_autodeploy.return_value = True
@@ -691,8 +702,7 @@ class RenewableCertTests(BaseRenewableCertTest):
                                "example.com.conf"), "w") as f:
             f.write("cert = cert.pem\nprivkey = privkey.pem\n")
             f.write("chain = chain.pem\nfullchain = fullchain.pem\n")
-        renewer.main(self.defaults, args=[
-            '--config-dir', self.cli_config.config_dir])
+        renewer.main(self.defaults, cli_args=self._common_cli_args())
         self.assertEqual(mock_rc.call_count, 2)
         self.assertEqual(mock_rc_instance.update_all_links_to.call_count, 2)
         self.assertEqual(mock_notify.notify.call_count, 4)
@@ -705,8 +715,7 @@ class RenewableCertTests(BaseRenewableCertTest):
         mock_happy_instance.should_autorenew.return_value = False
         mock_happy_instance.latest_common_version.return_value = 10
         mock_rc.return_value = mock_happy_instance
-        renewer.main(self.defaults, args=[
-            '--config-dir', self.cli_config.config_dir])
+        renewer.main(self.defaults, cli_args=self._common_cli_args())
         self.assertEqual(mock_rc.call_count, 4)
         self.assertEqual(mock_happy_instance.update_all_links_to.call_count, 0)
         self.assertEqual(mock_notify.notify.call_count, 4)
@@ -717,8 +726,7 @@ class RenewableCertTests(BaseRenewableCertTest):
         with open(os.path.join(self.cli_config.renewal_configs_dir,
                                "bad.conf"), "w") as f:
             f.write("incomplete = configfile\n")
-        renewer.main(self.defaults, args=[
-            '--config-dir', self.cli_config.config_dir])
+        renewer.main(self.defaults, cli_args=self._common_cli_args())
         # The errors.CertStorageError is caught inside and nothing happens.
 
 

letshelp-letsencrypt/setup.py

@@ -4,7 +4,7 @@ from setuptools import setup
 from setuptools import find_packages
 
 
-version = '0.0.0.dev20151008'
+version = '0.1.0.dev0'
 
 install_requires = [
     'setuptools',  # pkg_resources