diff --git a/.travis.yml b/.travis.yml index 46b14fe63..3041fdd82 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,15 @@ env: - TOXENV=lint - TOXENV=cover + +# Only build pushes to the master branch, PRs, and branches beginning with +# `test-`. This reduces the number of simultaneous Travis runs, which speeds +# turnaround time on review since there is a cap of 5 simultaneous runs. +branches: + only: + - master + - /^test-.*$/ + sudo: false # containers addons: # make sure simplehttp simple verification works (custom /etc/hosts) diff --git a/acme/setup.py b/acme/setup.py index 36a724f97..6448b7fe9 100644 --- a/acme/setup.py +++ b/acme/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ # load_pem_private/public_key (>=0.6) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3277d321a..614f6f2aa 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -267,6 +267,22 @@ Please: .. _PEP 8 - Style Guide for Python Code: https://www.python.org/dev/peps/pep-0008 +Submitting a pull request +========================= + +Steps: + +1. Write your code! +2. Make sure your environment is set up properly and that you're in your + virtualenv. You can do this by running ``./bootstrap/dev/venv.sh``. + (this is a **very important** step) +3. Run ``./pep8.travis.sh`` to do a cursory check of your code style. + Fix any errors. +4. Run ``tox -e lint`` to check for pylint errors. Fix any errors. +5. Run ``tox`` to run the entire test suite including coverage. Fix any errors. +6. If your code touches communication with an ACME server/Boulder, you + should run the integration tests, see `integration`_. +7. Submit the PR. Updating the documentation ========================== diff --git a/letsencrypt-apache/setup.py b/letsencrypt-apache/setup.py index ef9299b2c..626e700b2 100644 --- a/letsencrypt-apache/setup.py +++ b/letsencrypt-apache/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/letsencrypt-nginx/setup.py b/letsencrypt-nginx/setup.py index dde7243a9..a37b8222b 100644 --- a/letsencrypt-nginx/setup.py +++ b/letsencrypt-nginx/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'acme=={0}'.format(version), diff --git a/letsencrypt/__init__.py b/letsencrypt/__init__.py index d7ec9c5e3..1155a5b0c 100644 --- a/letsencrypt/__init__.py +++ b/letsencrypt/__init__.py @@ -1,4 +1,4 @@ """Let's Encrypt client.""" # version number like 1.2.3a0, must have at least 2 parts, like 1.2 -__version__ = '0.0.0.dev20151008' +__version__ = '0.1.0.dev0' diff --git a/letsencrypt/cli.py b/letsencrypt/cli.py index 64cba508d..4d109a9c1 100644 --- a/letsencrypt/cli.py +++ b/letsencrypt/cli.py @@ -839,9 +839,23 @@ def _plugins_parsing(helpful, plugins): helpful.add_plugin_args(plugins) -def _setup_logging(args): - level = -args.verbose_count * 10 - fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" +def setup_log_file_handler(args, logfile, fmt): + """Setup file debug logging.""" + log_file_path = os.path.join(args.logs_dir, logfile) + handler = logging.handlers.RotatingFileHandler( + log_file_path, maxBytes=2 ** 20, backupCount=10) + # rotate on each invocation, rollover only possible when maxBytes + # is nonzero and backupCount is nonzero, so we set maxBytes as big + # as possible not to overrun in single CLI invocation (1MB). + handler.doRollover() # TODO: creates empty letsencrypt.log.1 file + handler.setLevel(logging.DEBUG) + handler_formatter = logging.Formatter(fmt=fmt) + handler_formatter.converter = time.gmtime # don't use localtime + handler.setFormatter(handler_formatter) + return handler, log_file_path + + +def _cli_log_handler(args, level, fmt): if args.text_mode: handler = colored_logging.StreamHandler() handler.setFormatter(logging.Formatter(fmt)) @@ -850,30 +864,26 @@ def _setup_logging(args): # dialog box is small, display as less as possible handler.setFormatter(logging.Formatter("%(message)s")) handler.setLevel(level) + return handler + + +def setup_logging(args, cli_handler_factory, logfile): + """Setup logging.""" + fmt = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" + level = -args.verbose_count * 10 + file_handler, log_file_path = setup_log_file_handler( + args, logfile=logfile, fmt=fmt) + cli_handler = cli_handler_factory(args, level, fmt) # TODO: use fileConfig? - # unconditionally log to file for debugging purposes - # TODO: change before release? - log_file_name = os.path.join(args.logs_dir, 'letsencrypt.log') - file_handler = logging.handlers.RotatingFileHandler( - log_file_name, maxBytes=2 ** 20, backupCount=10) - # rotate on each invocation, rollover only possible when maxBytes - # is nonzero and backupCount is nonzero, so we set maxBytes as big - # as possible not to overrun in single CLI invocation (1MB). - file_handler.doRollover() # TODO: creates empty letsencrypt.log.1 file - file_handler.setLevel(logging.DEBUG) - file_handler_formatter = logging.Formatter(fmt=fmt) - file_handler_formatter.converter = time.gmtime # don't use localtime - file_handler.setFormatter(file_handler_formatter) - root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) # send all records to handlers - root_logger.addHandler(handler) + root_logger.addHandler(cli_handler) root_logger.addHandler(file_handler) logger.debug("Root logging level set at %d", level) - logger.info("Saving debug log to %s", log_file_name) + logger.info("Saving debug log to %s", log_file_path) def _handle_exception(exc_type, exc_value, trace, args): @@ -942,7 +952,7 @@ def main(cli_args=sys.argv[1:]): # private key! #525 le_util.make_or_verify_dir( args.logs_dir, 0o700, os.geteuid(), "--strict-permissions" in cli_args) - _setup_logging(args) + setup_logging(args, _cli_log_handler, logfile='letsencrypt.log') # do not log `args`, as it contains sensitive data (e.g. revoke --key)! logger.debug("Arguments: %r", cli_args) diff --git a/letsencrypt/renewer.py b/letsencrypt/renewer.py index 1c9cddc95..98ecc83b3 100644 --- a/letsencrypt/renewer.py +++ b/letsencrypt/renewer.py @@ -8,6 +8,7 @@ within lineages of successor certificates, according to configuration. """ import argparse +import logging import os import sys @@ -17,10 +18,12 @@ import zope.component from letsencrypt import account from letsencrypt import configuration +from letsencrypt import colored_logging from letsencrypt import cli from letsencrypt import client from letsencrypt import crypto_util from letsencrypt import errors +from letsencrypt import le_util from letsencrypt import notify from letsencrypt import storage @@ -28,6 +31,9 @@ from letsencrypt.display import util as display_util from letsencrypt.plugins import disco as plugins_disco +logger = logging.getLogger(__name__) + + class _AttrDict(dict): """Attribute dictionary. @@ -104,6 +110,12 @@ def renew(cert, old_version): # (where fewer than all names were renewed) +def _cli_log_handler(args, level, fmt): # pylint: disable=unused-argument + handler = colored_logging.StreamHandler() + handler.setFormatter(logging.Formatter(fmt)) + return handler + + def _paths_parser(parser): add = parser.add_argument_group("paths").add_argument add("--config-dir", default=cli.flag_default("config_dir"), @@ -119,11 +131,16 @@ def _paths_parser(parser): def _create_parser(): parser = argparse.ArgumentParser() #parser.add_argument("--cron", action="store_true", help="Run as cronjob.") - # pylint: disable=protected-access + parser.add_argument( + "-v", "--verbose", dest="verbose_count", action="count", + default=cli.flag_default("verbose_count"), help="This flag can be used " + "multiple times to incrementally increase the verbosity of output, " + "e.g. -vvv.") + return _paths_parser(parser) -def main(config=None, args=sys.argv[1:]): +def main(config=None, cli_args=sys.argv[1:]): """Main function for autorenewer script.""" # TODO: Distinguish automated invocation from manual invocation, # perhaps by looking at sys.argv[0] and inhibiting automated @@ -133,8 +150,12 @@ def main(config=None, args=sys.argv[1:]): zope.component.provideUtility(display_util.FileDisplay(sys.stdout)) - cli_config = configuration.RenewerConfiguration( - _create_parser().parse_args(args)) + args = _create_parser().parse_args(cli_args) + + le_util.make_or_verify_dir(args.logs_dir, 0o700, os.geteuid()) + cli.setup_logging(args, _cli_log_handler, logfile='renewer.log') + + cli_config = configuration.RenewerConfiguration(args) config = storage.config_with_defaults(config) # Now attempt to read the renewer config file and augment or replace diff --git a/letsencrypt/tests/renewer_test.py b/letsencrypt/tests/renewer_test.py index 6f115abf9..bc8afbcb5 100644 --- a/letsencrypt/tests/renewer_test.py +++ b/letsencrypt/tests/renewer_test.py @@ -44,8 +44,15 @@ class BaseRenewableCertTest(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.cli_config = configuration.RenewerConfiguration( - namespace=mock.MagicMock(config_dir=self.tempdir)) + namespace=mock.MagicMock( + config_dir=self.tempdir, + work_dir=self.tempdir, + logs_dir=self.tempdir, + ) + ) + # TODO: maybe provide RenewerConfiguration.make_dirs? + # TODO: main() should create those dirs, c.f. #902 os.makedirs(os.path.join(self.tempdir, "live", "example.org")) os.makedirs(os.path.join(self.tempdir, "archive", "example.org")) os.makedirs(os.path.join(self.tempdir, "renewal")) @@ -62,6 +69,9 @@ class BaseRenewableCertTest(unittest.TestCase): self.test_rc = storage.RenewableCert( self.config, self.defaults, self.cli_config) + def tearDown(self): + shutil.rmtree(self.tempdir) + def _write_out_ex_kinds(self): for kind in ALL_FOUR: where = getattr(self.test_rc, kind) @@ -79,11 +89,6 @@ class BaseRenewableCertTest(unittest.TestCase): class RenewableCertTests(BaseRenewableCertTest): # pylint: disable=too-many-public-methods """Tests for letsencrypt.renewer.*.""" - def setUp(self): - super(RenewableCertTests, self).setUp() - - def tearDown(self): - shutil.rmtree(self.tempdir) def test_initialization(self): self.assertEqual(self.test_rc.lineagename, "example.org") @@ -665,11 +670,17 @@ class RenewableCertTests(BaseRenewableCertTest): # This should fail because the renewal itself appears to fail self.assertFalse(renewer.renew(self.test_rc, 1)) + def _common_cli_args(self): + return [ + "--config-dir", self.cli_config.config_dir, + "--work-dir", self.cli_config.work_dir, + "--logs-dir", self.cli_config.logs_dir, + ] + @mock.patch("letsencrypt.renewer.notify") @mock.patch("letsencrypt.storage.RenewableCert") @mock.patch("letsencrypt.renewer.renew") def test_main(self, mock_renew, mock_rc, mock_notify): - """Test for main() function.""" from letsencrypt import renewer mock_rc_instance = mock.MagicMock() mock_rc_instance.should_autodeploy.return_value = True @@ -691,8 +702,7 @@ class RenewableCertTests(BaseRenewableCertTest): "example.com.conf"), "w") as f: f.write("cert = cert.pem\nprivkey = privkey.pem\n") f.write("chain = chain.pem\nfullchain = fullchain.pem\n") - renewer.main(self.defaults, args=[ - '--config-dir', self.cli_config.config_dir]) + renewer.main(self.defaults, cli_args=self._common_cli_args()) self.assertEqual(mock_rc.call_count, 2) self.assertEqual(mock_rc_instance.update_all_links_to.call_count, 2) self.assertEqual(mock_notify.notify.call_count, 4) @@ -705,8 +715,7 @@ class RenewableCertTests(BaseRenewableCertTest): mock_happy_instance.should_autorenew.return_value = False mock_happy_instance.latest_common_version.return_value = 10 mock_rc.return_value = mock_happy_instance - renewer.main(self.defaults, args=[ - '--config-dir', self.cli_config.config_dir]) + renewer.main(self.defaults, cli_args=self._common_cli_args()) self.assertEqual(mock_rc.call_count, 4) self.assertEqual(mock_happy_instance.update_all_links_to.call_count, 0) self.assertEqual(mock_notify.notify.call_count, 4) @@ -717,8 +726,7 @@ class RenewableCertTests(BaseRenewableCertTest): with open(os.path.join(self.cli_config.renewal_configs_dir, "bad.conf"), "w") as f: f.write("incomplete = configfile\n") - renewer.main(self.defaults, args=[ - '--config-dir', self.cli_config.config_dir]) + renewer.main(self.defaults, cli_args=self._common_cli_args()) # The errors.CertStorageError is caught inside and nothing happens. diff --git a/letshelp-letsencrypt/setup.py b/letshelp-letsencrypt/setup.py index ad517c74e..a83fc8843 100644 --- a/letshelp-letsencrypt/setup.py +++ b/letshelp-letsencrypt/setup.py @@ -4,7 +4,7 @@ from setuptools import setup from setuptools import find_packages -version = '0.0.0.dev20151008' +version = '0.1.0.dev0' install_requires = [ 'setuptools', # pkg_resources