From bb1242c04767c8686de39af78f4f562ae33ded1d Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 18 Jan 2019 00:21:13 +0100 Subject: [PATCH] Process isolated local oldest requirements for DNS plugins (#6653) * Include local-oldest-requirements in the constraints when oldest is invoked * Add oldest constraints for lexicon * Skip editable requirements during merge * Pin to lexicon 2.2.1 for oldest requirements. Override locally oldest if needed for specific providers. --- .../local-oldest-requirements.txt | 1 - certbot-dns-ovh/local-oldest-requirements.txt | 1 + tools/dev_constraints.txt | 6 ++-- tools/merge_requirements.py | 34 +++++++++---------- tools/oldest_constraints.txt | 9 +++-- tools/pip_install.py | 25 +++++++++----- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/certbot-dns-dnsimple/local-oldest-requirements.txt b/certbot-dns-dnsimple/local-oldest-requirements.txt index 5ecd3cbc7..65f5a758e 100644 --- a/certbot-dns-dnsimple/local-oldest-requirements.txt +++ b/certbot-dns-dnsimple/local-oldest-requirements.txt @@ -1,3 +1,2 @@ -e acme[dev] -e .[dev] -dns-lexicon==2.2.1 \ No newline at end of file diff --git a/certbot-dns-ovh/local-oldest-requirements.txt b/certbot-dns-ovh/local-oldest-requirements.txt index 65f5a758e..01cbcb317 100644 --- a/certbot-dns-ovh/local-oldest-requirements.txt +++ b/certbot-dns-ovh/local-oldest-requirements.txt @@ -1,2 +1,3 @@ -e acme[dev] -e .[dev] +dns-lexicon==2.7.14 diff --git a/tools/dev_constraints.txt b/tools/dev_constraints.txt index 111dc5495..88340cb00 100644 --- a/tools/dev_constraints.txt +++ b/tools/dev_constraints.txt @@ -1,5 +1,7 @@ -# Specifies Python package versions for packages not specified in -# letsencrypt-auto's requirements file. +# Specifies Python package versions for development. +# It includes in particular packages not specified in letsencrypt-auto's requirements file. +# Some dev package versions specified here may be overridden by higher level constraints +# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt). alabaster==0.7.10 apipkg==1.4 asn1crypto==0.22.0 diff --git a/tools/merge_requirements.py b/tools/merge_requirements.py index ad44a55d0..4205e6bcf 100755 --- a/tools/merge_requirements.py +++ b/tools/merge_requirements.py @@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only simple SomeProject==1.2.3 format is currently supported. """ - from __future__ import print_function import sys + def read_file(file_path): """Reads in a Python requirements file. + Ignore empty lines, comments and editable requirements :param str file_path: path to requirements file @@ -19,16 +20,16 @@ def read_file(file_path): :rtype: dict """ - d = {} - with open(file_path) as f: - for line in f: + data = {} + with open(file_path) as file_h: + for line in file_h: line = line.strip() - if line and not line.startswith('#'): + if line and not line.startswith('#') and not line.startswith('-e'): project, version = line.split('==') if not version: raise ValueError("Unexpected syntax '{0}'".format(line)) - d[project] = version - return d + data[project] = version + return data def output_requirements(requirements): @@ -37,25 +38,24 @@ def output_requirements(requirements): :param dict requirements: mapping from a project to its pinned version """ - return '\n'.join('{0}=={1}'.format(k, v) - for k, v in sorted(requirements.items())) + return '\n'.join('{0}=={1}'.format(key, value) + for key, value in sorted(requirements.items())) -def main(*files): +def main(*paths): """Merges multiple requirements files together and prints the result. Requirement files specified later in the list take precedence over earlier files. - :param tuple files: paths to requirements files + :param tuple paths: paths to requirements files """ - d = {} - for f in files: - d.update(read_file(f)) - return output_requirements(d) + data = {} + for path in paths: + data.update(read_file(path)) + return output_requirements(data) if __name__ == '__main__': - merged_requirements = main(*sys.argv[1:]) - print(merged_requirements) + print(main(*sys.argv[1:])) # pylint: disable=star-args diff --git a/tools/oldest_constraints.txt b/tools/oldest_constraints.txt index 642af660e..e48d6b13c 100644 --- a/tools/oldest_constraints.txt +++ b/tools/oldest_constraints.txt @@ -50,11 +50,10 @@ ConfigArgParse==0.10.0 funcsigs==0.4 zope.hookable==4.0.4 -# Ubuntu Bionic constraints -# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some -# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure -# that Lexicon 2.x works with Certbot. -dns-lexicon==2.7.14 +# Ubuntu Bionic constraints. +# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins +# using their local-oldest-requirements.txt +dns-lexicon==2.2.1 # Plugin constraints # These aren't necessarily the oldest versions we need to support diff --git a/tools/pip_install.py b/tools/pip_install.py index 332422019..a03ed70d7 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints): # remove any extras such as [dev] pkg_dir = re.sub(r'\[\w+\]', '', args[1]) requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt') + shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) # packages like acme don't have any local oldest requirements if not os.path.isfile(requirements): - requirements = None - shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints) + return None return requirements @@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints): fd.write('{0}{1}'.format(search.group(1), os.linesep)) -def merge_requirements(tools_path, test_constraints, all_constraints): - merged_requirements = merge_module.main( - os.path.join(tools_path, 'dev_constraints.txt'), - test_constraints - ) +def merge_requirements(tools_path, requirements, test_constraints, all_constraints): + # Order of the files in the merge function matters. + # Indeed version retained for a given package will be the last version + # found when following all requirements in the given order. + # Here is the order by increasing priority: + # 1) The general development constraints (tools/dev_constraints.txt) + # 2) The general tests constraints (oldest_requirements.txt or + # certbot-auto's dependency-requirements.txt for the normal processing) + # 3) The local requirement file, typically local-oldest-requirement in oldest tests + files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints] + if requirements: + files.append(requirements) + merged_requirements = merge_module.main(*files) with open(all_constraints, 'w') as fd: fd.write(merged_requirements) @@ -92,8 +100,7 @@ def main(args): else: certbot_normal_processing(tools_path, test_constraints) - merge_requirements(tools_path, test_constraints, all_constraints) - + merge_requirements(tools_path, requirements, test_constraints, all_constraints) if requirements: call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"' .format(sys.executable, all_constraints, requirements))