Process isolated local oldest requirements for DNS plugins (#6653)

* Include local-oldest-requirements in the constraints when oldest is invoked

* Add oldest constraints for lexicon

* Skip editable requirements during merge

* Pin to lexicon 2.2.1 for oldest requirements. Override locally oldest if needed for specific providers.

certbot-dns-dnsimple/local-oldest-requirements.txt

@@ -1,3 +1,2 @@
 -e acme[dev]
 -e .[dev]
-dns-lexicon==2.2.1

certbot-dns-ovh/local-oldest-requirements.txt

@@ -1,2 +1,3 @@
 -e acme[dev]
 -e .[dev]
+dns-lexicon==2.7.14

tools/dev_constraints.txt

@@ -1,5 +1,7 @@
-# Specifies Python package versions for packages not specified in
-# letsencrypt-auto's requirements file.
+# Specifies Python package versions for development.
+# It includes in particular packages not specified in letsencrypt-auto's requirements file.
+# Some dev package versions specified here may be overridden by higher level constraints
+# files during tests (eg. letsencrypt-auto-source/pieces/dependency-requirements.txt).
 alabaster==0.7.10
 apipkg==1.4
 asn1crypto==0.22.0

tools/merge_requirements.py

@@ -5,13 +5,14 @@ Requirements files specified later take precedence over earlier ones. Only
 simple SomeProject==1.2.3 format is currently supported.
 
 """
-
 from __future__ import print_function
 
 import sys
 
+
 def read_file(file_path):
     """Reads in a Python requirements file.
+    Ignore empty lines, comments and editable requirements
 
     :param str file_path: path to requirements file
 
@@ -19,16 +20,16 @@ def read_file(file_path):
     :rtype: dict
 
     """
-    d = {}
-    with open(file_path) as f:
-        for line in f:
+    data = {}
+    with open(file_path) as file_h:
+        for line in file_h:
             line = line.strip()
-            if line and not line.startswith('#'):
+            if line and not line.startswith('#') and not line.startswith('-e'):
                 project, version = line.split('==')
                 if not version:
                     raise ValueError("Unexpected syntax '{0}'".format(line))
-                d[project] = version
-    return d
+                data[project] = version
+    return data
 
 
 def output_requirements(requirements):
@@ -37,25 +38,24 @@ def output_requirements(requirements):
     :param dict requirements: mapping from a project to its pinned version
 
     """
-    return '\n'.join('{0}=={1}'.format(k, v)
-          for k, v in sorted(requirements.items()))
+    return '\n'.join('{0}=={1}'.format(key, value)
+                     for key, value in sorted(requirements.items()))
 
 
-def main(*files):
+def main(*paths):
     """Merges multiple requirements files together and prints the result.
 
     Requirement files specified later in the list take precedence over earlier
     files.
 
-    :param tuple files: paths to requirements files
+    :param tuple paths: paths to requirements files
 
     """
-    d = {}
-    for f in files:
-        d.update(read_file(f))
-    return output_requirements(d)
+    data = {}
+    for path in paths:
+        data.update(read_file(path))
+    return output_requirements(data)
 
 
 if __name__ == '__main__':
-    merged_requirements = main(*sys.argv[1:])
-    print(merged_requirements)
+    print(main(*sys.argv[1:]))  # pylint: disable=star-args

tools/oldest_constraints.txt

@@ -50,11 +50,10 @@ ConfigArgParse==0.10.0
 funcsigs==0.4
 zope.hookable==4.0.4
 
-# Ubuntu Bionic constraints
-# Formerly only lexicon==2.2.1 is available on Bionic. But we cannot put that here because some
-# DNS plugins require higher versions. We put to the least minimal Lexicon version to ensure
-# that Lexicon 2.x works with Certbot.
-dns-lexicon==2.7.14
+# Ubuntu Bionic constraints.
+# Lexicon oldest constraint is overridden appropriately on relevant DNS provider plugins
+# using their local-oldest-requirements.txt
+dns-lexicon==2.2.1
 
 # Plugin constraints
 # These aren't necessarily the oldest versions we need to support

tools/pip_install.py

@@ -32,10 +32,10 @@ def certbot_oldest_processing(tools_path, args, test_constraints):
     # remove any extras such as [dev]
     pkg_dir = re.sub(r'\[\w+\]', '', args[1])
     requirements = os.path.join(pkg_dir, 'local-oldest-requirements.txt')
+    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
     # packages like acme don't have any local oldest requirements
     if not os.path.isfile(requirements):
-        requirements = None
-    shutil.copy(os.path.join(tools_path, 'oldest_constraints.txt'), test_constraints)
+        return None
 
     return requirements
 
@@ -53,11 +53,19 @@ def certbot_normal_processing(tools_path, test_constraints):
                 fd.write('{0}{1}'.format(search.group(1), os.linesep))
 
 
-def merge_requirements(tools_path, test_constraints, all_constraints):
-    merged_requirements = merge_module.main(
-        os.path.join(tools_path, 'dev_constraints.txt'),
-        test_constraints
-    )
+def merge_requirements(tools_path, requirements, test_constraints, all_constraints):
+    # Order of the files in the merge function matters.
+    # Indeed version retained for a given package will be the last version
+    # found when following all requirements in the given order.
+    # Here is the order by increasing priority:
+    # 1) The general development constraints (tools/dev_constraints.txt)
+    # 2) The general tests constraints (oldest_requirements.txt or
+    #    certbot-auto's dependency-requirements.txt for the normal processing)
+    # 3) The local requirement file, typically local-oldest-requirement in oldest tests
+    files = [os.path.join(tools_path, 'dev_constraints.txt'), test_constraints]
+    if requirements:
+        files.append(requirements)
+    merged_requirements = merge_module.main(*files)
     with open(all_constraints, 'w') as fd:
         fd.write(merged_requirements)
 
@@ -92,8 +100,7 @@ def main(args):
             else:
                 certbot_normal_processing(tools_path, test_constraints)
 
-            merge_requirements(tools_path, test_constraints, all_constraints)
-
+            merge_requirements(tools_path, requirements, test_constraints, all_constraints)
             if requirements:
                 call_with_print('"{0}" -m pip install --constraint "{1}" --requirement "{2}"'
                                 .format(sys.executable, all_constraints, requirements))