From 5073090a20fa59fae45b4d90bfb41635bc181911 Mon Sep 17 00:00:00 2001 From: Adrien Ferrand Date: Fri, 16 Nov 2018 00:17:36 +0100 Subject: [PATCH] Update tools/venv3.py to support py launcher on Windows (#6493) Following some inconsistencies occurred during by developments, and in the light of #6508, it decided to wrote a PR that will take fully advantage of the conversion from bash to python to the development setup tools. This PR adresses several issues when trying to use the development setup tools (`tools/venv.py` and `tools/venv3.py`: * on Windows, `python` executable is not always in PATH (default behavior) * even if the option is checked, the `python` executable is not associated to the usually symlink `python3` on Windows * on Windows again, really powerful introspection of the available Python environments can be done with `py`, the Windows Python launcher * in general for all systems, `tools/venv.py` and `tools/venv3.py` ensures that the respective Python major version will be used to setup the virtual environment if available. * finally, the best and first candidate to test should be the Python executable used to launch the `tools/venv*.py` script. It was not relevant before because it was shell scripts, but do it is. The logic is shared in `_venv_common.py`, and will be called appropriately for both scripts. In priority decreasing order, python executable will be search and tested: * from the current Python executable, as exposed by `sys.executable` * from any python or pythonX (X as a python version like 2, 3 or 2.7 or 3.4) executable available in PATH * from the Windows Python launched `py` if available Individual changes were: * Update tools/venv3.py to support py launcher on Windows * Fix typo in help message * More explicit calls with space protection * Complete refactoring to take advantage of the python runtime, and control of the compatible version to use. --- tools/_venv_common.py | 104 ++++++++++++++++++++++++++++++++++++++---- tools/pip_install.py | 17 ++++--- tools/venv.py | 22 +-------- tools/venv3.py | 22 +-------- 4 files changed, 110 insertions(+), 55 deletions(-) diff --git a/tools/_venv_common.py b/tools/_venv_common.py index 6134bd29d..c44d05bf7 100755 --- a/tools/_venv_common.py +++ b/tools/_venv_common.py @@ -8,11 +8,94 @@ import glob import time import subprocess import sys +import re + +VERSION_PATTERN = re.compile(r'^(\d+)\.(\d+).*$') + + +class PythonExecutableNotFoundError(Exception): + pass + + +def find_python_executable(python_major): + # type: (int) -> str + """ + Find the relevant python executable that is of the given python major version. + Will test, in decreasing priority order: + * the current Python interpreter + * 'pythonX' executable in PATH (with X the given major version) if available + * 'python' executable in PATH if available + * Windows Python launcher 'py' executable in PATH if available + Incompatible python versions for Certbot will be evicted (eg. Python < 3.5 on Windows) + :param int python_major: the Python major version to target (2 or 3) + :rtype: str + :return: the relevant python executable path + :raise RuntimeError: if no relevant python executable path could be found + """ + python_executable_path = None + + # First try, current python executable + if _check_version('{0}.{1}.{2}'.format( + sys.version_info[0], sys.version_info[1], sys.version_info[2]), python_major): + return sys.executable + + # Second try, with python executables in path + versions_to_test = ['2.7', '2', ''] if python_major == 2 else ['3', ''] + for one_version in versions_to_test: + try: + one_python = 'python{0}'.format(one_version) + output = subprocess.check_output([one_python, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output.strip().split()[1], python_major): + return subprocess.check_output([one_python, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + # Last try, with Windows Python launcher + try: + env_arg = '-{0}'.format(python_major) + output_version = subprocess.check_output(['py', env_arg, '--version'], + universal_newlines=True, stderr=subprocess.STDOUT) + if _check_version(output_version.strip().split()[1], python_major): + return subprocess.check_output(['py', env_arg, '-c', + 'import sys; sys.stdout.write(sys.executable);'], + universal_newlines=True) + except (subprocess.CalledProcessError, OSError): + pass + + if not python_executable_path: + raise RuntimeError('Error, no compatible Python {0} executable for Certbot could be found.' + .format(python_major)) + + +def _check_version(version_str, major_version): + search = VERSION_PATTERN.search(version_str) + + if not search: + return False + + version = (int(search.group(1)), int(search.group(2))) + + minimal_version_supported = (2, 7) + if major_version == 3 and os.name == 'nt': + minimal_version_supported = (3, 5) + elif major_version == 3: + minimal_version_supported = (3, 4) + + if version >= minimal_version_supported: + return True + + print('Incompatible python version for Certbot found: {0}'.format(version_str)) + return False + def subprocess_with_print(command): print(command) subprocess.check_call(command, shell=True) + def get_venv_python(venv_path): python_linux = os.path.join(venv_path, 'bin/python') if os.path.isfile(python_linux): @@ -25,6 +108,7 @@ def get_venv_python(venv_path): 'Error, could not find python executable in venv path {0}: is it a valid venv ?' .format(venv_path))) + def main(venv_name, venv_args, args): for path in glob.glob('*.egg-info'): if os.path.isdir(path): @@ -35,17 +119,18 @@ def main(venv_name, venv_args, args): if os.path.isdir(venv_name): os.rename(venv_name, '{0}.{1}.bak'.format(venv_name, int(time.time()))) - subprocess_with_print(' '.join([ - sys.executable, '-m', 'virtualenv', '--no-site-packages', '--setuptools', - venv_name, venv_args])) + subprocess_with_print('"{0}" -m virtualenv --no-site-packages --setuptools {1} {2}' + .format(sys.executable, venv_name, venv_args)) python_executable = get_venv_python(venv_name) - subprocess_with_print(' '.join([ - python_executable, os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py')])) - command = [python_executable, os.path.normpath('./tools/pip_install.py')] - command.extend(args) - subprocess_with_print(' '.join(command)) + subprocess_with_print('"{0}" {1}'.format( + python_executable, + os.path.normpath('./letsencrypt-auto-source/pieces/pipstrap.py'))) + subprocess_with_print('"{0}" {1} {2}'.format( + python_executable, + os.path.normpath('./tools/pip_install.py'), + ' '.join(args))) if os.path.isdir(os.path.join(venv_name, 'bin')): # Linux/OSX specific @@ -57,12 +142,13 @@ def main(venv_name, venv_args, args): # Windows specific print('---------------------------------------------------------------------------') print('Please run one of the following commands to activate developer environment:') - print('{0}\\bin\\activate.bat (for Batch)'.format(venv_name)) + print('{0}\\Scripts\\activate.bat (for Batch)'.format(venv_name)) print('.\\{0}\\Scripts\\Activate.ps1 (for Powershell)'.format(venv_name)) print('---------------------------------------------------------------------------') else: raise ValueError('Error, directory {0} is not a valid venv.'.format(venv_name)) + if __name__ == '__main__': main(os.environ.get('VENV_NAME', 'venv'), os.environ.get('VENV_ARGS', ''), diff --git a/tools/pip_install.py b/tools/pip_install.py index 2c4a47c21..8878674c9 100755 --- a/tools/pip_install.py +++ b/tools/pip_install.py @@ -20,9 +20,11 @@ import tempfile import merge_requirements as merge_module import readlink + def find_tools_path(): return os.path.dirname(readlink.main(__file__)) + def certbot_oldest_processing(tools_path, args, test_constraints): if args[0] != '-e' or len(args) != 2: raise ValueError('When CERTBOT_OLDEST is set, this script must be run ' @@ -37,6 +39,7 @@ def certbot_oldest_processing(tools_path, args, test_constraints): return requirements + def certbot_normal_processing(tools_path, test_constraints): repo_path = os.path.dirname(tools_path) certbot_requirements = os.path.normpath(os.path.join( @@ -49,6 +52,7 @@ def certbot_normal_processing(tools_path, test_constraints): if search: fd.write('{0}{1}'.format(search.group(1), os.linesep)) + def merge_requirements(tools_path, test_constraints, all_constraints): merged_requirements = merge_module.main( os.path.join(tools_path, 'dev_constraints.txt'), @@ -57,10 +61,12 @@ def merge_requirements(tools_path, test_constraints, all_constraints): with open(all_constraints, 'w') as fd: fd.write(merged_requirements) + def call_with_print(command, cwd=None): print(command) subprocess.check_call(command, shell=True, cwd=cwd or os.getcwd()) + def main(args): tools_path = find_tools_path() working_dir = tempfile.mkdtemp() @@ -77,15 +83,14 @@ def main(args): merge_requirements(tools_path, test_constraints, all_constraints) if requirements: - call_with_print(' '.join([ - sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints, - '--requirement', requirements])) + call_with_print('"{0}" -m pip install -q --constraint "{1}" --requirement "{2}"' + .format(sys.executable, all_constraints, requirements)) - command = [sys.executable, '-m', 'pip', 'install', '-q', '--constraint', all_constraints] - command.extend(args) - call_with_print(' '.join(command)) + call_with_print('"{0}" -m pip install -q --constraint "{1}" {2}' + .format(sys.executable, all_constraints, ' '.join(args))) finally: shutil.rmtree(working_dir) + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/tools/venv.py b/tools/venv.py index 8b6f92a56..93b012e76 100755 --- a/tools/venv.py +++ b/tools/venv.py @@ -1,11 +1,6 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client - -from __future__ import absolute_import - import os -import subprocess -import sys import _venv_common @@ -33,27 +28,14 @@ REQUIREMENTS = [ '-e certbot-compatibility-test', ] -def get_venv_args(): - with open(os.devnull, 'w') as fnull: - command_python2_st_code = subprocess.call( - 'command -v python2', shell=True, stdout=fnull, stderr=fnull) - if not command_python2_st_code: - return '--python python2' - - command_python27_st_code = subprocess.call( - 'command -v python2.7', shell=True, stdout=fnull, stderr=fnull) - if not command_python27_st_code: - return '--python python2.7' - - raise ValueError('Couldn\'t find python2 or python2.7 in {0}'.format(os.environ.get('PATH'))) def main(): if os.name == 'nt': raise ValueError('Certbot for Windows is not supported on Python 2.x.') - venv_args = get_venv_args() - + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(2)) _venv_common.main('venv', venv_args, REQUIREMENTS) + if __name__ == '__main__': main() diff --git a/tools/venv3.py b/tools/venv3.py index 9710806c5..c2374ba5a 100755 --- a/tools/venv3.py +++ b/tools/venv3.py @@ -1,12 +1,5 @@ #!/usr/bin/env python # Developer virtualenv setup for Certbot client - -from __future__ import absolute_import - -import os -import subprocess -import sys - import _venv_common REQUIREMENTS = [ @@ -33,22 +26,11 @@ REQUIREMENTS = [ '-e certbot-compatibility-test', ] -def get_venv_args(): - with open(os.devnull, 'w') as fnull: - where_python3_st_code = subprocess.call( - 'where python3', shell=True, stdout=fnull, stderr=fnull) - command_python3_st_code = subprocess.call( - 'command -v python3', shell=True, stdout=fnull, stderr=fnull) - - if not where_python3_st_code or not command_python3_st_code: - return '--python python3' - - raise ValueError('Couldn\'t find python3 in {0}'.format(os.environ.get('PATH'))) def main(): - venv_args = get_venv_args() - + venv_args = '--python "{0}"'.format(_venv_common.find_python_executable(3)) _venv_common.main('venv3', venv_args, REQUIREMENTS) + if __name__ == '__main__': main()