From d6929a8efb3a8cec25d6c73c5e61f99a06c8942a Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Fri, 4 Dec 2015 05:10:44 -0800
Subject: [PATCH 01/19] Initial commit

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 README.md

diff --git a/README.md b/README.md
new file mode 100644
index 000000000..bf0416817
--- /dev/null
+++ b/README.md
@@ -0,0 +1,2 @@
+# letstest
+simple aws testfarm scripts for letsencrypt client testing

From 6dab44816d214a3be861f62a17bdaa551381a757 Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Fri, 4 Dec 2015 06:32:24 -0800
Subject: [PATCH 02/19] initial commit of scripts

---
 .pylintrc                                     | 335 ++++++++++++
 README.md                                     |  34 ++
 apache2_targets.yaml                          |  57 +++
 multitester.py                                | 482 ++++++++++++++++++
 scripts/boulder_config.sh                     |  32 ++
 scripts/boulder_install.sh                    |  28 +
 scripts/test_letsencrypt_auto_apache2.sh      |  24 +
 ...st_letsencrypt_auto_certonly_standalone.sh |  14 +
 scripts/test_letsencrypt_auto_venv_only.sh    |   7 +
 targets.yaml                                  |  99 ++++
 10 files changed, 1112 insertions(+)
 create mode 100644 .pylintrc
 create mode 100644 apache2_targets.yaml
 create mode 100644 multitester.py
 create mode 100755 scripts/boulder_config.sh
 create mode 100755 scripts/boulder_install.sh
 create mode 100755 scripts/test_letsencrypt_auto_apache2.sh
 create mode 100755 scripts/test_letsencrypt_auto_certonly_standalone.sh
 create mode 100755 scripts/test_letsencrypt_auto_venv_only.sh
 create mode 100644 targets.yaml

diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 000000000..4f978cdd2
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,335 @@
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Profiled execution.
+profile=no
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+#load-plugins=linter_plugin
+
+
+[MESSAGES CONTROL]
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time. See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes
+# abstract-class-not-used cannot be disabled locally (at least in
+# pylint 1.4.1), same for abstract-class-little-used
+
+
+[REPORTS]
+
+# Set the output format. Available formats are text, parseable, colorized, msvs
+# (visual studio) and html. You can also give a reporter class, eg
+# mypackage.mymodule.MyReporterClass.
+output-format=text
+
+# Put messages in a separate file for each module / package specified on the
+# command line instead of printing them on stdout. Reports (if any) will be
+# written in a file name "pylint_global.[txt|html]".
+files-output=no
+
+# Tells whether to display a full report or only the messages
+reports=yes
+
+# Python expression which should return a note less than 10 (10 is the highest
+# note). You have access to the variables errors warning, statement which
+# respectively contain the number of errors / warnings messages and the total
+# number of statements analyzed. This is used by the global evaluation report
+# (RP0004).
+evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+
+# Add a comment according to your evaluation note. This is used by the global
+# evaluation report (RP0004).
+comment=no
+
+# Template used to display messages. This is a python new-style format string
+# used to format the message information. See doc for all details
+#msg-template=
+
+
+[BASIC]
+
+# Required attributes for module, separated by a comma
+required-attributes=
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,apply,input,file
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=f,i,j,k,ex,Run,_,fd,logger
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Regular expression matching correct function names
+function-rgx=[a-z_][a-z0-9_]{2,40}$
+
+# Naming hint for function names
+function-name-hint=[a-z_][a-z0-9_]{2,40}$
+
+# Regular expression matching correct variable names
+variable-rgx=[a-z_][a-z0-9_]{1,30}$
+
+# Naming hint for variable names
+variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for attribute names
+attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for argument names
+argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression matching correct method names
+method-rgx=[a-z_][a-z0-9_]{2,50}$
+
+# Naming hint for method names
+method-name-hint=[a-z_][a-z0-9_]{2,50}$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=(__.*__)|(test_[A-Za-z0-9_]*)|(_.*)|(.*Test$)
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging,logger
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=(unused)?_.*|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=6
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=yes
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=100
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+# List of optional constructs for which whitespace checking is disabled
+no-space-check=trailing-comma
+
+# Maximum number of lines in a module
+max-module-lines=1250
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Number of spaces of indent required inside a hanging or continued line.
+# This does something silly/broken...
+#indent-after-paren=4
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis
+ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib
+# import errors ignored only in 1.4.4
+# https://bitbucket.org/logilab/pylint/commits/cd000904c9e2
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject
+
+# When zope mode is activated, add a predefined set of Zope acquired attributes
+# to generated-members.
+zope=yes
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E0201 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[CLASSES]
+
+# List of interface methods to ignore, separated by a comma. This is used for
+# instance to not check methods defined in Zope's Interface base class.
+ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by,implementedBy,providedBy
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=6
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=15
+
+# Maximum number of return / yield for function / method body
+max-returns=6
+
+# Maximum number of branch for function / method body
+max-branches=12
+
+# Maximum number of statements in function / method body
+max-statements=50
+
+# Maximum number of parents for a class (see R0901).
+max-parents=12
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=7
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=20
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception
diff --git a/README.md b/README.md
index bf0416817..35950b18c 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,36 @@
 # letstest
 simple aws testfarm scripts for letsencrypt client testing
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+## Notes
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+## Usage
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+```
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair --query 'KeyMaterial' --output text > MyKeyPair.pem
+```
+then:
+```
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
+```
+
+see:
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+
+https://github.com/letsencrypt/boulder
+https://github.com/letsencrypt/letsencrypt
\ No newline at end of file
diff --git a/apache2_targets.yaml b/apache2_targets.yaml
new file mode 100644
index 000000000..e707b8636
--- /dev/null
+++ b/apache2_targets.yaml
@@ -0,0 +1,57 @@
+targets:
+  #-----------------------------------------------------------------------------
+  # Apache 2.4
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-116d857a
+    name: debian8.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Apache 2.2
+  # - ami: ami-0611546c
+  #   name: ubuntu12.04LTS
+  #   type: ubuntu
+  #   virt: hvm
+  #   user: ubuntu
+  # - ami: ami-e0efab88
+  #   name: debian7.8.aws.1
+  #   type: debian
+  #   virt: hvm
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]
+  # - ami: ami-e6eeaa8e
+  #   name: debian7.8.aws.1_32bit
+  #   type: debian
+  #   virt: pv
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]
\ No newline at end of file
diff --git a/multitester.py b/multitester.py
new file mode 100644
index 000000000..116d5b5a8
--- /dev/null
+++ b/multitester.py
@@ -0,0 +1,482 @@
+"""
+Letsencrypt Integration Test Tool
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+Notes:
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+Usage:
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
+ --query 'KeyMaterial' --output text > MyKeyPair.pem
+then:
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker test_letsencrypt_auto_venv_only.sh
+see:
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+"""
+
+from __future__ import print_function
+from __future__ import with_statement
+
+import sys, os, time, argparse, socket
+import multiprocessing as mp
+from multiprocessing import Manager
+import urllib2
+import yaml
+import boto3
+import fabric
+from fabric.api import run, execute, local, env, sudo, cd, lcd
+from fabric.operations import get, put
+from fabric.context_managers import shell_env
+
+# Command line parser
+#-------------------------------------------------------------------------------
+parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.')
+parser.add_argument('config_file',
+                    help='yaml configuration file for AWS server cluster')
+parser.add_argument('key_file',
+                    help='key file (<keyname>.pem) for AWS')
+parser.add_argument('aws_profile',
+                    help='profile for AWS (i.e. as in ~/.aws/certificates)')
+parser.add_argument('test_script',
+                    default='test_letsencrypt_auto_certonly_standalone.sh',
+                    help='name of bash script in /scripts to deploy and run')
+#parser.add_argument('--script_args',
+#                    nargs='+',
+#                    help='space-delimited list of arguments to pass to the bash test script',
+#                    required=False)
+parser.add_argument('--repo',
+                    default='https://github.com/letsencrypt/letsencrypt.git',
+                    help='letsencrypt git repo to use')
+parser.add_argument('--branch',
+                    default='~',
+                    help='letsencrypt git branch to trial')
+parser.add_argument('--pull_request',
+                    default='~',
+                    help='letsencrypt/letsencrypt pull request to trial')
+parser.add_argument('--merge_master',
+                    action='store_true',
+                    help="if set merges PR into master branch of letsencrypt/letsencrypt")
+parser.add_argument('--saveinstances',
+                    action='store_true',
+                    help="don't kill EC2 instances after run, useful for debugging")
+cl_args = parser.parse_args()
+
+# Credential Variables
+#-------------------------------------------------------------------------------
+# assumes naming: <key_filename> = <keyname>.pem
+KEYFILE = cl_args.key_file
+KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0]
+PROFILE = cl_args.aws_profile
+
+# Globals
+#-------------------------------------------------------------------------------
+BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1
+LOGDIR = "" #points to logging / working directory
+# boto3/AWS api globals
+AWS_SESSION = None
+EC2 = None
+
+# Boto3/AWS automation functions
+#-------------------------------------------------------------------------------
+def make_security_group():
+    # will fail if security group of GroupName already exists
+    # cannot have duplicate SGs of the same name
+    mysg = EC2.create_security_group(GroupName="letsencrypt_test",
+                                     Description='security group for automated testing')
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443)
+    # for boulder wfe (http) server
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000)
+    # for mosh
+    mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000)
+    return mysg
+
+def make_instance(instance_name,
+                  ami_id,
+                  keyname,
+                  machine_type='t2.micro',
+                  security_groups=['letsencrypt_test'],
+                  userdata=""): #userdata contains bash or cloud-init script
+
+    new_instance = EC2.create_instances(
+        ImageId=ami_id,
+        SecurityGroups=security_groups,
+        KeyName=keyname,
+        MinCount=1,
+        MaxCount=1,
+        UserData=userdata,
+        InstanceType=machine_type)[0]
+
+    # brief pause to prevent rare EC2 error
+    time.sleep(0.5)
+
+    # give instance a name
+    new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
+    return new_instance
+
+def terminate_and_clean(instances):
+    """
+    Some AMIs specify EBS stores that won't delete on instance termination.
+    These must be manually deleted after shutdown.
+    """
+    volumes_to_delete = []
+    for instance in instances:
+        for bdmap in instance.block_device_mappings:
+            if 'Ebs' in bdmap.keys():
+                if not bdmap['Ebs']['DeleteOnTermination']:
+                    volumes_to_delete.append(bdmap['Ebs']['VolumeId'])
+
+    for instance in instances:
+        instance.terminate()
+
+    # can't delete volumes until all attaching instances are terminated
+    _ids = [instance.id for instance in instances]
+    all_terminated = False
+    while not all_terminated:
+        all_terminated = True
+        for _id in _ids:
+            # necessary to reinit object for boto3 to get true state
+            inst = EC2.Instance(id=_id)
+            if inst.state['Name'] != 'terminated':
+                all_terminated = False
+        time.sleep(5)
+
+    for vol_id in volumes_to_delete:
+        volume = EC2.Volume(id=vol_id)
+        volume.delete()
+
+    return volumes_to_delete
+
+
+# Helper Routines
+#-------------------------------------------------------------------------------
+def block_until_http_ready(urlstring, wait_time=10, timeout=240):
+    "Blocks until server at urlstring can respond to http requests"
+    server_ready = False
+    t_elapsed = 0
+    while not server_ready and t_elapsed < timeout:
+        try:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            req = urllib2.Request(urlstring)
+            response = urllib2.urlopen(req)
+            #if response.code == 200:
+            server_ready = True
+        except urllib2.URLError:
+            pass
+        time.sleep(wait_time)
+        t_elapsed += wait_time
+
+def block_until_ssh_open(ipstring, wait_time=10, timeout=120):
+    "Blocks until server at ipstring has an open port 22"
+    reached = False
+    t_elapsed = 0
+    while not reached and t_elapsed < timeout:
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect((ipstring, 22))
+            reached = True
+        except socket.error as err:
+            time.sleep(wait_time)
+            t_elapsed += wait_time
+    sock.close()
+
+def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20):
+    "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections"
+    # the reinstantiation from id is necessary to force boto3
+    # to correctly update the 'state' variable during init
+    _id = booting_instance.id
+    _instance = EC2.Instance(id=_id)
+    _state = _instance.state['Name']
+    _ip = _instance.public_ip_address
+    while _state != 'running' or _ip is None:
+        time.sleep(wait_time)
+        _instance = EC2.Instance(id=_id)
+        _state = _instance.state['Name']
+        _ip = _instance.public_ip_address
+    block_until_ssh_open(_ip)
+    time.sleep(extra_wait_time)
+    return _instance
+
+
+# Fabric Routines
+#-------------------------------------------------------------------------------
+def local_git_clone(repo_url):
+    "clones master of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_branch(repo_url, branch_name):
+    "clones branch <branch_name> of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s --branch %s --single-branch'%(repo_url, branch_name))
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_PR(repo_url, PRnumstr, merge_master=True):
+    "clones specified pull request from repo_url and optionally merges into master"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr)
+        local('cd letsencrypt && git co lePRtest')
+        if merge_master:
+            local('cd letsencrypt && git remote update origin')
+            local('cd letsencrypt && git merge origin/master -m "testmerge"')
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_repo_to_remote():
+    "copies local tarball of repo to remote"
+    with lcd(LOGDIR):
+        put(local_path='le.tar.gz', remote_path='')
+        run('tar xzf le.tar.gz')
+
+def local_repo_clean():
+    "delete tarball"
+    with lcd(LOGDIR):
+        local('rm le.tar.gz')
+
+def deploy_script(scriptname, *args):
+    "copies to remote and executes local script"
+    with lcd('scripts'):
+        put(local_path=scriptname, remote_path='', mirror_local_mode=True)
+    args_str = ' '.join(args)
+    run('./'+scriptname+' '+args_str)
+
+def run_boulder():
+    with cd('$GOPATH/src/github.com/letsencrypt/boulder'):
+        run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost')
+        run('nohup ./start.py >& /dev/null < /dev/null &')
+
+def config_and_launch_boulder(instance):
+    execute(deploy_script, 'boulder_config.sh')
+    execute(run_boulder)
+
+def install_and_launch_letsencrypt(instance, boulder_url):
+    execute(local_repo_to_remote)
+    with shell_env(BOULDER_URL=boulder_url):
+        execute(deploy_script, cl_args.test_script)
+
+def grab_letsencrypt_log():
+    "grabs letsencrypt.log via cat into logged stdout"
+    sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \
+    cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi')
+    # fallback file if /var/log is unwriteable...? correct?
+    sudo('if [ -f ./letsencrypt.log ]; then \
+    cat ./letsencrypt.log; else echo "[nolocallog]"; fi')
+
+#-------------------------------------------------------------------------------
+# SCRIPT BEGINS
+#-------------------------------------------------------------------------------
+
+# Set up local copy of git repo
+#-------------------------------------------------------------------------------
+LOGDIR = "letest-%d"%int(time.time())
+print("Making local dir for test repo and logs: %s"%LOGDIR)
+local('mkdir %s'%LOGDIR)
+
+# figure out what git object to test and locally create it in LOGDIR
+print("Making local git repo")
+try:
+    if cl_args.pull_request != '~':
+        print('Testing PR %s '%cl_args.pull_request,
+              "MERGING into master" if cl_args.merge_master else "")
+        execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
+    elif cl_args.branch != '~':
+        print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
+        execute(local_git_branch, cl_args.repo, cl_args.branch)
+    else:
+        print('Testing master of %s'%cl_args.repo)
+        execute(local_git_clone, cl_args.repo)
+except FabricException:
+    print("FAIL: trouble with git repo")
+    exit()
+
+
+# Set up EC2 instances
+#-------------------------------------------------------------------------------
+configdata = yaml.load(open(cl_args.config_file, 'r'))
+targetlist = configdata['targets']
+print('Testing against these images: [%d total]'%len(targetlist))
+for target in targetlist:
+    print(target['ami'], target['name'])
+
+print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
+AWS_SESSION = boto3.session.Session(profile_name=PROFILE)
+EC2 = AWS_SESSION.resource('ec2')
+
+print("Making Security Group")
+sg_exists = False
+for sg in EC2.security_groups.all():
+    if sg.group_name == 'letsencrypt_test':
+        sg_exists = True
+        print("  %s already exists"%'letsencrypt_test')
+if not sg_exists:
+    make_security_group()
+    time.sleep(30)
+
+print("Requesting Instances...")
+boulder_server = make_instance('le-boulderserver',
+                               BOULDER_AMI,
+                               KEYNAME,
+                               #machine_type='t2.micro',
+                               machine_type='t2.medium',
+                               security_groups=['letsencrypt_test'])
+
+instances = []
+for target in targetlist:
+    if target['virt'] == 'hvm':
+        machine_type = 't2.micro'
+    else:
+        machine_type = 't1.micro'
+    if 'userdata' in target.keys():
+        userdata = target['userdata']
+    else:
+        userdata = ''
+    instances.append( make_instance('le-%s'%target['name'],
+                                   target['ami'],
+                                   KEYNAME,
+                                   machine_type=machine_type,
+                                   userdata=userdata) )
+
+
+# Configure and launch boulder server
+#-------------------------------------------------------------------------------
+print("Waiting on Boulder Server")
+boulder_server = block_until_instance_ready(boulder_server)
+print(" server %s"%boulder_server)
+
+print("Configuring and Launching Boulder")
+
+# Fabric library controlled through global env parameters
+env.key_filename = KEYFILE
+env.shell = '/bin/bash -l -i -c'
+env.connection_attempts = 5
+env.timeout = 10
+# replace default SystemExit thrown by fabric during trouble
+class FabricException(Exception):
+    pass
+env['abort_exception'] = FabricException
+
+# env.host_string defines the ssh user and host for connection
+env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
+print("Boulder Server at (SSH):", env.host_string)
+config_and_launch_boulder(boulder_server)
+# blocking often unnecessary, but cheap EC2 VMs can get very slow
+block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
+                       wait_time=10,
+                       timeout=500)
+
+boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
+print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
+print("Boulder Server at (EC2 private ip): %s"%boulder_url)
+
+# Install and launch client scripts in parallel
+#-------------------------------------------------------------------------------
+print("Running letsencrypt clients in parallel - output routed to log files.")
+# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
+# the latter has implementation flaws that deadlock it in some circumstances)
+manager = Manager()
+outqueue = manager.Queue()
+inqueue = manager.Queue()
+SENTINEL = None #queue kill signal
+
+# launch as many processes as clients to test
+num_processes = len(targetlist)
+jobs = [] #keep a reference to current procs
+
+def test_client_process(inqueue, outqueue):
+    cur_proc = mp.current_process()
+    for inreq in iter(inqueue.get, SENTINEL):
+        ii, target = inreq
+
+        #save all stdout to log file
+        sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')
+
+        print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
+        instances[ii] = block_until_instance_ready(instances[ii])
+        print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
+        env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
+        print(env.host_string)
+
+        try:
+            install_and_launch_letsencrypt(instances[ii], boulder_url)
+            outqueue.put((ii, target, 'pass'))
+            print("%s - %s SUCCESS"%(target['ami'], target['name']))
+        except:
+            outqueue.put((ii, target, 'fail'))
+            print("%s - %s FAIL"%(target['ami'], target['name']))
+            pass
+
+        # append server letsencrypt.log to each per-machine output log
+        print("\n\nletsencrypt.log\n" + "-"*80 + "\n")
+        try:
+            execute(grab_letsencrypt_log)
+        except:
+            print("log fail\n")
+            pass
+
+# initiate process execution
+for i in range(num_processes):
+    p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
+    jobs.append(p)
+    p.daemon = True  # kills subprocesses if parent is killed
+    p.start()
+
+# fill up work queue
+for ii, target in enumerate(targetlist):
+    inqueue.put((ii, target))
+
+# add SENTINELs to end client processes
+for i in range(num_processes):
+    inqueue.put(SENTINEL)
+# wait on termination of client processes
+for p in jobs:
+    p.join()
+# add SENTINEL to output queue
+outqueue.put(SENTINEL)
+
+# clean up
+execute(local_repo_clean)
+
+# print and save summary results
+results_file = open(LOGDIR+'/results', 'w')
+outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
+outputs.sort(key=lambda x: x[0])
+for outq in outputs:
+    ii, target, status = outq
+    print('%d %s %s'%(ii, target['name'], status))
+    results_file.write('%d %s %s\n'%(ii, target['name'], status))
+results_file.close()
+
+if not cl_args.saveinstances:
+    print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
+    boulder_server.terminate()
+    terminate_and_clean(instances)
+else:
+    # print login information for the boxes for debugging
+    for ii, target in enumerate(targetlist):
+        print(target['name'],
+              target['ami'],
+              "%s@%s"%(target['user'],instances[ii].public_ip_address))
+
+# kill any connections
+fabric.network.disconnect_all()
diff --git a/scripts/boulder_config.sh b/scripts/boulder_config.sh
new file mode 100755
index 000000000..1ef63ca10
--- /dev/null
+++ b/scripts/boulder_config.sh
@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+# Configures and Launches Boulder Server installed on
+# us-east-1 ami-5f490b35 bouldertestserver (boulder commit 8b433f54dab)
+
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+# get local DNS resolver for VPC
+resolver_ip=$(grep nameserver /etc/resolv.conf |cut -d" " -f2 |head -1)
+resolver=$resolver_ip':53'
+
+# modifies integration testing boulder setup for local AWS VPC network
+# connections instead of localhost
+cd $GOPATH/src/github.com/letsencrypt/boulder
+# configure boulder to receive outside connection on 4000
+sed -i '/listenAddress/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+sed -i '/baseURL/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+# change test ports to real
+sed -i '/httpPort/ s/5002/80/' ./test/boulder-config.json
+sed -i '/httpsPort/ s/5001/443/' ./test/boulder-config.json
+sed -i '/tlsPort/ s/5001/443/' ./test/boulder-config.json
+# set local dns resolver
+sed -i '/dnsResolver/ s/127.0.0.1:8053/'$resolver'/' ./test/boulder-config.json
+
+# start rabbitMQ
+#go run cmd/rabbitmq-setup/main.go -server amqp://localhost
+# start acme services
+#nohup ./start.py >& /dev/null < /dev/null &
+#./start.py
diff --git a/scripts/boulder_install.sh b/scripts/boulder_install.sh
new file mode 100755
index 000000000..b5ddf2c5b
--- /dev/null
+++ b/scripts/boulder_install.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -x
+
+# >>>> only tested on Ubuntu 14.04LTS <<<<
+
+# non-interactive install of mariadb and other dependencies
+export DEBIAN_FRONTEND=noninteractive
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS'
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS'
+apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server
+sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');"
+
+# install go
+wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
+tar xzvf go1.5.1.linux-amd64.tar.gz
+mkdir gocode
+echo "export GOROOT=/home/ubuntu/go \n\
+      export GOPATH=/home/ubuntu/gocode\n\
+      export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc
+
+# install boulder and its go dependencies
+go get -d github.com/letsencrypt/boulder/...
+cd $GOPATH/src/github.com/letsencrypt/boulder
+wget https://github.com/jsha/boulder-tools/raw/master/goose.gz
+mkdir $GOPATH/bin
+zcat goose.gz > $GOPATH/bin/goose
+chmod +x $GOPATH/bin/goose
+./test/create_db.sh
+go get github.com/jsha/listenbuddy
diff --git a/scripts/test_letsencrypt_auto_apache2.sh b/scripts/test_letsencrypt_auto_apache2.sh
new file mode 100755
index 000000000..24980361a
--- /dev/null
+++ b/scripts/test_letsencrypt_auto_apache2.sh
@@ -0,0 +1,24 @@
+#!/bin/bash -x
+
+#install apache2 on apt systems
+# debian doesn't come with curl
+sudo apt-get update
+sudo apt-get -y --no-upgrade install apache2 curl
+
+# $BOULDER_URL is dynamically set at execution
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+# For apache 2.4, set up ServerName
+sudo sed -i '/ServerName/ s/#ServerName/ServerName/' \
+     /etc/apache2/sites-available/000-default.conf
+sudo sed -i '/ServerName/ s/www.example.com/'$public_host'/' \
+    /etc/apache2/sites-available/000-default.conf
+
+# run letsencrypt-apache2 via letsencrypt-auto
+cd letsencrypt
+./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect --register-unsafely-without-email \
+                   --domain $public_host --server $BOULDER_URL
diff --git a/scripts/test_letsencrypt_auto_certonly_standalone.sh b/scripts/test_letsencrypt_auto_certonly_standalone.sh
new file mode 100755
index 000000000..e82c81bd4
--- /dev/null
+++ b/scripts/test_letsencrypt_auto_certonly_standalone.sh
@@ -0,0 +1,14 @@
+#!/bin/bash -x
+
+# $BOULDER_URL is dynamically set at execution
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+cd letsencrypt
+./letsencrypt-auto certonly -v --standalone --debug \
+                   --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect \
+                   --register-unsafely-without-email \
+                   --domain $public_host --server $BOULDER_URL
diff --git a/scripts/test_letsencrypt_auto_venv_only.sh b/scripts/test_letsencrypt_auto_venv_only.sh
new file mode 100755
index 000000000..e6b7aed8d
--- /dev/null
+++ b/scripts/test_letsencrypt_auto_venv_only.sh
@@ -0,0 +1,7 @@
+#!/bin/bash -x
+
+# $BOULDER_URL is dynamically set at execution
+
+cd letsencrypt
+# help installs virtualenv and does nothing else
+./letsencrypt-auto -v --help all
diff --git a/targets.yaml b/targets.yaml
new file mode 100644
index 000000000..4547366b3
--- /dev/null
+++ b/targets.yaml
@@ -0,0 +1,99 @@
+targets:
+  #-----------------------------------------------------------------------------
+  #Ubuntu
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-0611546c
+    name: ubuntu12.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  #-----------------------------------------------------------------------------
+  # Debian
+  - ami: ami-116d857a
+    name: debian8.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  - ami: ami-e0efab88
+    name: debian7.8.aws.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  - ami: ami-e6eeaa8e
+    name: debian7.8.aws.1_32bit
+    type: debian
+    virt: pv
+    user: admin
+    userdata: |
+      cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Other Redhat Distros
+  - ami: ami-60b6c60a
+    name: amazonlinux-2015.09.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-0d4cfd66
+    name: amazonlinux-2015.03.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-a8d369c0
+    name: RHEL7
+    type: redhat
+    virt: hvm
+    user: ec2-user
+  - ami: ami-518bfb3b
+    name: fedora23
+    type: fedora
+    virt: hvm
+    user: fedora
+  #-----------------------------------------------------------------------------
+  # CentOS
+  # These Marketplace AMIs must, irritatingly, have their terms manually
+  # agreed to on the AWS marketplace site for any new AWS account using them...
+  # - ami: ami-61bbf104
+  #   name: centos7
+  #   type: centos
+  #   virt: hvm
+  #   user: centos
+  # # centos6 requires EPEL repo added
+  # - ami: ami-57cd8732
+  #   name: centos6
+  #   type: centos
+  #   virt: hvm
+  #   user: centos
+  #   userdata: |
+  #     #cloud-config
+  #     runcmd:
+  #       - [ yum, install, -y, epel-release ]
+  #       - [ iptables, -F ]

From 372578ca92b56fc5a7e55b3ae6779cb1b776d1ce Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Fri, 4 Dec 2015 14:35:57 -0800
Subject: [PATCH 03/19] passing in instance data as environment variables

---
 multitester.py                                     |  5 ++++-
 scripts/test_letsencrypt_auto_apache2.sh           | 14 +++++++-------
 .../test_letsencrypt_auto_certonly_standalone.sh   | 13 +++++++------
 scripts/test_letsencrypt_auto_venv_only.sh         |  2 +-
 4 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/multitester.py b/multitester.py
index 116d5b5a8..dbf91143b 100644
--- a/multitester.py
+++ b/multitester.py
@@ -272,7 +272,10 @@ def config_and_launch_boulder(instance):
 
 def install_and_launch_letsencrypt(instance, boulder_url):
     execute(local_repo_to_remote)
-    with shell_env(BOULDER_URL=boulder_url):
+    with shell_env(BOULDER_URL=boulder_url,
+                   PUBLIC_IP=instance.public_ip_address,
+                   PRIVATE_IP=instance.private_ip_address,
+                   PUBLIC_HOSTNAME=instance.public_dns_name):
         execute(deploy_script, cl_args.test_script)
 
 def grab_letsencrypt_log():
diff --git a/scripts/test_letsencrypt_auto_apache2.sh b/scripts/test_letsencrypt_auto_apache2.sh
index 24980361a..087a2eb13 100755
--- a/scripts/test_letsencrypt_auto_apache2.sh
+++ b/scripts/test_letsencrypt_auto_apache2.sh
@@ -3,22 +3,22 @@
 #install apache2 on apt systems
 # debian doesn't come with curl
 sudo apt-get update
-sudo apt-get -y --no-upgrade install apache2 curl
+sudo apt-get -y --no-upgrade install apache2 #curl
 
-# $BOULDER_URL is dynamically set at execution
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
 # fetch instance data from EC2 metadata service
-public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
 
 # For apache 2.4, set up ServerName
 sudo sed -i '/ServerName/ s/#ServerName/ServerName/' \
      /etc/apache2/sites-available/000-default.conf
-sudo sed -i '/ServerName/ s/www.example.com/'$public_host'/' \
+sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' \
     /etc/apache2/sites-available/000-default.conf
 
 # run letsencrypt-apache2 via letsencrypt-auto
 cd letsencrypt
 ./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
-                   --domain $public_host --server $BOULDER_URL
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
diff --git a/scripts/test_letsencrypt_auto_certonly_standalone.sh b/scripts/test_letsencrypt_auto_certonly_standalone.sh
index e82c81bd4..10d7c3b5e 100755
--- a/scripts/test_letsencrypt_auto_certonly_standalone.sh
+++ b/scripts/test_letsencrypt_auto_certonly_standalone.sh
@@ -1,14 +1,15 @@
 #!/bin/bash -x
 
-# $BOULDER_URL is dynamically set at execution
-# fetch instance data from EC2 metadata service
-public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
+
+# with curl, instance metadata available from EC2 metadata service:
+#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
 
 cd letsencrypt
 ./letsencrypt-auto certonly -v --standalone --debug \
                    --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect \
                    --register-unsafely-without-email \
-                   --domain $public_host --server $BOULDER_URL
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
diff --git a/scripts/test_letsencrypt_auto_venv_only.sh b/scripts/test_letsencrypt_auto_venv_only.sh
index e6b7aed8d..476ad8bde 100755
--- a/scripts/test_letsencrypt_auto_venv_only.sh
+++ b/scripts/test_letsencrypt_auto_venv_only.sh
@@ -1,6 +1,6 @@
 #!/bin/bash -x
 
-# $BOULDER_URL is dynamically set at execution
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
 
 cd letsencrypt
 # help installs virtualenv and does nothing else

From 261c421b2552e331939a05fbda4d8658292b9a36 Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Fri, 4 Dec 2015 14:41:50 -0800
Subject: [PATCH 04/19] minor cleanup

---
 .pylintrc      | 335 -------------------------------------------------
 multitester.py |   6 +-
 2 files changed, 3 insertions(+), 338 deletions(-)
 delete mode 100644 .pylintrc

diff --git a/.pylintrc b/.pylintrc
deleted file mode 100644
index 4f978cdd2..000000000
--- a/.pylintrc
+++ /dev/null
@@ -1,335 +0,0 @@
-[MASTER]
-
-# Specify a configuration file.
-#rcfile=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Profiled execution.
-profile=no
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-#load-plugins=linter_plugin
-
-
-[MESSAGES CONTROL]
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time. See also the "--disable" option for examples.
-#enable=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-disable=fixme,locally-disabled,abstract-class-not-used,abstract-class-little-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name,too-many-instance-attributes
-# abstract-class-not-used cannot be disabled locally (at least in
-# pylint 1.4.1), same for abstract-class-little-used
-
-
-[REPORTS]
-
-# Set the output format. Available formats are text, parseable, colorized, msvs
-# (visual studio) and html. You can also give a reporter class, eg
-# mypackage.mymodule.MyReporterClass.
-output-format=text
-
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]".
-files-output=no
-
-# Tells whether to display a full report or only the messages
-reports=yes
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Add a comment according to your evaluation note. This is used by the global
-# evaluation report (RP0004).
-comment=no
-
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details
-#msg-template=
-
-
-[BASIC]
-
-# Required attributes for module, separated by a comma
-required-attributes=
-
-# List of builtins function names that should not be used, separated by a comma
-bad-functions=map,filter,apply,input,file
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=f,i,j,k,ex,Run,_,fd,logger
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,bar,baz,toto,tutu,tata
-
-# Colon-delimited sets of names that determine each other's naming style when
-# the name regexes allow several styles.
-name-group=
-
-# Include a hint for the correct naming format with invalid-name
-include-naming-hint=no
-
-# Regular expression matching correct function names
-function-rgx=[a-z_][a-z0-9_]{2,40}$
-
-# Naming hint for function names
-function-name-hint=[a-z_][a-z0-9_]{2,40}$
-
-# Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{1,30}$
-
-# Naming hint for variable names
-variable-name-hint=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression matching correct constant names
-const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-
-# Naming hint for constant names
-const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
-
-# Regular expression matching correct attribute names
-attr-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Naming hint for attribute names
-attr-name-hint=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression matching correct argument names
-argument-rgx=[a-z_][a-z0-9_]{2,30}$
-
-# Naming hint for argument names
-argument-name-hint=[a-z_][a-z0-9_]{2,30}$
-
-# Regular expression matching correct class attribute names
-class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
-
-# Naming hint for class attribute names
-class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
-
-# Regular expression matching correct inline iteration names
-inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
-
-# Naming hint for inline iteration names
-inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
-
-# Regular expression matching correct class names
-class-rgx=[A-Z_][a-zA-Z0-9]+$
-
-# Naming hint for class names
-class-name-hint=[A-Z_][a-zA-Z0-9]+$
-
-# Regular expression matching correct module names
-module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-# Naming hint for module names
-module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
-
-# Regular expression matching correct method names
-method-rgx=[a-z_][a-z0-9_]{2,50}$
-
-# Naming hint for method names
-method-name-hint=[a-z_][a-z0-9_]{2,50}$
-
-# Regular expression which should only match function or class names that do
-# not require a docstring.
-no-docstring-rgx=(__.*__)|(test_[A-Za-z0-9_]*)|(_.*)|(.*Test$)
-
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,XXX,TODO
-
-
-[LOGGING]
-
-# Logging modules to check that the string format arguments are in logging
-# function parameter format
-logging-modules=logging,logger
-
-
-[VARIABLES]
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# A regular expression matching the name of dummy variables (i.e. expectedly
-# not used).
-dummy-variables-rgx=(unused)?_.*|dummy
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
-
-
-[SIMILARITIES]
-
-# Minimum lines number of a similarity.
-min-similarity-lines=6
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-# Ignore imports when computing similarities.
-ignore-imports=yes
-
-
-[FORMAT]
-
-# Maximum number of characters on a single line.
-max-line-length=100
-
-# Regexp for a line that is allowed to be longer than the limit.
-ignore-long-lines=^\s*(# )?<?https?://\S+>?$
-
-# Allow the body of an if to be on the same line as the test if there is no
-# else.
-single-line-if-stmt=no
-
-# List of optional constructs for which whitespace checking is disabled
-no-space-check=trailing-comma
-
-# Maximum number of lines in a module
-max-module-lines=1250
-
-# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
-# tab).
-indent-string='    '
-
-# Number of spaces of indent required inside a hanging or continued line.
-# This does something silly/broken...
-#indent-after-paren=4
-
-
-[TYPECHECK]
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis
-ignored-modules=pkg_resources,confargparse,argparse,six.moves,six.moves.urllib
-# import errors ignored only in 1.4.4
-# https://bitbucket.org/logilab/pylint/commits/cd000904c9e2
-
-# List of classes names for which member attributes should not be checked
-# (useful for classes with attributes dynamically set).
-ignored-classes=SQLObject
-
-# When zope mode is activated, add a predefined set of Zope acquired attributes
-# to generated-members.
-zope=yes
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E0201 when accessed. Python regular
-# expressions are accepted.
-generated-members=REQUEST,acl_users,aq_parent
-
-
-[IMPORTS]
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,TERMIOS,Bastion,rexec
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
-
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
-
-
-[CLASSES]
-
-# List of interface methods to ignore, separated by a comma. This is used for
-# instance to not check methods defined in Zope's Interface base class.
-ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by,implementedBy,providedBy
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,__new__,setUp
-
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=mcs
-
-
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=6
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*
-
-# Maximum number of locals for function / method body
-max-locals=15
-
-# Maximum number of return / yield for function / method body
-max-returns=6
-
-# Maximum number of branch for function / method body
-max-branches=12
-
-# Maximum number of statements in function / method body
-max-statements=50
-
-# Maximum number of parents for a class (see R0901).
-max-parents=12
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
-
-
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception
diff --git a/multitester.py b/multitester.py
index dbf91143b..02645eae4 100644
--- a/multitester.py
+++ b/multitester.py
@@ -354,11 +354,11 @@ for target in targetlist:
         userdata = target['userdata']
     else:
         userdata = ''
-    instances.append( make_instance('le-%s'%target['name'],
+    instances.append(make_instance('le-%s'%target['name'],
                                    target['ami'],
                                    KEYNAME,
                                    machine_type=machine_type,
-                                   userdata=userdata) )
+                                   userdata=userdata))
 
 
 # Configure and launch boulder server
@@ -479,7 +479,7 @@ else:
     for ii, target in enumerate(targetlist):
         print(target['name'],
               target['ami'],
-              "%s@%s"%(target['user'],instances[ii].public_ip_address))
+              "%s@%s"%(target['user'], instances[ii].public_ip_address))
 
 # kill any connections
 fabric.network.disconnect_all()

From 547b9b9244e07c1a8dba4061e2081a41a4b1110e Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Fri, 4 Dec 2015 15:18:51 -0800
Subject: [PATCH 05/19] specify test script by path

---
 multitester.py | 35 ++++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/multitester.py b/multitester.py
index 02645eae4..daf6d49e5 100644
--- a/multitester.py
+++ b/multitester.py
@@ -23,7 +23,7 @@ Usage:
 >aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
  --query 'KeyMaterial' --output text > MyKeyPair.pem
 then:
->python multitester.py targets.yaml MyKeyPair.pem HappyHacker test_letsencrypt_auto_venv_only.sh
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
 see:
   https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
   https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
@@ -54,7 +54,7 @@ parser.add_argument('aws_profile',
                     help='profile for AWS (i.e. as in ~/.aws/certificates)')
 parser.add_argument('test_script',
                     default='test_letsencrypt_auto_certonly_standalone.sh',
-                    help='name of bash script in /scripts to deploy and run')
+                    help='path of bash script in to deploy and run')
 #parser.add_argument('--script_args',
 #                    nargs='+',
 #                    help='space-delimited list of arguments to pass to the bash test script',
@@ -254,12 +254,13 @@ def local_repo_clean():
     with lcd(LOGDIR):
         local('rm le.tar.gz')
 
-def deploy_script(scriptname, *args):
+def deploy_script(scriptpath, *args):
     "copies to remote and executes local script"
-    with lcd('scripts'):
-        put(local_path=scriptname, remote_path='', mirror_local_mode=True)
+    #with lcd('scripts'):
+    put(local_path=scriptpath, remote_path='', mirror_local_mode=True)
+    scriptfile = os.path.split(scriptpath)[1]
     args_str = ' '.join(args)
-    run('./'+scriptname+' '+args_str)
+    run('./'+scriptfile+' '+args_str)
 
 def run_boulder():
     with cd('$GOPATH/src/github.com/letsencrypt/boulder'):
@@ -267,7 +268,7 @@ def run_boulder():
         run('nohup ./start.py >& /dev/null < /dev/null &')
 
 def config_and_launch_boulder(instance):
-    execute(deploy_script, 'boulder_config.sh')
+    execute(deploy_script, 'scripts/boulder_config.sh')
     execute(run_boulder)
 
 def install_and_launch_letsencrypt(instance, boulder_url):
@@ -290,6 +291,16 @@ def grab_letsencrypt_log():
 # SCRIPT BEGINS
 #-------------------------------------------------------------------------------
 
+# Fabric library controlled through global env parameters
+env.key_filename = KEYFILE
+env.shell = '/bin/bash -l -i -c'
+env.connection_attempts = 5
+env.timeout = 10
+# replace default SystemExit thrown by fabric during trouble
+class FabricException(Exception):
+    pass
+env['abort_exception'] = FabricException
+
 # Set up local copy of git repo
 #-------------------------------------------------------------------------------
 LOGDIR = "letest-%d"%int(time.time())
@@ -369,16 +380,6 @@ print(" server %s"%boulder_server)
 
 print("Configuring and Launching Boulder")
 
-# Fabric library controlled through global env parameters
-env.key_filename = KEYFILE
-env.shell = '/bin/bash -l -i -c'
-env.connection_attempts = 5
-env.timeout = 10
-# replace default SystemExit thrown by fabric during trouble
-class FabricException(Exception):
-    pass
-env['abort_exception'] = FabricException
-
 # env.host_string defines the ssh user and host for connection
 env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
 print("Boulder Server at (SSH):", env.host_string)

From 50232f3fec359522bfbb34d74eecbdbe7d20491e Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Fri, 4 Dec 2015 15:21:58 -0800
Subject: [PATCH 06/19] report script used and logging dir

---
 multitester.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/multitester.py b/multitester.py
index daf6d49e5..7c7bd3e2b 100644
--- a/multitester.py
+++ b/multitester.py
@@ -395,7 +395,8 @@ print("Boulder Server at (EC2 private ip): %s"%boulder_url)
 
 # Install and launch client scripts in parallel
 #-------------------------------------------------------------------------------
-print("Running letsencrypt clients in parallel - output routed to log files.")
+print("Uploading and running test script in parallel: %s"%cl_args.test_script)
+print("Output routed to log files in %s"%LOGDIR)
 # (Advice: always use Manager.Queue, never regular multiprocessing.Queue
 # the latter has implementation flaws that deadlock it in some circumstances)
 manager = Manager()

From 37c02927d59e247ed9b9da7496c79aa2ae1ac883 Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Wed, 9 Dec 2015 16:52:02 -0800
Subject: [PATCH 07/19] current nonworking progress at scripting install of
 httpd on centos-like systems

---
 README.md                                |  9 ++--
 multitester.py                           |  7 +--
 scripts/test_apache2.sh                  | 50 +++++++++++++++++
 scripts/test_letsencrypt_auto_apache2.sh | 24 ---------
 targets.yaml                             | 68 ++++++++++++++++--------
 5 files changed, 106 insertions(+), 52 deletions(-)
 create mode 100755 scripts/test_apache2.sh
 delete mode 100755 scripts/test_letsencrypt_auto_apache2.sh

diff --git a/README.md b/README.md
index 35950b18c..a5d365c8f 100644
--- a/README.md
+++ b/README.md
@@ -29,8 +29,9 @@ then:
 ```
 
 see:
-  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
-  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+- https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+- https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
 
-https://github.com/letsencrypt/boulder
-https://github.com/letsencrypt/letsencrypt
\ No newline at end of file
+main repos:
+- https://github.com/letsencrypt/boulder
+- https://github.com/letsencrypt/letsencrypt
diff --git a/multitester.py b/multitester.py
index 7c7bd3e2b..37e6a479c 100644
--- a/multitester.py
+++ b/multitester.py
@@ -271,12 +271,13 @@ def config_and_launch_boulder(instance):
     execute(deploy_script, 'scripts/boulder_config.sh')
     execute(run_boulder)
 
-def install_and_launch_letsencrypt(instance, boulder_url):
+def install_and_launch_letsencrypt(instance, boulder_url, target):
     execute(local_repo_to_remote)
     with shell_env(BOULDER_URL=boulder_url,
                    PUBLIC_IP=instance.public_ip_address,
                    PRIVATE_IP=instance.private_ip_address,
-                   PUBLIC_HOSTNAME=instance.public_dns_name):
+                   PUBLIC_HOSTNAME=instance.public_dns_name,
+                   OS_TYPE=target['type']):
         execute(deploy_script, cl_args.test_script)
 
 def grab_letsencrypt_log():
@@ -423,7 +424,7 @@ def test_client_process(inqueue, outqueue):
         print(env.host_string)
 
         try:
-            install_and_launch_letsencrypt(instances[ii], boulder_url)
+            install_and_launch_letsencrypt(instances[ii], boulder_url, target)
             outqueue.put((ii, target, 'pass'))
             print("%s - %s SUCCESS"%(target['ami'], target['name']))
         except:
diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
new file mode 100755
index 000000000..fe8b094fa
--- /dev/null
+++ b/scripts/test_apache2.sh
@@ -0,0 +1,50 @@
+#!/bin/bash -x
+
+#install apache2 on apt systems
+# debian doesn't come with curl
+#sudo apt-get update
+#sudo apt-get -y --no-upgrade install apache2 #curl
+
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
+# fetch instance data from EC2 metadata service
+#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+if [ $OS_TYPE = "ubuntu" ]
+then
+    CONFFILE=/etc/apache2/sites-available/000-default.conf
+    sudo apt-get update
+    sudo apt-get -y --no-upgrade install apache2 #curl
+    # For apache 2.4, set up ServerName
+    sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE
+    sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE
+elif [ $OS_TYPE = "centos" ]
+then
+    CONFFILE=/etc/httpd/conf/httpd.conf
+    sudo yum -y install httpd
+    sudo service httpd start
+    sudo mkdir -p /var/www/$PUBLIC_HOSTNAME/public_html
+    sudo chmod -R 777 /var/www
+    sudo echo '<html><head><title>foo</title></head>\n<body>bar</body></html>' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html
+    sudo mkdir /etc/httpd/sites-available
+    sudo mkdir /etc/httpd/sites-enabled
+    sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf
+    sudo echo """
+<VirtualHost *:80>
+    ServerName $PUBLIC_HOSTNAME
+    DocumentRoot /var/www/$PUBLIC_HOSTNAME/public_html
+    ErrorLog /var/www/$PUBLIC_HOSTNAME/error.log
+    CustomLog /var/www/$PUBLIC_HOSTNAME/requests.log combined
+</VirtualHost>""" >> /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf
+    sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/
+fi
+
+# run letsencrypt-apache2 via letsencrypt-auto
+cd letsencrypt
+./bootstrap/install-deps.sh
+./bootstrap/dev/venv.sh
+source ./venv/bin/activate
+sudo ./venv/bin/letsencrypt -v --debug --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect --register-unsafely-without-email \
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
diff --git a/scripts/test_letsencrypt_auto_apache2.sh b/scripts/test_letsencrypt_auto_apache2.sh
deleted file mode 100755
index 087a2eb13..000000000
--- a/scripts/test_letsencrypt_auto_apache2.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash -x
-
-#install apache2 on apt systems
-# debian doesn't come with curl
-sudo apt-get update
-sudo apt-get -y --no-upgrade install apache2 #curl
-
-# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
-# fetch instance data from EC2 metadata service
-#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
-
-# For apache 2.4, set up ServerName
-sudo sed -i '/ServerName/ s/#ServerName/ServerName/' \
-     /etc/apache2/sites-available/000-default.conf
-sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' \
-    /etc/apache2/sites-available/000-default.conf
-
-# run letsencrypt-apache2 via letsencrypt-auto
-cd letsencrypt
-./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
-                   --renew-by-default --redirect --register-unsafely-without-email \
-                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
diff --git a/targets.yaml b/targets.yaml
index 4547366b3..384f82df6 100644
--- a/targets.yaml
+++ b/targets.yaml
@@ -30,7 +30,7 @@ targets:
   # Debian
   - ami: ami-116d857a
     name: debian8.1
-    type: debian
+    type: ubuntu
     virt: hvm
     user: admin
     userdata: |
@@ -39,7 +39,7 @@ targets:
         - [ apt-get, install, -y, curl ]
   - ami: ami-e0efab88
     name: debian7.8.aws.1
-    type: debian
+    type: ubuntu
     virt: hvm
     user: admin
     userdata: |
@@ -48,7 +48,7 @@ targets:
         - [ apt-get, install, -y, curl ]
   - ami: ami-e6eeaa8e
     name: debian7.8.aws.1_32bit
-    type: debian
+    type: ubuntu
     virt: pv
     user: admin
     userdata: |
@@ -62,38 +62,64 @@ targets:
     type: centos
     virt: hvm
     user: ec2-user
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum -y install httpd
+        - service httpd start
   - ami: ami-0d4cfd66
     name: amazonlinux-2015.03.1
     type: centos
     virt: hvm
     user: ec2-user
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum -y install httpd
+        - service httpd start
   - ami: ami-a8d369c0
     name: RHEL7
-    type: redhat
+    type: centos
     virt: hvm
     user: ec2-user
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum -y install httpd
+        - service httpd start
   - ami: ami-518bfb3b
     name: fedora23
-    type: fedora
+    type: centos
     virt: hvm
     user: fedora
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum -y install httpd
+        - service httpd start
   #-----------------------------------------------------------------------------
   # CentOS
   # These Marketplace AMIs must, irritatingly, have their terms manually
   # agreed to on the AWS marketplace site for any new AWS account using them...
-  # - ami: ami-61bbf104
-  #   name: centos7
-  #   type: centos
-  #   virt: hvm
-  #   user: centos
-  # # centos6 requires EPEL repo added
-  # - ami: ami-57cd8732
-  #   name: centos6
-  #   type: centos
-  #   virt: hvm
-  #   user: centos
-  #   userdata: |
-  #     #cloud-config
-  #     runcmd:
-  #       - [ yum, install, -y, epel-release ]
-  #       - [ iptables, -F ]
+  - ami: ami-61bbf104
+    name: centos7
+    type: centos
+    virt: hvm
+    user: centos
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum -y install httpd
+        - service httpd start
+  # centos6 requires EPEL repo added
+  - ami: ami-57cd8732
+    name: centos6
+    type: centos
+    virt: hvm
+    user: centos
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum install -y epel-release httpd
+        - service httpd start
+        - iptables -F

From 822e2025ac6d29d1dd9527e7a6b6f65a3146923b Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Thu, 10 Dec 2015 06:19:32 -0800
Subject: [PATCH 08/19] fixed apache2 script, tested on centos7

---
 multitester.py          |  4 ++--
 scripts/test_apache2.sh | 28 +++++++++-------------
 targets.yaml            | 52 +++++++++++------------------------------
 3 files changed, 26 insertions(+), 58 deletions(-)

diff --git a/multitester.py b/multitester.py
index 37e6a479c..aaba1ed07 100644
--- a/multitester.py
+++ b/multitester.py
@@ -123,8 +123,8 @@ def make_instance(instance_name,
         UserData=userdata,
         InstanceType=machine_type)[0]
 
-    # brief pause to prevent rare EC2 error
-    time.sleep(0.5)
+    # brief pause to prevent rare error on EC2 delay, should block until ready instead
+    time.sleep(1.0)
 
     # give instance a name
     new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
index fe8b094fa..772300589 100755
--- a/scripts/test_apache2.sh
+++ b/scripts/test_apache2.sh
@@ -1,15 +1,7 @@
 #!/bin/bash -x
 
-#install apache2 on apt systems
-# debian doesn't come with curl
-#sudo apt-get update
-#sudo apt-get -y --no-upgrade install apache2 #curl
-
-# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
-# fetch instance data from EC2 metadata service
-#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
-#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
-#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
+# are dynamically set at execution
 
 if [ $OS_TYPE = "ubuntu" ]
 then
@@ -22,22 +14,24 @@ then
 elif [ $OS_TYPE = "centos" ]
 then
     CONFFILE=/etc/httpd/conf/httpd.conf
+    sudo setenforce 0 || true #disable selinux
     sudo yum -y install httpd
     sudo service httpd start
     sudo mkdir -p /var/www/$PUBLIC_HOSTNAME/public_html
-    sudo chmod -R 777 /var/www
-    sudo echo '<html><head><title>foo</title></head>\n<body>bar</body></html>' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html
-    sudo mkdir /etc/httpd/sites-available
-    sudo mkdir /etc/httpd/sites-enabled
-    sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf
+    sudo chmod -R oug+rwx /var/www
+    sudo chmod -R oug+rw /etc/httpd
+    sudo echo '<html><head><title>foo</title></head><body>bar</body></html>' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html
+    sudo mkdir /etc/httpd/sites-available #letsencrypt requires this...
+    sudo mkdir /etc/httpd/sites-enabled #letsencrypt requires this...
+    #sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf
     sudo echo """
 <VirtualHost *:80>
     ServerName $PUBLIC_HOSTNAME
     DocumentRoot /var/www/$PUBLIC_HOSTNAME/public_html
     ErrorLog /var/www/$PUBLIC_HOSTNAME/error.log
     CustomLog /var/www/$PUBLIC_HOSTNAME/requests.log combined
-</VirtualHost>""" >> /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf
-    sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/
+</VirtualHost>""" >> /etc/httpd/conf.d/$PUBLIC_HOSTNAME.conf
+    #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/
 fi
 
 # run letsencrypt-apache2 via letsencrypt-auto
diff --git a/targets.yaml b/targets.yaml
index 384f82df6..506225f86 100644
--- a/targets.yaml
+++ b/targets.yaml
@@ -33,28 +33,28 @@ targets:
     type: ubuntu
     virt: hvm
     user: admin
-    userdata: |
-      #cloud-init
-      runcmd:
-        - [ apt-get, install, -y, curl ]
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
   - ami: ami-e0efab88
     name: debian7.8.aws.1
     type: ubuntu
     virt: hvm
     user: admin
-    userdata: |
-      #cloud-init
-      runcmd:
-        - [ apt-get, install, -y, curl ]
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
   - ami: ami-e6eeaa8e
     name: debian7.8.aws.1_32bit
     type: ubuntu
     virt: pv
     user: admin
-    userdata: |
-      cloud-init
-      runcmd:
-        - [ apt-get, install, -y, curl ]
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
   #-----------------------------------------------------------------------------
   # Other Redhat Distros
   - ami: ami-60b6c60a
@@ -62,41 +62,21 @@ targets:
     type: centos
     virt: hvm
     user: ec2-user
-    userdata: |
-      #cloud-config
-      runcmd:
-        - yum -y install httpd
-        - service httpd start
   - ami: ami-0d4cfd66
     name: amazonlinux-2015.03.1
     type: centos
     virt: hvm
     user: ec2-user
-    userdata: |
-      #cloud-config
-      runcmd:
-        - yum -y install httpd
-        - service httpd start
   - ami: ami-a8d369c0
     name: RHEL7
     type: centos
     virt: hvm
     user: ec2-user
-    userdata: |
-      #cloud-config
-      runcmd:
-        - yum -y install httpd
-        - service httpd start
   - ami: ami-518bfb3b
     name: fedora23
     type: centos
     virt: hvm
     user: fedora
-    userdata: |
-      #cloud-config
-      runcmd:
-        - yum -y install httpd
-        - service httpd start
   #-----------------------------------------------------------------------------
   # CentOS
   # These Marketplace AMIs must, irritatingly, have their terms manually
@@ -106,11 +86,6 @@ targets:
     type: centos
     virt: hvm
     user: centos
-    userdata: |
-      #cloud-config
-      runcmd:
-        - yum -y install httpd
-        - service httpd start
   # centos6 requires EPEL repo added
   - ami: ami-57cd8732
     name: centos6
@@ -120,6 +95,5 @@ targets:
     userdata: |
       #cloud-config
       runcmd:
-        - yum install -y epel-release httpd
-        - service httpd start
+        - yum install -y epel-release
         - iptables -F

From ba400771192321c45abe55bb7451a0d366192b1a Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Thu, 10 Dec 2015 06:40:20 -0800
Subject: [PATCH 09/19] readme update

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index a5d365c8f..0d8506a3f 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,9 @@ then:
 >python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
 ```
 
+example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed
+to them at runtime via environment variables.  test_apache2.sh is a useful reference.
+
 see:
 - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
 - https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html

From a944603f430f0857f9423d95f712e8e68437488d Mon Sep 17 00:00:00 2001
From: Anselm Levskaya <levskaya@gmail.com>
Date: Thu, 10 Dec 2015 06:44:07 -0800
Subject: [PATCH 10/19] readme fix

---
 README.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 0d8506a3f..a085e9d91 100644
--- a/README.md
+++ b/README.md
@@ -25,12 +25,16 @@ simple aws testfarm scripts for letsencrypt client testing
 ```
 then:
 ```
->python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_apache2.sh
 ```
 
+## Scripts
 example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed
 to them at runtime via environment variables.  test_apache2.sh is a useful reference.
 
+Note that the <pre>test_letsencrypt_auto_*</pre> scripts pull code from PyPI using the letsencrypt-auto script,
+__not__ the local python code.  test_apache2 runs the dev venv and does local tests.
+
 see:
 - https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
 - https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html

From 3d20950fb89143e40b3fe1b22bf9a44840354725 Mon Sep 17 00:00:00 2001
From: Noah Swartz <noah@eff.org>
Date: Tue, 15 Dec 2015 10:59:13 -0800
Subject: [PATCH 11/19] modified to allow pip variable

---
 multitester.py          | 4 ++++
 scripts/test_apache2.sh | 5 +----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/multitester.py b/multitester.py
index aaba1ed07..ab8a9b29e 100644
--- a/multitester.py
+++ b/multitester.py
@@ -74,6 +74,9 @@ parser.add_argument('--merge_master',
 parser.add_argument('--saveinstances',
                     action='store_true',
                     help="don't kill EC2 instances after run, useful for debugging")
+parser.add_argument('--alt_pip',
+                    default='https://certainly.isnot.org',
+                    help="server from which to pull candidate release packages")
 cl_args = parser.parse_args()
 
 # Credential Variables
@@ -277,6 +280,7 @@ def install_and_launch_letsencrypt(instance, boulder_url, target):
                    PUBLIC_IP=instance.public_ip_address,
                    PRIVATE_IP=instance.private_ip_address,
                    PUBLIC_HOSTNAME=instance.public_dns_name,
+                   PIP_EXTRA_INDEX_URL=cl_args.alt_pip,
                    OS_TYPE=target['type']):
         execute(deploy_script, cl_args.test_script)
 
diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
index 772300589..803385e43 100755
--- a/scripts/test_apache2.sh
+++ b/scripts/test_apache2.sh
@@ -36,9 +36,6 @@ fi
 
 # run letsencrypt-apache2 via letsencrypt-auto
 cd letsencrypt
-./bootstrap/install-deps.sh
-./bootstrap/dev/venv.sh
-source ./venv/bin/activate
-sudo ./venv/bin/letsencrypt -v --debug --text --agree-dev-preview --agree-tos \
+letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL

From 22ca6a070a76e10f53fd411c68d285bc63fc45e7 Mon Sep 17 00:00:00 2001
From: Noah Swartz <noah@eff.org>
Date: Tue, 15 Dec 2015 11:52:17 -0800
Subject: [PATCH 12/19] added directory notation

---
 scripts/test_apache2.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
index 803385e43..65f514d65 100755
--- a/scripts/test_apache2.sh
+++ b/scripts/test_apache2.sh
@@ -36,6 +36,6 @@ fi
 
 # run letsencrypt-apache2 via letsencrypt-auto
 cd letsencrypt
-letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
+./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL

From 535782c6d5c010d90fc1738fa513471d9e310210 Mon Sep 17 00:00:00 2001
From: Noah Swartz <noah@eff.org>
Date: Tue, 15 Dec 2015 12:15:17 -0800
Subject: [PATCH 13/19] fix default url

---
 multitester.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/multitester.py b/multitester.py
index ab8a9b29e..fb29ab9eb 100644
--- a/multitester.py
+++ b/multitester.py
@@ -75,7 +75,7 @@ parser.add_argument('--saveinstances',
                     action='store_true',
                     help="don't kill EC2 instances after run, useful for debugging")
 parser.add_argument('--alt_pip',
-                    default='https://certainly.isnot.org',
+                    default='https://certainly.isnot.org/pip/',
                     help="server from which to pull candidate release packages")
 cl_args = parser.parse_args()
 

From cad254926e98bc27c607f989161de7019396e38a Mon Sep 17 00:00:00 2001
From: Peter Eckersley <pde@eff.org>
Date: Tue, 15 Dec 2015 12:47:01 -0800
Subject: [PATCH 14/19] A letsencrypt-auto upgrade test!

- should be run with --branch v0.1.0
---
 scripts/test_leauto_upgrades.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100755 scripts/test_leauto_upgrades.sh

diff --git a/scripts/test_leauto_upgrades.sh b/scripts/test_leauto_upgrades.sh
new file mode 100755
index 000000000..70f8a2293
--- /dev/null
+++ b/scripts/test_leauto_upgrades.sh
@@ -0,0 +1,18 @@
+#!/bin/bash -xe
+
+# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
+# are dynamically set at execution
+
+cd letsencrypt
+#git checkout v0.1.0     use --branch instead
+SAVE="$PIP_EXTRA_INDEX_URL"
+unset PIP_EXTRA_INDEX_URL
+./letsencrypt-auto -v --debug --version 
+
+export PIP_EXTRA_INDEX_URL="$SAVE"
+
+if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then
+    echo upgrade appeared to fail
+    exit 1
+fi
+echo upgrade appeared to be successful

From 03572a8d8e25163af85100ec7c08d99af43b32a3 Mon Sep 17 00:00:00 2001
From: Peter Eckersley <pde@eff.org>
Date: Wed, 16 Dec 2015 20:49:55 -0800
Subject: [PATCH 15/19] Add hackish-apache-tests to test_apache2.sh

---
 scripts/test_apache2.sh | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
index 65f514d65..db29dedd3 100755
--- a/scripts/test_apache2.sh
+++ b/scripts/test_apache2.sh
@@ -3,7 +3,7 @@
 # $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
 # are dynamically set at execution
 
-if [ $OS_TYPE = "ubuntu" ]
+if [ "$OS_TYPE" = "ubuntu" ]
 then
     CONFFILE=/etc/apache2/sites-available/000-default.conf
     sudo apt-get update
@@ -11,7 +11,7 @@ then
     # For apache 2.4, set up ServerName
     sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE
     sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE
-elif [ $OS_TYPE = "centos" ]
+elif [ "$OS_TYPE" = "centos" ]
 then
     CONFFILE=/etc/httpd/conf/httpd.conf
     sudo setenforce 0 || true #disable selinux
@@ -39,3 +39,25 @@ cd letsencrypt
 ./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
                    --renew-by-default --redirect --register-unsafely-without-email \
                    --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
+if [ $? -ne 0 ] ; then
+    FAIL=1
+fi
+
+if [ "$OS_TYPE" = "ubuntu" ] ; then
+    export LETSENCRYPT="$HOME/.local/share/letsencrypt/bin/letsencrypt"
+    for mod in `grep -v '^#' tests/apache-conf-files/passing/README.modules` ; do
+        sudo a2enmod $mod
+    done
+    tests/apache-conf-files/hackish-apache-test
+else
+    echo Not running hackish apache tests on $OS_TYPE
+fi
+
+if [ $? -ne 0 ] ; then
+    FAIL=1
+fi
+
+# return error if any of the subtests failed
+if [ "$FAIL" = 1 ] ; then
+    return 1
+fi

From 04aefcffac0ce885dcc79d5a79098018ac108ed6 Mon Sep 17 00:00:00 2001
From: Peter Eckersley <pde@eff.org>
Date: Wed, 16 Dec 2015 20:53:40 -0800
Subject: [PATCH 16/19] install modules needed for tests

---
 scripts/test_apache2.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
index db29dedd3..20dc10a71 100755
--- a/scripts/test_apache2.sh
+++ b/scripts/test_apache2.sh
@@ -45,6 +45,7 @@ fi
 
 if [ "$OS_TYPE" = "ubuntu" ] ; then
     export LETSENCRYPT="$HOME/.local/share/letsencrypt/bin/letsencrypt"
+    sudo apt-get install -y libapache2-mod-wsgi
     for mod in `grep -v '^#' tests/apache-conf-files/passing/README.modules` ; do
         sudo a2enmod $mod
     done

From e885536f9d1fd9dd8374ebb8a1038a33b69793b0 Mon Sep 17 00:00:00 2001
From: Peter Eckersley <pde@eff.org>
Date: Fri, 18 Dec 2015 10:13:15 -0800
Subject: [PATCH 17/19] Move module installation inside the test script

---
 scripts/test_apache2.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/test_apache2.sh b/scripts/test_apache2.sh
index 20dc10a71..a048a1ad0 100755
--- a/scripts/test_apache2.sh
+++ b/scripts/test_apache2.sh
@@ -45,11 +45,7 @@ fi
 
 if [ "$OS_TYPE" = "ubuntu" ] ; then
     export LETSENCRYPT="$HOME/.local/share/letsencrypt/bin/letsencrypt"
-    sudo apt-get install -y libapache2-mod-wsgi
-    for mod in `grep -v '^#' tests/apache-conf-files/passing/README.modules` ; do
-        sudo a2enmod $mod
-    done
-    tests/apache-conf-files/hackish-apache-test
+    tests/apache-conf-files/hackish-apache-test --debian-modules
 else
     echo Not running hackish apache tests on $OS_TYPE
 fi

From 9f02f264c5d2c2762cd470019845442d32bfbfc2 Mon Sep 17 00:00:00 2001
From: Peter Eckersley <pde@eff.org>
Date: Fri, 18 Dec 2015 11:07:39 -0800
Subject: [PATCH 18/19] test_tox : run unit tests through tox

---
 scripts/test_tox.sh | 80 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100755 scripts/test_tox.sh

diff --git a/scripts/test_tox.sh b/scripts/test_tox.sh
new file mode 100755
index 000000000..f7f325d5c
--- /dev/null
+++ b/scripts/test_tox.sh
@@ -0,0 +1,80 @@
+#!/bin/bash -x
+XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share}
+VENV_NAME="venv"
+# The path to the letsencrypt-auto script.  Everything that uses these might
+# at some point be inlined...
+LEA_PATH=./letsencrypt/
+VENV_PATH=${LEA_PATH/$VENV_NAME}
+VENV_BIN=${VENV_PATH}/bin
+BOOTSTRAP=${LEA_PATH}/bootstrap
+
+SUDO=sudo
+
+ExperimentalBootstrap() {
+  # Arguments: Platform name, boostrap script name, SUDO command (iff needed)
+    if [ "$2" != "" ]  ; then
+      echo "Bootstrapping dependencies for $1..."
+      if [ "$3" != "" ] ; then
+        "$3" "$BOOTSTRAP/$2"
+      else
+        "$BOOTSTRAP/$2"
+      fi
+    fi
+}
+
+# virtualenv call is not idempotent: it overwrites pip upgraded in
+# later steps, causing "ImportError: cannot import name unpack_url"
+if [ ! -f $BOOTSTRAP/debian.sh ] ; then
+  echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP"
+  exit 1
+fi
+
+if [ -f /etc/debian_version ] ; then
+  echo "Bootstrapping dependencies for Debian-based OSes..."
+  $SUDO $BOOTSTRAP/_deb_common.sh
+elif [ -f /etc/redhat-release ] ; then
+  echo "Bootstrapping dependencies for RedHat-based OSes..."
+  $SUDO $BOOTSTRAP/_rpm_common.sh
+elif `grep -q openSUSE /etc/os-release` ; then
+  echo "Bootstrapping dependencies for openSUSE-based OSes..."
+  $SUDO $BOOTSTRAP/_suse_common.sh
+elif [ -f /etc/arch-release ] ; then
+  if [ "$DEBUG" = 1 ] ; then
+    echo "Bootstrapping dependencies for Archlinux..."
+    $SUDO $BOOTSTRAP/archlinux.sh
+  else
+    echo "Please use pacman to install letsencrypt packages:"
+    echo "# pacman -S letsencrypt letsencrypt-apache"
+    echo
+    echo "If you would like to use the virtualenv way, please run the script again with the"
+    echo "--debug flag."
+    exit 1
+  fi
+elif [ -f /etc/manjaro-release ] ; then
+  ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO"
+elif [ -f /etc/gentoo-release ] ; then
+  ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO"
+elif uname | grep -iq FreeBSD ; then
+  ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO"
+elif uname | grep -iq Darwin ; then
+  ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root
+elif grep -iq "Amazon Linux" /etc/issue ; then
+  ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO"
+else
+  echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!"
+  echo
+  echo "You will need to bootstrap, configure virtualenv, and run a pip install manually"
+  echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites"
+  echo "for more info"
+fi
+echo "Bootstrapped!"
+
+cd letsencrypt
+./bootstrap/dev/venv.sh
+PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+
+if [ $PYVER -eq 26 ] ; then
+    venv/bin/tox -e py26
+else
+    venv/bin/tox -e py27
+fi

From 87dfe8c2b23a9205d77a453c2ff459fd3cc73ddc Mon Sep 17 00:00:00 2001
From: Peter Eckersley <pde@eff.org>
Date: Mon, 21 Dec 2015 11:12:01 -0800
Subject: [PATCH 19/19] Move everything into tests/letstest

---
 README.md => tests/letstest/README.md                             | 0
 apache2_targets.yaml => tests/letstest/apache2_targets.yaml       | 0
 multitester.py => tests/letstest/multitester.py                   | 0
 {scripts => tests/letstest/scripts}/boulder_config.sh             | 0
 {scripts => tests/letstest/scripts}/boulder_install.sh            | 0
 {scripts => tests/letstest/scripts}/test_apache2.sh               | 0
 {scripts => tests/letstest/scripts}/test_leauto_upgrades.sh       | 0
 .../scripts}/test_letsencrypt_auto_certonly_standalone.sh         | 0
 .../letstest/scripts}/test_letsencrypt_auto_venv_only.sh          | 0
 {scripts => tests/letstest/scripts}/test_tox.sh                   | 0
 targets.yaml => tests/letstest/targets.yaml                       | 0
 11 files changed, 0 insertions(+), 0 deletions(-)
 rename README.md => tests/letstest/README.md (100%)
 rename apache2_targets.yaml => tests/letstest/apache2_targets.yaml (100%)
 rename multitester.py => tests/letstest/multitester.py (100%)
 rename {scripts => tests/letstest/scripts}/boulder_config.sh (100%)
 rename {scripts => tests/letstest/scripts}/boulder_install.sh (100%)
 rename {scripts => tests/letstest/scripts}/test_apache2.sh (100%)
 rename {scripts => tests/letstest/scripts}/test_leauto_upgrades.sh (100%)
 rename {scripts => tests/letstest/scripts}/test_letsencrypt_auto_certonly_standalone.sh (100%)
 rename {scripts => tests/letstest/scripts}/test_letsencrypt_auto_venv_only.sh (100%)
 rename {scripts => tests/letstest/scripts}/test_tox.sh (100%)
 rename targets.yaml => tests/letstest/targets.yaml (100%)

diff --git a/README.md b/tests/letstest/README.md
similarity index 100%
rename from README.md
rename to tests/letstest/README.md
diff --git a/apache2_targets.yaml b/tests/letstest/apache2_targets.yaml
similarity index 100%
rename from apache2_targets.yaml
rename to tests/letstest/apache2_targets.yaml
diff --git a/multitester.py b/tests/letstest/multitester.py
similarity index 100%
rename from multitester.py
rename to tests/letstest/multitester.py
diff --git a/scripts/boulder_config.sh b/tests/letstest/scripts/boulder_config.sh
similarity index 100%
rename from scripts/boulder_config.sh
rename to tests/letstest/scripts/boulder_config.sh
diff --git a/scripts/boulder_install.sh b/tests/letstest/scripts/boulder_install.sh
similarity index 100%
rename from scripts/boulder_install.sh
rename to tests/letstest/scripts/boulder_install.sh
diff --git a/scripts/test_apache2.sh b/tests/letstest/scripts/test_apache2.sh
similarity index 100%
rename from scripts/test_apache2.sh
rename to tests/letstest/scripts/test_apache2.sh
diff --git a/scripts/test_leauto_upgrades.sh b/tests/letstest/scripts/test_leauto_upgrades.sh
similarity index 100%
rename from scripts/test_leauto_upgrades.sh
rename to tests/letstest/scripts/test_leauto_upgrades.sh
diff --git a/scripts/test_letsencrypt_auto_certonly_standalone.sh b/tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
similarity index 100%
rename from scripts/test_letsencrypt_auto_certonly_standalone.sh
rename to tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh
diff --git a/scripts/test_letsencrypt_auto_venv_only.sh b/tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh
similarity index 100%
rename from scripts/test_letsencrypt_auto_venv_only.sh
rename to tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh
diff --git a/scripts/test_tox.sh b/tests/letstest/scripts/test_tox.sh
similarity index 100%
rename from scripts/test_tox.sh
rename to tests/letstest/scripts/test_tox.sh
diff --git a/targets.yaml b/tests/letstest/targets.yaml
similarity index 100%
rename from targets.yaml
rename to tests/letstest/targets.yaml