Merge remote-tracking branch 'letstest/git-move-to-subdir' into testfarm

tests/letstest/README.md

@@ -0,0 +1,44 @@
+# letstest
+simple aws testfarm scripts for letsencrypt client testing
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+## Notes
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+## Usage
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+```
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair --query 'KeyMaterial' --output text > MyKeyPair.pem
+```
+then:
+```
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_apache2.sh
+```
+
+## Scripts
+example scripts are in the 'scripts' directory, these are just bash scripts that have a few parameters passed
+to them at runtime via environment variables.  test_apache2.sh is a useful reference.
+
+Note that the <pre>test_letsencrypt_auto_*</pre> scripts pull code from PyPI using the letsencrypt-auto script,
+__not__ the local python code.  test_apache2 runs the dev venv and does local tests.
+
+see:
+- https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+- https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+
+main repos:
+- https://github.com/letsencrypt/boulder
+- https://github.com/letsencrypt/letsencrypt

tests/letstest/apache2_targets.yaml

@@ -0,0 +1,57 @@
+targets:
+  #-----------------------------------------------------------------------------
+  # Apache 2.4
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-116d857a
+    name: debian8.1
+    type: debian
+    virt: hvm
+    user: admin
+    userdata: |
+      #cloud-init
+      runcmd:
+        - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Apache 2.2
+  # - ami: ami-0611546c
+  #   name: ubuntu12.04LTS
+  #   type: ubuntu
+  #   virt: hvm
+  #   user: ubuntu
+  # - ami: ami-e0efab88
+  #   name: debian7.8.aws.1
+  #   type: debian
+  #   virt: hvm
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]
+  # - ami: ami-e6eeaa8e
+  #   name: debian7.8.aws.1_32bit
+  #   type: debian
+  #   virt: pv
+  #   user: admin
+  #   userdata: |
+  #     #cloud-init
+  #     runcmd:
+  #       - [ apt-get, install, -y, curl ]

tests/letstest/multitester.py

@@ -0,0 +1,492 @@
+"""
+Letsencrypt Integration Test Tool
+
+- Configures (canned) boulder server
+- Launches EC2 instances with a given list of AMIs for different distros
+- Copies letsencrypt repo and puts it on the instances
+- Runs letsencrypt tests (bash scripts) on all of these
+- Logs execution and success/fail for debugging
+
+Notes:
+  - Some AWS images, e.g. official CentOS and FreeBSD images
+    require acceptance of user terms on the AWS marketplace
+    website.  This can't be automated.
+  - AWS EC2 has a default limit of 20 t2/t1 instances, if more
+    are needed, they need to be requested via online webform.
+
+Usage:
+  - Requires AWS IAM secrets to be set up with aws cli
+  - Requires an AWS associated keyfile <keyname>.pem
+
+>aws configure --profile HappyHacker
+[interactive: enter secrets for IAM role]
+>aws ec2 create-key-pair --profile HappyHacker --key-name MyKeyPair \
+ --query 'KeyMaterial' --output text > MyKeyPair.pem
+then:
+>python multitester.py targets.yaml MyKeyPair.pem HappyHacker scripts/test_letsencrypt_auto_venv_only.sh
+see:
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
+  https://docs.aws.amazon.com/cli/latest/userguide/cli-ec2-keypairs.html
+"""
+
+from __future__ import print_function
+from __future__ import with_statement
+
+import sys, os, time, argparse, socket
+import multiprocessing as mp
+from multiprocessing import Manager
+import urllib2
+import yaml
+import boto3
+import fabric
+from fabric.api import run, execute, local, env, sudo, cd, lcd
+from fabric.operations import get, put
+from fabric.context_managers import shell_env
+
+# Command line parser
+#-------------------------------------------------------------------------------
+parser = argparse.ArgumentParser(description='Builds EC2 cluster for testing.')
+parser.add_argument('config_file',
+                    help='yaml configuration file for AWS server cluster')
+parser.add_argument('key_file',
+                    help='key file (<keyname>.pem) for AWS')
+parser.add_argument('aws_profile',
+                    help='profile for AWS (i.e. as in ~/.aws/certificates)')
+parser.add_argument('test_script',
+                    default='test_letsencrypt_auto_certonly_standalone.sh',
+                    help='path of bash script in to deploy and run')
+#parser.add_argument('--script_args',
+#                    nargs='+',
+#                    help='space-delimited list of arguments to pass to the bash test script',
+#                    required=False)
+parser.add_argument('--repo',
+                    default='https://github.com/letsencrypt/letsencrypt.git',
+                    help='letsencrypt git repo to use')
+parser.add_argument('--branch',
+                    default='~',
+                    help='letsencrypt git branch to trial')
+parser.add_argument('--pull_request',
+                    default='~',
+                    help='letsencrypt/letsencrypt pull request to trial')
+parser.add_argument('--merge_master',
+                    action='store_true',
+                    help="if set merges PR into master branch of letsencrypt/letsencrypt")
+parser.add_argument('--saveinstances',
+                    action='store_true',
+                    help="don't kill EC2 instances after run, useful for debugging")
+parser.add_argument('--alt_pip',
+                    default='https://certainly.isnot.org/pip/',
+                    help="server from which to pull candidate release packages")
+cl_args = parser.parse_args()
+
+# Credential Variables
+#-------------------------------------------------------------------------------
+# assumes naming: <key_filename> = <keyname>.pem
+KEYFILE = cl_args.key_file
+KEYNAME = os.path.split(cl_args.key_file)[1].split('.pem')[0]
+PROFILE = cl_args.aws_profile
+
+# Globals
+#-------------------------------------------------------------------------------
+BOULDER_AMI = 'ami-5f490b35' # premade shared boulder AMI 14.04LTS us-east-1
+LOGDIR = "" #points to logging / working directory
+# boto3/AWS api globals
+AWS_SESSION = None
+EC2 = None
+
+# Boto3/AWS automation functions
+#-------------------------------------------------------------------------------
+def make_security_group():
+    # will fail if security group of GroupName already exists
+    # cannot have duplicate SGs of the same name
+    mysg = EC2.create_security_group(GroupName="letsencrypt_test",
+                                     Description='security group for automated testing')
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=22, ToPort=22)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=80, ToPort=80)
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=443, ToPort=443)
+    # for boulder wfe (http) server
+    mysg.authorize_ingress(IpProtocol="tcp", CidrIp="0.0.0.0/0", FromPort=4000, ToPort=4000)
+    # for mosh
+    mysg.authorize_ingress(IpProtocol="udp", CidrIp="0.0.0.0/0", FromPort=60000, ToPort=61000)
+    return mysg
+
+def make_instance(instance_name,
+                  ami_id,
+                  keyname,
+                  machine_type='t2.micro',
+                  security_groups=['letsencrypt_test'],
+                  userdata=""): #userdata contains bash or cloud-init script
+
+    new_instance = EC2.create_instances(
+        ImageId=ami_id,
+        SecurityGroups=security_groups,
+        KeyName=keyname,
+        MinCount=1,
+        MaxCount=1,
+        UserData=userdata,
+        InstanceType=machine_type)[0]
+
+    # brief pause to prevent rare error on EC2 delay, should block until ready instead
+    time.sleep(1.0)
+
+    # give instance a name
+    new_instance.create_tags(Tags=[{'Key': 'Name', 'Value': instance_name}])
+    return new_instance
+
+def terminate_and_clean(instances):
+    """
+    Some AMIs specify EBS stores that won't delete on instance termination.
+    These must be manually deleted after shutdown.
+    """
+    volumes_to_delete = []
+    for instance in instances:
+        for bdmap in instance.block_device_mappings:
+            if 'Ebs' in bdmap.keys():
+                if not bdmap['Ebs']['DeleteOnTermination']:
+                    volumes_to_delete.append(bdmap['Ebs']['VolumeId'])
+
+    for instance in instances:
+        instance.terminate()
+
+    # can't delete volumes until all attaching instances are terminated
+    _ids = [instance.id for instance in instances]
+    all_terminated = False
+    while not all_terminated:
+        all_terminated = True
+        for _id in _ids:
+            # necessary to reinit object for boto3 to get true state
+            inst = EC2.Instance(id=_id)
+            if inst.state['Name'] != 'terminated':
+                all_terminated = False
+        time.sleep(5)
+
+    for vol_id in volumes_to_delete:
+        volume = EC2.Volume(id=vol_id)
+        volume.delete()
+
+    return volumes_to_delete
+
+
+# Helper Routines
+#-------------------------------------------------------------------------------
+def block_until_http_ready(urlstring, wait_time=10, timeout=240):
+    "Blocks until server at urlstring can respond to http requests"
+    server_ready = False
+    t_elapsed = 0
+    while not server_ready and t_elapsed < timeout:
+        try:
+            sys.stdout.write('.')
+            sys.stdout.flush()
+            req = urllib2.Request(urlstring)
+            response = urllib2.urlopen(req)
+            #if response.code == 200:
+            server_ready = True
+        except urllib2.URLError:
+            pass
+        time.sleep(wait_time)
+        t_elapsed += wait_time
+
+def block_until_ssh_open(ipstring, wait_time=10, timeout=120):
+    "Blocks until server at ipstring has an open port 22"
+    reached = False
+    t_elapsed = 0
+    while not reached and t_elapsed < timeout:
+        try:
+            sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            sock.connect((ipstring, 22))
+            reached = True
+        except socket.error as err:
+            time.sleep(wait_time)
+            t_elapsed += wait_time
+    sock.close()
+
+def block_until_instance_ready(booting_instance, wait_time=5, extra_wait_time=20):
+    "Blocks booting_instance until AWS EC2 instance is ready to accept SSH connections"
+    # the reinstantiation from id is necessary to force boto3
+    # to correctly update the 'state' variable during init
+    _id = booting_instance.id
+    _instance = EC2.Instance(id=_id)
+    _state = _instance.state['Name']
+    _ip = _instance.public_ip_address
+    while _state != 'running' or _ip is None:
+        time.sleep(wait_time)
+        _instance = EC2.Instance(id=_id)
+        _state = _instance.state['Name']
+        _ip = _instance.public_ip_address
+    block_until_ssh_open(_ip)
+    time.sleep(extra_wait_time)
+    return _instance
+
+
+# Fabric Routines
+#-------------------------------------------------------------------------------
+def local_git_clone(repo_url):
+    "clones master of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_branch(repo_url, branch_name):
+    "clones branch <branch_name> of repo_url"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s --branch %s --single-branch'%(repo_url, branch_name))
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_git_PR(repo_url, PRnumstr, merge_master=True):
+    "clones specified pull request from repo_url and optionally merges into master"
+    with lcd(LOGDIR):
+        local('if [ -d letsencrypt ]; then rm -rf letsencrypt; fi')
+        local('git clone %s'% repo_url)
+        local('cd letsencrypt && git fetch origin pull/%s/head:lePRtest'%PRnumstr)
+        local('cd letsencrypt && git co lePRtest')
+        if merge_master:
+            local('cd letsencrypt && git remote update origin')
+            local('cd letsencrypt && git merge origin/master -m "testmerge"')
+        local('tar czf le.tar.gz letsencrypt')
+
+def local_repo_to_remote():
+    "copies local tarball of repo to remote"
+    with lcd(LOGDIR):
+        put(local_path='le.tar.gz', remote_path='')
+        run('tar xzf le.tar.gz')
+
+def local_repo_clean():
+    "delete tarball"
+    with lcd(LOGDIR):
+        local('rm le.tar.gz')
+
+def deploy_script(scriptpath, *args):
+    "copies to remote and executes local script"
+    #with lcd('scripts'):
+    put(local_path=scriptpath, remote_path='', mirror_local_mode=True)
+    scriptfile = os.path.split(scriptpath)[1]
+    args_str = ' '.join(args)
+    run('./'+scriptfile+' '+args_str)
+
+def run_boulder():
+    with cd('$GOPATH/src/github.com/letsencrypt/boulder'):
+        run('go run cmd/rabbitmq-setup/main.go -server amqp://localhost')
+        run('nohup ./start.py >& /dev/null < /dev/null &')
+
+def config_and_launch_boulder(instance):
+    execute(deploy_script, 'scripts/boulder_config.sh')
+    execute(run_boulder)
+
+def install_and_launch_letsencrypt(instance, boulder_url, target):
+    execute(local_repo_to_remote)
+    with shell_env(BOULDER_URL=boulder_url,
+                   PUBLIC_IP=instance.public_ip_address,
+                   PRIVATE_IP=instance.private_ip_address,
+                   PUBLIC_HOSTNAME=instance.public_dns_name,
+                   PIP_EXTRA_INDEX_URL=cl_args.alt_pip,
+                   OS_TYPE=target['type']):
+        execute(deploy_script, cl_args.test_script)
+
+def grab_letsencrypt_log():
+    "grabs letsencrypt.log via cat into logged stdout"
+    sudo('if [ -f /var/log/letsencrypt/letsencrypt.log ]; then \
+    cat /var/log/letsencrypt/letsencrypt.log; else echo "[novarlog]"; fi')
+    # fallback file if /var/log is unwriteable...? correct?
+    sudo('if [ -f ./letsencrypt.log ]; then \
+    cat ./letsencrypt.log; else echo "[nolocallog]"; fi')
+
+#-------------------------------------------------------------------------------
+# SCRIPT BEGINS
+#-------------------------------------------------------------------------------
+
+# Fabric library controlled through global env parameters
+env.key_filename = KEYFILE
+env.shell = '/bin/bash -l -i -c'
+env.connection_attempts = 5
+env.timeout = 10
+# replace default SystemExit thrown by fabric during trouble
+class FabricException(Exception):
+    pass
+env['abort_exception'] = FabricException
+
+# Set up local copy of git repo
+#-------------------------------------------------------------------------------
+LOGDIR = "letest-%d"%int(time.time())
+print("Making local dir for test repo and logs: %s"%LOGDIR)
+local('mkdir %s'%LOGDIR)
+
+# figure out what git object to test and locally create it in LOGDIR
+print("Making local git repo")
+try:
+    if cl_args.pull_request != '~':
+        print('Testing PR %s '%cl_args.pull_request,
+              "MERGING into master" if cl_args.merge_master else "")
+        execute(local_git_PR, cl_args.repo, cl_args.pull_request, cl_args.merge_master)
+    elif cl_args.branch != '~':
+        print('Testing branch %s of %s'%(cl_args.branch, cl_args.repo))
+        execute(local_git_branch, cl_args.repo, cl_args.branch)
+    else:
+        print('Testing master of %s'%cl_args.repo)
+        execute(local_git_clone, cl_args.repo)
+except FabricException:
+    print("FAIL: trouble with git repo")
+    exit()
+
+
+# Set up EC2 instances
+#-------------------------------------------------------------------------------
+configdata = yaml.load(open(cl_args.config_file, 'r'))
+targetlist = configdata['targets']
+print('Testing against these images: [%d total]'%len(targetlist))
+for target in targetlist:
+    print(target['ami'], target['name'])
+
+print("Connecting to EC2 using\n profile %s\n keyname %s\n keyfile %s"%(PROFILE, KEYNAME, KEYFILE))
+AWS_SESSION = boto3.session.Session(profile_name=PROFILE)
+EC2 = AWS_SESSION.resource('ec2')
+
+print("Making Security Group")
+sg_exists = False
+for sg in EC2.security_groups.all():
+    if sg.group_name == 'letsencrypt_test':
+        sg_exists = True
+        print("  %s already exists"%'letsencrypt_test')
+if not sg_exists:
+    make_security_group()
+    time.sleep(30)
+
+print("Requesting Instances...")
+boulder_server = make_instance('le-boulderserver',
+                               BOULDER_AMI,
+                               KEYNAME,
+                               #machine_type='t2.micro',
+                               machine_type='t2.medium',
+                               security_groups=['letsencrypt_test'])
+
+instances = []
+for target in targetlist:
+    if target['virt'] == 'hvm':
+        machine_type = 't2.micro'
+    else:
+        machine_type = 't1.micro'
+    if 'userdata' in target.keys():
+        userdata = target['userdata']
+    else:
+        userdata = ''
+    instances.append(make_instance('le-%s'%target['name'],
+                                   target['ami'],
+                                   KEYNAME,
+                                   machine_type=machine_type,
+                                   userdata=userdata))
+
+
+# Configure and launch boulder server
+#-------------------------------------------------------------------------------
+print("Waiting on Boulder Server")
+boulder_server = block_until_instance_ready(boulder_server)
+print(" server %s"%boulder_server)
+
+print("Configuring and Launching Boulder")
+
+# env.host_string defines the ssh user and host for connection
+env.host_string = "ubuntu@%s"%boulder_server.public_ip_address
+print("Boulder Server at (SSH):", env.host_string)
+config_and_launch_boulder(boulder_server)
+# blocking often unnecessary, but cheap EC2 VMs can get very slow
+block_until_http_ready('http://%s:4000'%boulder_server.public_ip_address,
+                       wait_time=10,
+                       timeout=500)
+
+boulder_url = "http://%s:4000/directory"%boulder_server.private_ip_address
+print("Boulder Server at (public ip): http://%s:4000/directory"%boulder_server.public_ip_address)
+print("Boulder Server at (EC2 private ip): %s"%boulder_url)
+
+# Install and launch client scripts in parallel
+#-------------------------------------------------------------------------------
+print("Uploading and running test script in parallel: %s"%cl_args.test_script)
+print("Output routed to log files in %s"%LOGDIR)
+# (Advice: always use Manager.Queue, never regular multiprocessing.Queue
+# the latter has implementation flaws that deadlock it in some circumstances)
+manager = Manager()
+outqueue = manager.Queue()
+inqueue = manager.Queue()
+SENTINEL = None #queue kill signal
+
+# launch as many processes as clients to test
+num_processes = len(targetlist)
+jobs = [] #keep a reference to current procs
+
+def test_client_process(inqueue, outqueue):
+    cur_proc = mp.current_process()
+    for inreq in iter(inqueue.get, SENTINEL):
+        ii, target = inreq
+
+        #save all stdout to log file
+        sys.stdout = open(LOGDIR+'/'+'%d_%s.log'%(ii,target['name']), 'w')
+
+        print("[%s : client %d %s %s]" % (cur_proc.name, ii, target['ami'], target['name']))
+        instances[ii] = block_until_instance_ready(instances[ii])
+        print("server %s at %s"%(instances[ii], instances[ii].public_ip_address))
+        env.host_string = "%s@%s"%(target['user'], instances[ii].public_ip_address)
+        print(env.host_string)
+
+        try:
+            install_and_launch_letsencrypt(instances[ii], boulder_url, target)
+            outqueue.put((ii, target, 'pass'))
+            print("%s - %s SUCCESS"%(target['ami'], target['name']))
+        except:
+            outqueue.put((ii, target, 'fail'))
+            print("%s - %s FAIL"%(target['ami'], target['name']))
+            pass
+
+        # append server letsencrypt.log to each per-machine output log
+        print("\n\nletsencrypt.log\n" + "-"*80 + "\n")
+        try:
+            execute(grab_letsencrypt_log)
+        except:
+            print("log fail\n")
+            pass
+
+# initiate process execution
+for i in range(num_processes):
+    p = mp.Process(target=test_client_process, args=(inqueue, outqueue))
+    jobs.append(p)
+    p.daemon = True  # kills subprocesses if parent is killed
+    p.start()
+
+# fill up work queue
+for ii, target in enumerate(targetlist):
+    inqueue.put((ii, target))
+
+# add SENTINELs to end client processes
+for i in range(num_processes):
+    inqueue.put(SENTINEL)
+# wait on termination of client processes
+for p in jobs:
+    p.join()
+# add SENTINEL to output queue
+outqueue.put(SENTINEL)
+
+# clean up
+execute(local_repo_clean)
+
+# print and save summary results
+results_file = open(LOGDIR+'/results', 'w')
+outputs = [outq for outq in iter(outqueue.get, SENTINEL)]
+outputs.sort(key=lambda x: x[0])
+for outq in outputs:
+    ii, target, status = outq
+    print('%d %s %s'%(ii, target['name'], status))
+    results_file.write('%d %s %s\n'%(ii, target['name'], status))
+results_file.close()
+
+if not cl_args.saveinstances:
+    print('Terminating EC2 Instances and Cleaning Dangling EBS Volumes')
+    boulder_server.terminate()
+    terminate_and_clean(instances)
+else:
+    # print login information for the boxes for debugging
+    for ii, target in enumerate(targetlist):
+        print(target['name'],
+              target['ami'],
+              "%s@%s"%(target['user'], instances[ii].public_ip_address))
+
+# kill any connections
+fabric.network.disconnect_all()

tests/letstest/scripts/boulder_config.sh

@@ -0,0 +1,32 @@
+#!/bin/bash -x
+
+# Configures and Launches Boulder Server installed on
+# us-east-1 ami-5f490b35 bouldertestserver (boulder commit 8b433f54dab)
+
+# fetch instance data from EC2 metadata service
+public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+# get local DNS resolver for VPC
+resolver_ip=$(grep nameserver /etc/resolv.conf |cut -d" " -f2 |head -1)
+resolver=$resolver_ip':53'
+
+# modifies integration testing boulder setup for local AWS VPC network
+# connections instead of localhost
+cd $GOPATH/src/github.com/letsencrypt/boulder
+# configure boulder to receive outside connection on 4000
+sed -i '/listenAddress/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+sed -i '/baseURL/ s/127.0.0.1:4000/'$private_ip':4000/' ./test/boulder-config.json
+# change test ports to real
+sed -i '/httpPort/ s/5002/80/' ./test/boulder-config.json
+sed -i '/httpsPort/ s/5001/443/' ./test/boulder-config.json
+sed -i '/tlsPort/ s/5001/443/' ./test/boulder-config.json
+# set local dns resolver
+sed -i '/dnsResolver/ s/127.0.0.1:8053/'$resolver'/' ./test/boulder-config.json
+
+# start rabbitMQ
+#go run cmd/rabbitmq-setup/main.go -server amqp://localhost
+# start acme services
+#nohup ./start.py >& /dev/null < /dev/null &
+#./start.py

tests/letstest/scripts/boulder_install.sh

@@ -0,0 +1,28 @@
+#!/bin/bash -x
+
+# >>>> only tested on Ubuntu 14.04LTS <<<<
+
+# non-interactive install of mariadb and other dependencies
+export DEBIAN_FRONTEND=noninteractive
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password password PASS'
+sudo debconf-set-selections <<< 'mariadb-server mysql-server/root_password_again password PASS'
+apt-get -y --no-upgrade install git make libltdl3-dev mariadb-server rabbitmq-server
+sudo mysql -uroot -pPASS -e "SET PASSWORD = PASSWORD(\'\');"
+
+# install go
+wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
+tar xzvf go1.5.1.linux-amd64.tar.gz
+mkdir gocode
+echo "export GOROOT=/home/ubuntu/go \n\
+      export GOPATH=/home/ubuntu/gocode\n\
+      export PATH=/home/ubuntu/go/bin:/home/ubuntu/gocode/bin:$PATH" >> .bashrc
+
+# install boulder and its go dependencies
+go get -d github.com/letsencrypt/boulder/...
+cd $GOPATH/src/github.com/letsencrypt/boulder
+wget https://github.com/jsha/boulder-tools/raw/master/goose.gz
+mkdir $GOPATH/bin
+zcat goose.gz > $GOPATH/bin/goose
+chmod +x $GOPATH/bin/goose
+./test/create_db.sh
+go get github.com/jsha/listenbuddy

tests/letstest/scripts/test_apache2.sh

@@ -0,0 +1,60 @@
+#!/bin/bash -x
+
+# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
+# are dynamically set at execution
+
+if [ "$OS_TYPE" = "ubuntu" ]
+then
+    CONFFILE=/etc/apache2/sites-available/000-default.conf
+    sudo apt-get update
+    sudo apt-get -y --no-upgrade install apache2 #curl
+    # For apache 2.4, set up ServerName
+    sudo sed -i '/ServerName/ s/#ServerName/ServerName/' $CONFFILE
+    sudo sed -i '/ServerName/ s/www.example.com/'$PUBLIC_HOSTNAME'/' $CONFFILE
+elif [ "$OS_TYPE" = "centos" ]
+then
+    CONFFILE=/etc/httpd/conf/httpd.conf
+    sudo setenforce 0 || true #disable selinux
+    sudo yum -y install httpd
+    sudo service httpd start
+    sudo mkdir -p /var/www/$PUBLIC_HOSTNAME/public_html
+    sudo chmod -R oug+rwx /var/www
+    sudo chmod -R oug+rw /etc/httpd
+    sudo echo '<html><head><title>foo</title></head><body>bar</body></html>' > /var/www/$PUBLIC_HOSTNAME/public_html/index.html
+    sudo mkdir /etc/httpd/sites-available #letsencrypt requires this...
+    sudo mkdir /etc/httpd/sites-enabled #letsencrypt requires this...
+    #sudo echo "IncludeOptional sites-enabled/*.conf" >> /etc/httpd/conf/httpd.conf
+    sudo echo """
+<VirtualHost *:80>
+    ServerName $PUBLIC_HOSTNAME
+    DocumentRoot /var/www/$PUBLIC_HOSTNAME/public_html
+    ErrorLog /var/www/$PUBLIC_HOSTNAME/error.log
+    CustomLog /var/www/$PUBLIC_HOSTNAME/requests.log combined
+</VirtualHost>""" >> /etc/httpd/conf.d/$PUBLIC_HOSTNAME.conf
+    #sudo cp /etc/httpd/sites-available/$PUBLIC_HOSTNAME.conf /etc/httpd/sites-enabled/
+fi
+
+# run letsencrypt-apache2 via letsencrypt-auto
+cd letsencrypt
+./letsencrypt-auto -v --debug --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect --register-unsafely-without-email \
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL
+if [ $? -ne 0 ] ; then
+    FAIL=1
+fi
+
+if [ "$OS_TYPE" = "ubuntu" ] ; then
+    export LETSENCRYPT="$HOME/.local/share/letsencrypt/bin/letsencrypt"
+    tests/apache-conf-files/hackish-apache-test --debian-modules
+else
+    echo Not running hackish apache tests on $OS_TYPE
+fi
+
+if [ $? -ne 0 ] ; then
+    FAIL=1
+fi
+
+# return error if any of the subtests failed
+if [ "$FAIL" = 1 ] ; then
+    return 1
+fi

tests/letstest/scripts/test_leauto_upgrades.sh

@@ -0,0 +1,18 @@
+#!/bin/bash -xe
+
+# $OS_TYPE $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL
+# are dynamically set at execution
+
+cd letsencrypt
+#git checkout v0.1.0     use --branch instead
+SAVE="$PIP_EXTRA_INDEX_URL"
+unset PIP_EXTRA_INDEX_URL
+./letsencrypt-auto -v --debug --version 
+
+export PIP_EXTRA_INDEX_URL="$SAVE"
+
+if ! ./letsencrypt-auto -v --debug --version | grep 0.1.1 ; then
+    echo upgrade appeared to fail
+    exit 1
+fi
+echo upgrade appeared to be successful

tests/letstest/scripts/test_letsencrypt_auto_certonly_standalone.sh

@@ -0,0 +1,15 @@
+#!/bin/bash -x
+
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
+
+# with curl, instance metadata available from EC2 metadata service:
+#public_host=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-hostname)
+#public_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/public-ipv4)
+#private_ip=$(curl -s http://169.254.169.254/2014-11-05/meta-data/local-ipv4)
+
+cd letsencrypt
+./letsencrypt-auto certonly -v --standalone --debug \
+                   --text --agree-dev-preview --agree-tos \
+                   --renew-by-default --redirect \
+                   --register-unsafely-without-email \
+                   --domain $PUBLIC_HOSTNAME --server $BOULDER_URL

tests/letstest/scripts/test_letsencrypt_auto_venv_only.sh

@@ -0,0 +1,7 @@
+#!/bin/bash -x
+
+# $PUBLIC_IP $PRIVATE_IP $PUBLIC_HOSTNAME $BOULDER_URL are dynamically set at execution
+
+cd letsencrypt
+# help installs virtualenv and does nothing else
+./letsencrypt-auto -v --help all

tests/letstest/scripts/test_tox.sh

@@ -0,0 +1,80 @@
+#!/bin/bash -x
+XDG_DATA_HOME=${XDG_DATA_HOME:-~/.local/share}
+VENV_NAME="venv"
+# The path to the letsencrypt-auto script.  Everything that uses these might
+# at some point be inlined...
+LEA_PATH=./letsencrypt/
+VENV_PATH=${LEA_PATH/$VENV_NAME}
+VENV_BIN=${VENV_PATH}/bin
+BOOTSTRAP=${LEA_PATH}/bootstrap
+
+SUDO=sudo
+
+ExperimentalBootstrap() {
+  # Arguments: Platform name, boostrap script name, SUDO command (iff needed)
+    if [ "$2" != "" ]  ; then
+      echo "Bootstrapping dependencies for $1..."
+      if [ "$3" != "" ] ; then
+        "$3" "$BOOTSTRAP/$2"
+      else
+        "$BOOTSTRAP/$2"
+      fi
+    fi
+}
+
+# virtualenv call is not idempotent: it overwrites pip upgraded in
+# later steps, causing "ImportError: cannot import name unpack_url"
+if [ ! -f $BOOTSTRAP/debian.sh ] ; then
+  echo "Cannot find the letsencrypt bootstrap scripts in $BOOTSTRAP"
+  exit 1
+fi
+
+if [ -f /etc/debian_version ] ; then
+  echo "Bootstrapping dependencies for Debian-based OSes..."
+  $SUDO $BOOTSTRAP/_deb_common.sh
+elif [ -f /etc/redhat-release ] ; then
+  echo "Bootstrapping dependencies for RedHat-based OSes..."
+  $SUDO $BOOTSTRAP/_rpm_common.sh
+elif `grep -q openSUSE /etc/os-release` ; then
+  echo "Bootstrapping dependencies for openSUSE-based OSes..."
+  $SUDO $BOOTSTRAP/_suse_common.sh
+elif [ -f /etc/arch-release ] ; then
+  if [ "$DEBUG" = 1 ] ; then
+    echo "Bootstrapping dependencies for Archlinux..."
+    $SUDO $BOOTSTRAP/archlinux.sh
+  else
+    echo "Please use pacman to install letsencrypt packages:"
+    echo "# pacman -S letsencrypt letsencrypt-apache"
+    echo
+    echo "If you would like to use the virtualenv way, please run the script again with the"
+    echo "--debug flag."
+    exit 1
+  fi
+elif [ -f /etc/manjaro-release ] ; then
+  ExperimentalBootstrap "Manjaro Linux" manjaro.sh "$SUDO"
+elif [ -f /etc/gentoo-release ] ; then
+  ExperimentalBootstrap "Gentoo" _gentoo_common.sh "$SUDO"
+elif uname | grep -iq FreeBSD ; then
+  ExperimentalBootstrap "FreeBSD" freebsd.sh "$SUDO"
+elif uname | grep -iq Darwin ; then
+  ExperimentalBootstrap "Mac OS X" mac.sh # homebrew doesn't normally run as root
+elif grep -iq "Amazon Linux" /etc/issue ; then
+  ExperimentalBootstrap "Amazon Linux" _rpm_common.sh "$SUDO"
+else
+  echo "Sorry, I don't know how to bootstrap Let's Encrypt on your operating system!"
+  echo
+  echo "You will need to bootstrap, configure virtualenv, and run a pip install manually"
+  echo "Please see https://letsencrypt.readthedocs.org/en/latest/contributing.html#prerequisites"
+  echo "for more info"
+fi
+echo "Bootstrapped!"
+
+cd letsencrypt
+./bootstrap/dev/venv.sh
+PYVER=`python --version 2>&1 | cut -d" " -f 2 | cut -d. -f1,2 | sed 's/\.//'`
+
+if [ $PYVER -eq 26 ] ; then
+    venv/bin/tox -e py26
+else
+    venv/bin/tox -e py27
+fi

tests/letstest/targets.yaml

@@ -0,0 +1,99 @@
+targets:
+  #-----------------------------------------------------------------------------
+  #Ubuntu
+  - ami: ami-26d5af4c
+    name: ubuntu15.10
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-d92e6bb3
+    name: ubuntu15.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-7b89cc11
+    name: ubuntu14.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  - ami: ami-9295d0f8
+    name: ubuntu14.04LTS_32bit
+    type: ubuntu
+    virt: pv
+    user: ubuntu
+  - ami: ami-0611546c
+    name: ubuntu12.04LTS
+    type: ubuntu
+    virt: hvm
+    user: ubuntu
+  #-----------------------------------------------------------------------------
+  # Debian
+  - ami: ami-116d857a
+    name: debian8.1
+    type: ubuntu
+    virt: hvm
+    user: admin
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
+  - ami: ami-e0efab88
+    name: debian7.8.aws.1
+    type: ubuntu
+    virt: hvm
+    user: admin
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
+  - ami: ami-e6eeaa8e
+    name: debian7.8.aws.1_32bit
+    type: ubuntu
+    virt: pv
+    user: admin
+    # userdata: |
+    #   #cloud-init
+    #   runcmd:
+    #     - [ apt-get, install, -y, curl ]
+  #-----------------------------------------------------------------------------
+  # Other Redhat Distros
+  - ami: ami-60b6c60a
+    name: amazonlinux-2015.09.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-0d4cfd66
+    name: amazonlinux-2015.03.1
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-a8d369c0
+    name: RHEL7
+    type: centos
+    virt: hvm
+    user: ec2-user
+  - ami: ami-518bfb3b
+    name: fedora23
+    type: centos
+    virt: hvm
+    user: fedora
+  #-----------------------------------------------------------------------------
+  # CentOS
+  # These Marketplace AMIs must, irritatingly, have their terms manually
+  # agreed to on the AWS marketplace site for any new AWS account using them...
+  - ami: ami-61bbf104
+    name: centos7
+    type: centos
+    virt: hvm
+    user: centos
+  # centos6 requires EPEL repo added
+  - ami: ami-57cd8732
+    name: centos6
+    type: centos
+    virt: hvm
+    user: centos
+    userdata: |
+      #cloud-config
+      runcmd:
+        - yum install -y epel-release
+        - iptables -F