From a19333d84befdb526fc262afddb3020cc2b0be3f Mon Sep 17 00:00:00 2001 From: James Kasten Date: Wed, 3 Dec 2014 04:35:49 -0800 Subject: [PATCH 01/22] First draft of apache configurator unit tests --- letsencrypt/client/tests/__init__.py | 1 + .../client/tests/apache_configurator_test.py | 152 ++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 letsencrypt/client/tests/__init__.py create mode 100644 letsencrypt/client/tests/apache_configurator_test.py diff --git a/letsencrypt/client/tests/__init__.py b/letsencrypt/client/tests/__init__.py new file mode 100644 index 000000000..d9db68022 --- /dev/null +++ b/letsencrypt/client/tests/__init__.py @@ -0,0 +1 @@ +"""Let's Encrypt Tests""" diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py new file mode 100644 index 000000000..780b13336 --- /dev/null +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -0,0 +1,152 @@ +"""test_letsencrypt - Integration Test + +A series of basic full integration tests to ensure that Letsencrypt is +still running smoothly. + +.. note:: This code is not complete nor has it been tested +.. note:: Do not document this code... it will change quickly + +""" + +import os +import shutil +import tarfile +import unittest +import sys + +from letsencrypt.client import apache_configurator +from letsencrypt.client import CONFIG +from letsencrypt.client import display +from letsencrypt.client import le_util +from letsencrypt.client import logger + +# Some of these will likely go into a letsencrypt.tests.CONFIG file +TESTING_DIR = "/home/ubuntu/testing/" +UBUNTU_CONFIGS = os.path.join(TESTING_DIR, "ubuntu_apache_2_4/") +TEMP_DIR = os.path.join(TESTING_DIR, "temp") +# I have not put this up on my website yet... it will not work +# This might end up going into the repo... but this is more of +# a user run test as opposed to a Travis CI test. +CONFIG_TGZ_URL = "https://jdkasten.com/letsencrypt/config.tgz" + + +def setUpModule(): + logger.setLogger(logger.FileLogger(sys.stdout)) + logger.setLogLevel(logger.INFO) + display.set_display(display.NcursesDisplay()) + + if not os.path.isdir(UBUNTU_CONFIGS): + print "Please place the configuration directory: %s" % UBUNTU_CONFIGS + sys.exit(1) + shutil.copytree(UBUNTU_CONFIGS, TEMP_DIR) + +def tearDownModule(): + shutil.rmtree(TEMP_DIR) + +class TwoVhosts_80(unittest.TestCase): + def setUp(self): + config_path = os.path.join(UBUNTU_CONFIGS, "two_vhosts_*80/apache2") + sites_path = os.path.join(UBUNTU_CONFIGS, "two_vhosts_*80/sites") + + self.config = apache_configurator.ApacheConfigurator(config_path, 2.47) + + prefix = os.path.join(TEMP_DIR, "sites-available") + aug_pre = os.path.join("/files", prefix) + self.vh_truth = [] + self.vh_truth.append(apache_configurator.VH( + os.path.join(prefix + "encryption-example.conf"), + os.path.join(aug_pre, "encryption-example.conf/VirtualHost"), + ["*:80"], False, True)) + self.vh_truth.append(apache_configurator.VH( + os.path.join(prefix, "default-ssl.conf"), + os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"), + ["_default_:443"], True, False)) + self.vh_truth.append(apache_configurator.VH( + os.path.join(prefix, "000-default.conf"), + os.path.join(aug_pre, "000-default.conf/VirtualHost"), + ["*:80"], False, True)) + self.vh_truth.append(apache_configurator.VH( + os.path.join(prefix, "letsencrypt.conf"), + os.path.join(aug_pre, "letsencrypt.conf/VirtualHost"), + ["*:80"], False, True)) + self.vh_truth[0].add_name("encryption-example.demo") + self.vh_truth[2].add_name("ip-172-30-0-17") + self.vh_truth[3].add_name("letsencrypt.demo") + + def test_get_all_names(self): + names = self.config.get_all_names() + + print names + self.assertTrue(set(names) == set( + ["letsencrypt.demo", "encryption-example.com", "ip-172-30-0-17"])) + + def test_find_directive(self): + self.assertTrue( + len(self.config.find_directive( + apache_configurator.case_i("Listen"), "443") == 2)) + self.assertTrue( + len(self.config.find_directive( + apache_configurator.case_i("documentroot"))) == 4) + + def test_get_virtual_hosts(self): + vhs = self.config.get_virtual_hosts() + + self.assertTrue(len(vhs) == 4) + self.assertTrue(set(self.vh_truth) == set(vhs)) + + def test_is_site_enabled(self): + self.assertTrue(self.config.is_site_enabled(self.vh_truth[0].file)) + self.assertTrue(not self.config.is_site_enabled(self.vh_truth[1].file)) + self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].file)) + self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].file)) + + def test_deploy_cert(self): + self.config.deploy_cert( + self.vh_truth[1], + "example/cert.pem", "example/key.pem", "example/cert_chain.pem") + + loc_cert = self.config.find_directive( + apache_configurator.case_i("sslcertificatefile"), "example/cert.pem") + loc_key = self.config.find_directive( + apache_configurator.case_i("sslcertificateKeyfile"), "example/key.pem") + loc_chain = self.config.find_directive( + apache_configurator.case_i("SSLCertificateChainFile"), "example/chain.pem") + + self.assertTrue(len(loc_cert) == 1 and + apache_configurator.get_file_path( + loc_cert[0]) == self.vh_truth[1].file) + + self.assertTrue(len(loc_key) == 1 and + apache_configurator.get_file_path( + loc_key[0]) == self.vh_truth[1].file) + + self.assertTrue(len(loc_chain) == 1 and + apache_configurator.get_file_path( + loc_chain[0]) == self.vh_truth[1].file) + + def test_is_name_vhost(self): + self.assertTrue(not self.config.is_name_vhost("*:80")) + + def test_add_name_vhost(self): + self.config.add_name_vhost("*:443") + self.config.save(temporary=True) + + self.assertTrue(self.config.is_name_vhost("*:443")) + + + def _verify_redirect(self, config_path): + with open(config_path, 'r') as config_fd: + conf = config_fd.read() + + return CONFIG.REWRITE_HTTPS_ARGS[1] in conf + +# def download_unpack_tests(url=CONFIG_TGZ_URL): +# r = requests.get(url) +# local_tgz_file = os.path.join(TESTING_DIR, 'ubuntu_2_4.tgz') +# with open(local_tgz_file, 'w') as tgz_file: +# tgz_file.write(r.content) + +# if tarfile.is_tarfile(local_tgz_file): +# tar = tarfile.open(local_tgz_file) +# tar.extractall() +# tar.close() From 30641db85f90b907ce1584947cc897fbc392cb66 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 4 Dec 2014 04:00:22 -0800 Subject: [PATCH 02/22] Made apache_configurator work in any location/first basic tests complete successfully --- letsencrypt/client/CONFIG.py | 34 +++-- letsencrypt/client/apache_configurator.py | 55 +++++-- .../client/tests/apache_configurator_test.py | 134 +++++++++++------- 3 files changed, 148 insertions(+), 75 deletions(-) diff --git a/letsencrypt/client/CONFIG.py b/letsencrypt/client/CONFIG.py index 1c2fd496c..3cc9d09a6 100644 --- a/letsencrypt/client/CONFIG.py +++ b/letsencrypt/client/CONFIG.py @@ -10,6 +10,7 @@ If you create your own server... change this line Note: the server certificate must be trusted in order to avoid further modifications to the client.""" +# Directories SERVER_ROOT = "/etc/apache2/" """Apache server root directory""" @@ -39,6 +40,7 @@ KEY_DIR = os.path.join(SERVER_ROOT, "ssl/") CERT_DIR = os.path.join(SERVER_ROOT, "certs/") """Certificate storage""" +# Files and extensions OPTIONS_SSL_CONF = os.path.join(CONFIG_DIR, "options-ssl.conf") """Contains standard Apache SSL directives""" @@ -48,18 +50,6 @@ LE_VHOST_EXT = "-le-ssl.conf" APACHE_CHALLENGE_CONF = os.path.join(CONFIG_DIR, "le_dvsni_cert_challenge.conf") """Temporary file for challenge virtual hosts""" -S_SIZE = 32 -"""Byte size of S""" - -NONCE_SIZE = 16 -"""byte size of Nonce""" - -RSA_KEY_SIZE = 2048 -"""Key size""" - -DIFFICULTY = 23 -"""bits of hashcash to generate""" - CERT_PATH = CERT_DIR + "cert-letsencrypt.pem" """Let's Encrypt cert file.""" @@ -69,6 +59,7 @@ CHAIN_PATH = CERT_DIR + "chain-letsencrypt.pem" INVALID_EXT = ".acme.invalid" """Invalid Extension""" +# Challenge Information CHALLENGE_PREFERENCES = ["dvsni", "recoveryToken"] """Challenge Preferences Dict for currently supported challenges""" @@ -78,6 +69,25 @@ EXCLUSIVE_CHALLENGES = [frozenset(["dvsni", "simpleHttps"])] CONFIG_CHALLENGES = frozenset(["dvsni", "simpleHttps"]) """These are challenges that must be solved by a Configurator object""" +# Challenge Constants +S_SIZE = 32 +"""Byte size of S""" + +NONCE_SIZE = 16 +"""byte size of Nonce""" + +# Key Sizes +RSA_KEY_SIZE = 2048 +"""Key size""" + +# Config Optimizations REWRITE_HTTPS_ARGS = [ "^.*$", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,R=permanent]"] """Rewrite rule arguments used for redirections to https vhost""" + +# Apache Interaction +APACHE_CTL = "/usr/sbin/apache2ctl" +"""Command used for configtest and version number.""" + +APACHE2 = "/etc/init.d/apache2" +"""Command used for reload and restart.""" diff --git a/letsencrypt/client/apache_configurator.py b/letsencrypt/client/apache_configurator.py index f297e1cf9..995834bc6 100644 --- a/letsencrypt/client/apache_configurator.py +++ b/letsencrypt/client/apache_configurator.py @@ -92,9 +92,11 @@ class VH(object): if isinstance(other, self.__class__): return (self.file == other.file and self.path == other.path and set(self.addrs) == set(other.addrs) and - set(self.names) == set(other.naems) and + set(self.names) == set(other.names) and self.ssl == other.ssl and self.enabled == other.enabled) + return False + class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """Apache configurator. @@ -145,6 +147,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # vhosts self.recovery_routine() + # Find configuration root and make sure augeas can parse it. + self.config_root = self._find_config_root() + self._parse_file(self.config_root) + + # Must also attempt to parse sites-available or equivalent + # Sites-available is not included naturally in configuration + self._parse_file(os.path.join(self.server_root, "sites-available/*")) + # Set Version if not version: self.version = self.get_version() @@ -335,6 +345,16 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return all_names + def _find_config_root(self): + location = ["apache2.conf", "httpd.conf"] + + for name in location: + if os.path.isfile(os.path.join(self.server_root, name)): + return os.path.join(self.server_root, name) + + raise errors.LetsEncryptConfiguratorError( + "Could not find configuration root") + def _set_user_config_file(self, filename=''): """Set the appropriate user configuration file @@ -370,6 +390,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): case_i('ServerName'), host.path, case_i('ServerAlias')))) + for name in name_match: args = self.aug.match(name + "/*") for arg in args: @@ -413,6 +434,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): ("/files%ssites-available//*[label()=~regexp('%s')]" % (self.server_root, case_i('VirtualHost')))) vhs = [] + for path in paths: vhs.append(self._create_vhost(path)) @@ -601,7 +623,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Cannot place member variable in the definition of the function so... if not start: - start = "/files%sapache2.conf" % self.server_root + start = "/files%s" % self.config_root # Debug code # print "find_dir:", directive, "arg:", arg, " | Looking in:", start @@ -624,9 +646,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "[self::arg=~regexp('%s')]" % (start, directive, arg))) + incl_regex = "(%s)|(%s)" % (case_i('Include'), + case_i('IncludeOptional')) + includes = self.aug.match(("%s//* [self::directive=~regexp('%s')]/* " - "[label()='arg']" % - (start, case_i('Include')))) + "[label()='arg']" % (start, incl_regex))) + + # for inc in includes: + # print inc, self.aug.get(inc) for include in includes: # start[6:] to strip off /files @@ -1273,15 +1300,18 @@ LogLevel warn \n\ def get_version(self): # pylint: disable=no-self-use """Return version of Apache Server. - Version is returned as float. (ie. 2.4.7 = 2.47) + Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)) :returns: version - :rtype: float + :rtype: tuple + + :raises errors.LetsEncryptConfiguratorError: + Unable to find Apache version """ try: proc = subprocess.Popen( - ['sudo', '/usr/sbin/apache2ctl', '-v'], + ['/usr/sbin/apache2ctl', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = proc.communicate()[0] @@ -1296,10 +1326,7 @@ LogLevel warn \n\ raise errors.LetsEncryptConfiguratorError( "Unable to find Apache version") - num_decimal = matches[0].count(".") - - # Format return value such as 2.47 rather than 2.4.7 - return float("".join(matches[0].rsplit(".", num_decimal-1))) + num_decimal = tuple(matches[0].split('.')) ########################################################################### # Challenges Section @@ -1481,7 +1508,7 @@ def enable_mod(mod_name): stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) # Hopefully this waits for output - subprocess.check_call(["sudo", "/etc/init.d/apache2", "restart"], + subprocess.check_call(["sudo", APACHE2, "restart"], stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) except (OSError, subprocess.CalledProcessError) as err: @@ -1518,7 +1545,7 @@ def check_ssl_loaded(): return False -def apache_restart(quiet=False): +def apache_restart(): """Restarts the Apache Server. .. todo:: Try to use reload instead. (This caused timing problems before) @@ -1526,7 +1553,7 @@ def apache_restart(quiet=False): """ try: - proc = subprocess.Popen(['/etc/init.d/apache2', 'restart'], + proc = subprocess.Popen([APACHE2, 'restart'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = proc.communicate() diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py index 780b13336..6d3e70c6a 100644 --- a/letsencrypt/client/tests/apache_configurator_test.py +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -1,9 +1,9 @@ -"""test_letsencrypt - Integration Test +"""apache_configurator_test - unittests A series of basic full integration tests to ensure that Letsencrypt is still running smoothly. -.. note:: This code is not complete nor has it been tested +.. note:: This code is not complete .. note:: Do not document this code... it will change quickly """ @@ -24,10 +24,9 @@ from letsencrypt.client import logger TESTING_DIR = "/home/ubuntu/testing/" UBUNTU_CONFIGS = os.path.join(TESTING_DIR, "ubuntu_apache_2_4/") TEMP_DIR = os.path.join(TESTING_DIR, "temp") -# I have not put this up on my website yet... it will not work -# This might end up going into the repo... but this is more of -# a user run test as opposed to a Travis CI test. -CONFIG_TGZ_URL = "https://jdkasten.com/letsencrypt/config.tgz" + +# This will end up going into the repo... +CONFIG_TGZ_URL = "https://jdkasten.com/projects/config.tgz" def setUpModule(): @@ -38,20 +37,23 @@ def setUpModule(): if not os.path.isdir(UBUNTU_CONFIGS): print "Please place the configuration directory: %s" % UBUNTU_CONFIGS sys.exit(1) - shutil.copytree(UBUNTU_CONFIGS, TEMP_DIR) + shutil.copytree(UBUNTU_CONFIGS, TEMP_DIR, symlinks=True) + def tearDownModule(): shutil.rmtree(TEMP_DIR) + class TwoVhosts_80(unittest.TestCase): + def setUp(self): - config_path = os.path.join(UBUNTU_CONFIGS, "two_vhosts_*80/apache2") - sites_path = os.path.join(UBUNTU_CONFIGS, "two_vhosts_*80/sites") + # Final slash is currently important + self.config_path = os.path.join(TEMP_DIR, "two_vhost_80/apache2/") + self.config = apache_configurator.ApacheConfigurator( + self.config_path, (2, 4, 7)) - self.config = apache_configurator.ApacheConfigurator(config_path, 2.47) - - prefix = os.path.join(TEMP_DIR, "sites-available") - aug_pre = os.path.join("/files", prefix) + prefix = os.path.join(TEMP_DIR, "two_vhost_80/apache2/sites-available/") + aug_pre = "/files" + prefix self.vh_truth = [] self.vh_truth.append(apache_configurator.VH( os.path.join(prefix + "encryption-example.conf"), @@ -73,66 +75,97 @@ class TwoVhosts_80(unittest.TestCase): self.vh_truth[2].add_name("ip-172-30-0-17") self.vh_truth[3].add_name("letsencrypt.demo") - def test_get_all_names(self): - names = self.config.get_all_names() + def test_parse_file(self): + """test parse_file. - print names + letsencrypt.conf is chosen as the test file as it will not be + included during the normal course of execution. + + """ + file_path = os.path.join( + self.config_path, "sites-available", "letsencrypt.conf") + self.config._parse_file(file_path) + + # search for the httpd incl + matches = self.config.aug.match( + "/augeas/load/Httpd/incl [. ='%s']" % file_path) + + self.assertTrue(matches) + + def test_get_all_names(self): + """test get_all_names.""" + names = self.config.get_all_names() self.assertTrue(set(names) == set( - ["letsencrypt.demo", "encryption-example.com", "ip-172-30-0-17"])) + ['letsencrypt.demo', 'encryption-example.demo', 'ip-172-30-0-17'])) def test_find_directive(self): - self.assertTrue( - len(self.config.find_directive( - apache_configurator.case_i("Listen"), "443") == 2)) - self.assertTrue( - len(self.config.find_directive( - apache_configurator.case_i("documentroot"))) == 4) + """test find_directive.""" + test = self.config.find_directive( + apache_configurator.case_i("Listen"), "443") + # This will only look in enabled hosts + test2 = self.config.find_directive( + apache_configurator.case_i("documentroot")) + self.assertTrue(len(test) == 2) + self.assertTrue(len(test2) == 3) def test_get_virtual_hosts(self): + """get_virtual_hosts.""" vhs = self.config.get_virtual_hosts() - self.assertTrue(len(vhs) == 4) - self.assertTrue(set(self.vh_truth) == set(vhs)) + failed = False + for vhost in vhs: + for truth in self.vh_truth: + if vhost == truth: + break + failed = True + + self.assertTrue(failed) def test_is_site_enabled(self): + """test is_site_enabled""" self.assertTrue(self.config.is_site_enabled(self.vh_truth[0].file)) self.assertTrue(not self.config.is_site_enabled(self.vh_truth[1].file)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].file)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].file)) - def test_deploy_cert(self): - self.config.deploy_cert( - self.vh_truth[1], - "example/cert.pem", "example/key.pem", "example/cert_chain.pem") + # def test_deploy_cert(self): + # """test deploy_cert.""" + # self.config.deploy_cert( + # self.vh_truth[1], + # "example/cert.pem", "example/key.pem", "example/cert_chain.pem") - loc_cert = self.config.find_directive( - apache_configurator.case_i("sslcertificatefile"), "example/cert.pem") - loc_key = self.config.find_directive( - apache_configurator.case_i("sslcertificateKeyfile"), "example/key.pem") - loc_chain = self.config.find_directive( - apache_configurator.case_i("SSLCertificateChainFile"), "example/chain.pem") + # loc_cert = self.config.find_directive( + # apache_configurator.case_i( + # "sslcertificatefile"), "example/cert.pem") + # loc_key = self.config.find_directive( + # apache_configurator.case_i( + # "sslcertificateKeyfile"), "example/key.pem") + # loc_chain = self.config.find_directive( + # apache_configurator.case_i( + # "SSLCertificateChainFile"), "example/chain.pem") - self.assertTrue(len(loc_cert) == 1 and - apache_configurator.get_file_path( - loc_cert[0]) == self.vh_truth[1].file) + # self.assertTrue(len(loc_cert) == 1 and + # apache_configurator.get_file_path( + # loc_cert[0]) == self.vh_truth[1].file) - self.assertTrue(len(loc_key) == 1 and - apache_configurator.get_file_path( - loc_key[0]) == self.vh_truth[1].file) + # self.assertTrue(len(loc_key) == 1 and + # apache_configurator.get_file_path( + # loc_key[0]) == self.vh_truth[1].file) - self.assertTrue(len(loc_chain) == 1 and - apache_configurator.get_file_path( - loc_chain[0]) == self.vh_truth[1].file) + # self.assertTrue(len(loc_chain) == 1 and + # apache_configurator.get_file_path( + # loc_chain[0]) == self.vh_truth[1].file) def test_is_name_vhost(self): + """test is_name_vhost.""" self.assertTrue(not self.config.is_name_vhost("*:80")) - def test_add_name_vhost(self): - self.config.add_name_vhost("*:443") - self.config.save(temporary=True) - - self.assertTrue(self.config.is_name_vhost("*:443")) + # def test_add_name_vhost(self): + # """test add_name_vhost.""" + # self.config.add_name_vhost("*:443") + # self.config.save(temporary=True) + # self.assertTrue(self.config.is_name_vhost("*:443")) def _verify_redirect(self, config_path): with open(config_path, 'r') as config_fd: @@ -140,6 +173,9 @@ class TwoVhosts_80(unittest.TestCase): return CONFIG.REWRITE_HTTPS_ARGS[1] in conf + +if __name__ == '__main__': + unittest.main() # def download_unpack_tests(url=CONFIG_TGZ_URL): # r = requests.get(url) # local_tgz_file = os.path.join(TESTING_DIR, 'ubuntu_2_4.tgz') From 8a8eb0d617e44c60b4de5ead20059ced7bcd07d9 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 4 Dec 2014 04:02:33 -0800 Subject: [PATCH 03/22] Stop testing py26 until we sort out pylint issues --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 54ed84263..d55cfdf77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ language: python python: - - "2.6" +# - "2.6" - "2.7" before_install: > From 48ae79e8c4729448c1f5e94bb3e04291625dad4f Mon Sep 17 00:00:00 2001 From: James Kasten Date: Thu, 4 Dec 2014 04:16:22 -0800 Subject: [PATCH 04/22] Correct checking of get_virtual_hosts test --- letsencrypt/client/tests/apache_configurator_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py index 6d3e70c6a..1ca3c878d 100644 --- a/letsencrypt/client/tests/apache_configurator_test.py +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -109,17 +109,17 @@ class TwoVhosts_80(unittest.TestCase): self.assertTrue(len(test2) == 3) def test_get_virtual_hosts(self): - """get_virtual_hosts.""" + """inefficient get_virtual_hosts check.""" vhs = self.config.get_virtual_hosts() self.assertTrue(len(vhs) == 4) - failed = False + found = 0 for vhost in vhs: for truth in self.vh_truth: if vhost == truth: + found += 1 break - failed = True - self.assertTrue(failed) + self.assertTrue(found == 4) def test_is_site_enabled(self): """test is_site_enabled""" From 779dd755cad4afe293e3b2861b723dcc68704daf Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 5 Dec 2014 17:31:36 -0800 Subject: [PATCH 05/22] More unit tests... better configurator --- letsencrypt/client/apache_configurator.py | 139 ++++++++++-------- .../client/tests/apache_configurator_test.py | 94 ++++++++---- 2 files changed, 145 insertions(+), 88 deletions(-) diff --git a/letsencrypt/client/apache_configurator.py b/letsencrypt/client/apache_configurator.py index 995834bc6..ef0d88651 100644 --- a/letsencrypt/client/apache_configurator.py +++ b/letsencrypt/client/apache_configurator.py @@ -116,13 +116,11 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): and the typical directories are parsed by the Augeas configuration parser automatically. + .. todo:: Add support for config file variables Define rootDir /var/www/ + The API of this class will change in the coming weeks as the exact needs of client's are clarified with the new and developing protocol. - This class will eventually derive from a generic Configurator class - so that other Configurators (like Nginx) can be developed and interoperate - with the client. - :ivar str server_root: Path to Apache root directory :ivar float version: version of Apache @@ -180,9 +178,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # TODO: attempt to make the check faster... this enable should # be asynchronous as it shouldn't be that time sensitive # on initialization - if not check_ssl_loaded(): - logger.info("Loading mod_ssl into Apache Server") - enable_mod("ssl") + self._prepare_server_https() # Note: initialization doesn't check to see if the config is correct # by Apache's standards. This should be done by the client (client.py) @@ -238,6 +234,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return False logger.info("Deploying Certificate to VirtualHost %s" % vhost.file) + print path self.aug.set(path["cert_file"][0], cert) self.aug.set(path["cert_key"][0], key) @@ -443,7 +440,9 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def is_name_vhost(self, target_addr): """Returns if vhost is a name based vhost - Checks if addr has a NameVirtualHost directive in the Apache config + NameVirtualHost was deprecated in Apache 2.4 as all VirtualHosts are + now NameVirtualHosts. If version is earlier than 2.4, check if addr + has a NameVirtualHost directive in the Apache config :param str addr: vhost address ie. \*:443 @@ -451,49 +450,67 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :rtype: bool """ - # search for NameVirtualHost directive for ip_addr - # check httpd.conf, ports.conf, - # note ip_addr can be FQDN although Apache does not recommend it - paths = self.find_directive(case_i("NameVirtualHost"), None) - name_vh = [] - for path in paths: - name_vh.append(self.aug.get(path)) - # Mixed and matched wildcard NameVirtualHost with VirtualHost # behavior is undefined. Make sure that an exact match exists - # Check for exact match - for addr in name_vh: - if addr == target_addr: - return True - + # search for NameVirtualHost directive for ip_addr + # check httpd.conf, ports.conf, + # note ip_addr can be FQDN although Apache does not recommend it + if (self.version >= (2, 4) or + self.find_directive( + case_i("NameVirtualHost"), case_i(target_addr))): + return True return False def add_name_vhost(self, addr): """Adds NameVirtualHost directive for given address. Directive is added to ports.conf unless the file doesn't exist - It is added to httpd.conf as a backup + It is added to user_config_file as a backup :param str addr: Address that will be added as NameVirtualHost directive """ - aug_file_path = "/files%sports.conf" % self.server_root - self.add_dir_to_ifmodssl(aug_file_path, "NameVirtualHost", addr) + aug_path = "/files%sports.conf" % self.server_root + path = self.add_gen_ssl_dir_config(aug_path, "NameVirtualHost", addr) - # TODO: Check to see if len(find_dir) can just be if find_dir() - if len(self.find_directive( - case_i("NameVirtualHost"), case_i(addr))) == 0: - logger.warn("ports.conf is not included in your Apache config...") - logger.warn("Adding NameVirtualHost directive to httpd.conf") + self.save_notes += "Setting %s to be NameBasedVirtualHost\n" % addr + self.save_notes += "\tDirective added to %s\n" % path - self.add_dir_to_ifmodssl("/files%shttpd.conf" % self.server_root, - "NameVirtualHost", - addr) + def add_gen_ssl_dir_config(self, aug_path, directive, val): + """Adds directive to ifmodssl somewhere in config. - self.save_notes += 'Setting %s to be NameBasedVirtualHost\n' % addr + First the function tries to place the ssl directive along the + aug_path, if that is not available it places the directive + in the self.user_config_file which is expected to be valid. - def add_dir_to_ifmodssl(self, aug_conf_path, directive, val): + :param str aug_path: Augeas configuration path + :param str directive: Directive you would like to add to config + :param str val: Value of the directive + + :returns: Path to file that directive was added + :rtype: str + + """ + # First try original path + self._add_dir_to_ifmodssl( + aug_path, re.escape(directive), re.escape(val)) + + if self.find_directive(case_i(directive), case_i(val), start=aug_path): + return get_file_path(aug_path) + + # Try default path + logger.debug( + "%s is not included in your Apache config..." % + os.path.basename(get_file_path(aug_path))) + logger.debug( + "Adding %s directive to %s" % (directive, self.user_config_file)) + + self._add_dir_to_ifmodssl( + "/files%s" % self.user_config_file, directive, val) + return self.user_config_file + + def _add_dir_to_ifmodssl(self, aug_conf_path, directive, val): """Adds directive and value to IfMod ssl block. Adds given directive and value along configuration path within @@ -507,13 +524,34 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # TODO: Add error checking code... does the path given even exist? # Does it throw exceptions? - if_mod_path = self.get_ifmod(aug_conf_path, "mod_ssl.c") + if_mod_path = self._get_ifmod(aug_conf_path, "mod_ssl.c") # IfModule can have only one valid argument, so append after self.aug.insert(if_mod_path + "arg", "directive", False) nvh_path = if_mod_path + "directive[1]" self.aug.set(nvh_path, directive) self.aug.set(nvh_path + "/arg", val) + def _prepare_server_https(self): + """Prepare the server for HTTPS. + + Make sure that the ssl_module is loaded and that the server + is appropriately listening on port 443. + + """ + if not check_ssl_loaded(): + logger.info("Loading mod_ssl into Apache Server") + enable_mod("ssl") + + # Check for Listen 443 + # Note: This could be made to also look for ip:443 combo + # TODO: Need to search only open directives and IfMod mod_ssl.c + if len(self.find_directive(case_i("Listen"), "443")) == 0: + logger.debug("No Listen 443 directive found") + logger.debug("Setting the Apache Server to Listen on port 443") + path = self.add_gen_ssl_dir_config( + "/files%sports.conf" % self.server_root, "Listen", "443") + self.save_notes += "Added Listen 443 directive to %s\n" % path + def make_server_sni_ready(self, vhost, default_addr="*:443"): """Checks to see if the server is ready for SNI challenges. @@ -525,21 +563,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str default_addr: TODO - investigate function further """ - # Check if mod_ssl is loaded - if not check_ssl_loaded(): - logger.error("Please load the SSL module with Apache") - return False - - # Check for Listen 443 - # TODO: This could be made to also look for ip:443 combo - # TODO: Need to search only open directives and IfMod mod_ssl.c - if len(self.find_directive(case_i("Listen"), "443")) == 0: - logger.debug("No Listen 443 directive found") - logger.debug("Setting the Apache Server to Listen on port 443") - self.add_dir_to_ifmodssl("/files%sports.conf" % self.server_root, - "Listen", "443") - self.save_notes += "Added Listen 443 directive to ports.conf\n" - # Check for NameVirtualHost # First see if any of the vhost addresses is a _default_ addr for addr in vhost.addrs: @@ -560,7 +583,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return True - def get_ifmod(self, aug_conf_path, mod): + def _get_ifmod(self, aug_conf_path, mod): """Returns the path to and creates one if it doesn't exist. :param str aug_conf_path: Augeas configuration path @@ -586,7 +609,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str directive: Directive to add :param str arg: Value of the directive. ie. Listen 443, 443 is arg - """ self.aug.set(aug_conf_path + "/directive[last() + 1]", directive) if type(arg) is not list: @@ -1256,7 +1278,7 @@ LogLevel warn \n\ :rtype: bool """ - return apache_restart(quiet) + return apache_restart() def _add_httpd_transform(self, incl): """Add a transform to Augeas. @@ -1326,7 +1348,7 @@ LogLevel warn \n\ raise errors.LetsEncryptConfiguratorError( "Unable to find Apache version") - num_decimal = tuple(matches[0].split('.')) + return tuple(matches[0].split('.')) ########################################################################### # Challenges Section @@ -1508,7 +1530,7 @@ def enable_mod(mod_name): stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) # Hopefully this waits for output - subprocess.check_call(["sudo", APACHE2, "restart"], + subprocess.check_call(["sudo", CONFIG.APACHE2, "restart"], stdout=open("/dev/null", 'w'), stderr=open("/dev/null", 'w')) except (OSError, subprocess.CalledProcessError) as err: @@ -1531,12 +1553,13 @@ def check_ssl_loaded(): try: # p=subprocess.check_output(['sudo', '/usr/sbin/apache2ctl', '-M'], # stderr=open("/dev/null", 'w')) - proc = subprocess.Popen(['sudo', '/usr/sbin/apache2ctl', '-M'], + proc = subprocess.Popen([CONFIG.APACHE_CTL, '-M'], stdout=subprocess.PIPE, stderr=open( "/dev/null", 'w')).communicate()[0] except (OSError, ValueError): - logger.error("Error accessing apache2ctl for loaded modules!") + logger.error( + "Error accessing %s for loaded modules!" % CONFIG.APACHE_CTL) logger.error("This may be caused by an Apache Configuration Error") return False @@ -1553,7 +1576,7 @@ def apache_restart(): """ try: - proc = subprocess.Popen([APACHE2, 'restart'], + proc = subprocess.Popen([CONFIG.APACHE2, 'restart'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) text = proc.communicate() diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py index 1ca3c878d..3a74d227a 100644 --- a/letsencrypt/client/tests/apache_configurator_test.py +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -8,11 +8,12 @@ still running smoothly. """ +import re import os import shutil +import sys import tarfile import unittest -import sys from letsencrypt.client import apache_configurator from letsencrypt.client import CONFIG @@ -49,9 +50,14 @@ class TwoVhosts_80(unittest.TestCase): def setUp(self): # Final slash is currently important self.config_path = os.path.join(TEMP_DIR, "two_vhost_80/apache2/") + + # Using a new configurator every time allows the Configurator to clean + # up after itself self.config = apache_configurator.ApacheConfigurator( self.config_path, (2, 4, 7)) + self.aug_path = "/files" + self.config_path + prefix = os.path.join(TEMP_DIR, "two_vhost_80/apache2/sites-available/") aug_pre = "/files" + prefix self.vh_truth = [] @@ -128,44 +134,68 @@ class TwoVhosts_80(unittest.TestCase): self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].file)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].file)) - # def test_deploy_cert(self): - # """test deploy_cert.""" - # self.config.deploy_cert( - # self.vh_truth[1], - # "example/cert.pem", "example/key.pem", "example/cert_chain.pem") + def test_deploy_cert(self): + """test deploy_cert. - # loc_cert = self.config.find_directive( - # apache_configurator.case_i( - # "sslcertificatefile"), "example/cert.pem") - # loc_key = self.config.find_directive( - # apache_configurator.case_i( - # "sslcertificateKeyfile"), "example/key.pem") - # loc_chain = self.config.find_directive( - # apache_configurator.case_i( - # "SSLCertificateChainFile"), "example/chain.pem") + This test modifies the default-ssl vhost SSL directives. - # self.assertTrue(len(loc_cert) == 1 and - # apache_configurator.get_file_path( - # loc_cert[0]) == self.vh_truth[1].file) + """ + self.config.deploy_cert( + self.vh_truth[1], + "example/cert.pem", "example/key.pem", "example/cert_chain.pem") - # self.assertTrue(len(loc_key) == 1 and - # apache_configurator.get_file_path( - # loc_key[0]) == self.vh_truth[1].file) + loc_cert = self.config.find_directive( + apache_configurator.case_i("sslcertificatefile"), + re.escape("example/cert.pem"), self.vh_truth[1].path) + loc_key = self.config.find_directive( + apache_configurator.case_i("sslcertificateKeyfile"), + re.escape("example/key.pem"), self.vh_truth[1].path) + loc_chain = self.config.find_directive( + apache_configurator.case_i("SSLCertificateChainFile"), + re.escape("example/cert_chain.pem"), self.vh_truth[1].path) - # self.assertTrue(len(loc_chain) == 1 and - # apache_configurator.get_file_path( - # loc_chain[0]) == self.vh_truth[1].file) + # debug_file(self.vh_truth[1].file) + + # Verify one directive was found in the correct file + self.assertTrue(len(loc_cert) == 1 and + apache_configurator.get_file_path(loc_cert[0]) == + self.vh_truth[1].file) + + self.assertTrue(len(loc_key) == 1 and + apache_configurator.get_file_path(loc_key[0]) == + self.vh_truth[1].file) + + self.assertTrue(len(loc_chain) == 1 and + apache_configurator.get_file_path(loc_chain[0]) == + self.vh_truth[1].file) def test_is_name_vhost(self): """test is_name_vhost.""" - self.assertTrue(not self.config.is_name_vhost("*:80")) + self.assertTrue(self.config.is_name_vhost("*:80")) + self.config.version = (2, 2) + self.assertFalse(self.config.is_name_vhost("*:80")) - # def test_add_name_vhost(self): - # """test add_name_vhost.""" - # self.config.add_name_vhost("*:443") - # self.config.save(temporary=True) + def test_add_name_vhost(self): + """test add_name_vhost.""" + self.config.add_name_vhost("*:443") + self.config.save(temporary=True) + self.assertTrue(self.config.find_directive( + "NameVirtualHost", re.escape("*:443"))) - # self.assertTrue(self.config.is_name_vhost("*:443")) + def test_add_dir_to_ifmodssl(self): + """test _add_dir_to_ifmodssl. + + .. todo:: test what happens when a bad path is given... ie. ports.conf + doesn't exist + + """ + self.config._add_dir_to_ifmodssl( + self.aug_path + "ports.conf", "FakeDirective", "123") + + matches = self.config.find_directive("FakeDirective", "123") + print matches + self.assertTrue(len(matches) == 1) + self.assertTrue("IfModule" in matches[0]) def _verify_redirect(self, config_path): with open(config_path, 'r') as config_fd: @@ -174,6 +204,10 @@ class TwoVhosts_80(unittest.TestCase): return CONFIG.REWRITE_HTTPS_ARGS[1] in conf +def debug_file(filepath): + with open(filepath, 'r')as file_d: + print file_d.read() + if __name__ == '__main__': unittest.main() # def download_unpack_tests(url=CONFIG_TGZ_URL): From 0673a966c5b597e8c2722e2a27fe5734eccfc0f9 Mon Sep 17 00:00:00 2001 From: James Kasten Date: Fri, 5 Dec 2014 23:54:37 -0800 Subject: [PATCH 06/22] Generalize variables in configurator --- letsencrypt/client/apache_configurator.py | 106 ++++++++---------- .../client/tests/apache_configurator_test.py | 4 +- 2 files changed, 50 insertions(+), 60 deletions(-) diff --git a/letsencrypt/client/apache_configurator.py b/letsencrypt/client/apache_configurator.py index ef0d88651..9f6da6b33 100644 --- a/letsencrypt/client/apache_configurator.py +++ b/letsencrypt/client/apache_configurator.py @@ -48,16 +48,12 @@ class VH(object): """Represents an Apache Virtualhost. :ivar str file: filename path of VH - :ivar str path: Augeas path to virtual host - :ivar list addrs: Virtual Host addresses (:class:`list` of :class:`str`) - :ivar list names: Server names/aliases of vhost (:class:`list` of :class:`str`) :ivar bool ssl: SSLEngine on in vhost - :ivar bool enabled: Virtual host is enabled """ @@ -107,10 +103,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): This class was originally developed for Apache 2.2 and has not seen a an overhaul to include proper setup of new Apache configurations. - The biggest changes have been the IncludeOptional directive, the - deprecation of the NameVirtualHost directive, and the name change of - mod_ssl.c to ssl_module. Although these changes - have not been implemented yet, they will be shortly. + I have implemented most of the changes... the missing ones are + mod_ssl.c vs ssl_mod, and I need to account for configuration variables. That being said, this class can still adequately configure most typical Apache 2.4 servers as the deprecated NameVirtualHost has no effect and the typical directories are parsed by the Augeas configuration @@ -122,11 +116,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): needs of client's are clarified with the new and developing protocol. :ivar str server_root: Path to Apache root directory - :ivar float version: version of Apache - :ivar str user_config_file: Path to the user's configuration file - :ivar list vhosts: All vhosts found in the configuration (:class:`list` of :class:`VH`) @@ -146,8 +137,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): self.recovery_routine() # Find configuration root and make sure augeas can parse it. - self.config_root = self._find_config_root() - self._parse_file(self.config_root) + self.location = self._set_locations() + self._parse_file(self.location["root"]) # Must also attempt to parse sites-available or equivalent # Sites-available is not included naturally in configuration @@ -342,7 +333,32 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return all_names + def _set_locations(self): + """Set default location for directives. + + Locations are given as file_paths + .. todo:: Make sure that files are included + + """ + + root = self._find_config_root() + default = self._set_user_config_file() + temp = os.path.join(self.server_root, "ports.conf") + if os.path.isfile(temp): + listen = temp + name = temp + else: + listen = default + name = default + + return {"root": root, + "default": default, + "listen": listen, + "name": name} + def _find_config_root(self): + """Find the Apache Configuration Root file.""" + location = ["apache2.conf", "httpd.conf"] for name in location: @@ -454,7 +470,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # behavior is undefined. Make sure that an exact match exists # search for NameVirtualHost directive for ip_addr - # check httpd.conf, ports.conf, # note ip_addr can be FQDN although Apache does not recommend it if (self.version >= (2, 4) or self.find_directive( @@ -471,45 +486,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): :param str addr: Address that will be added as NameVirtualHost directive """ - aug_path = "/files%sports.conf" % self.server_root - path = self.add_gen_ssl_dir_config(aug_path, "NameVirtualHost", addr) + path = self._add_dir_to_ifmodssl( + get_aug_path(self.location["name"]), "NameVirtualHost", addr) self.save_notes += "Setting %s to be NameBasedVirtualHost\n" % addr self.save_notes += "\tDirective added to %s\n" % path - def add_gen_ssl_dir_config(self, aug_path, directive, val): - """Adds directive to ifmodssl somewhere in config. - - First the function tries to place the ssl directive along the - aug_path, if that is not available it places the directive - in the self.user_config_file which is expected to be valid. - - :param str aug_path: Augeas configuration path - :param str directive: Directive you would like to add to config - :param str val: Value of the directive - - :returns: Path to file that directive was added - :rtype: str - - """ - # First try original path - self._add_dir_to_ifmodssl( - aug_path, re.escape(directive), re.escape(val)) - - if self.find_directive(case_i(directive), case_i(val), start=aug_path): - return get_file_path(aug_path) - - # Try default path - logger.debug( - "%s is not included in your Apache config..." % - os.path.basename(get_file_path(aug_path))) - logger.debug( - "Adding %s directive to %s" % (directive, self.user_config_file)) - - self._add_dir_to_ifmodssl( - "/files%s" % self.user_config_file, directive, val) - return self.user_config_file - def _add_dir_to_ifmodssl(self, aug_conf_path, directive, val): """Adds directive and value to IfMod ssl block. @@ -548,8 +530,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): if len(self.find_directive(case_i("Listen"), "443")) == 0: logger.debug("No Listen 443 directive found") logger.debug("Setting the Apache Server to Listen on port 443") - path = self.add_gen_ssl_dir_config( - "/files%sports.conf" % self.server_root, "Listen", "443") + path = self._add_dir_to_ifmodssl( + get_aug_path(self.location["listen"]), "Listen", "443") self.save_notes += "Added Listen 443 directive to %s\n" % path def make_server_sni_ready(self, vhost, default_addr="*:443"): @@ -645,7 +627,7 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): """ # Cannot place member variable in the definition of the function so... if not start: - start = "/files%s" % self.config_root + start = get_aug_path(self.location["root"]) # Debug code # print "find_dir:", directive, "arg:", arg, " | Looking in:", start @@ -751,8 +733,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): # If the include is a directory, just return the directory as a file if arg.endswith("/"): - return "/files" + arg[:len(arg)-1] - return "/files"+arg + return get_aug_path(arg[:len(arg)-1]) + return get_aug_path(arg) def make_vhost_ssl(self, nonssl_vhost): """Makes an ssl_vhost version of a nonssl_vhost. @@ -1008,7 +990,7 @@ LogLevel warn \n\ self.aug.load() # Make a new vhost data structure and add it to the lists new_fp = self.server_root + "sites-available/" + redirect_filename - new_vhost = self._create_vhost("/files" + new_fp) + new_vhost = self._create_vhost(get_aug_path(new_fp)) self.vhosts.append(new_vhost) # Finally create documentation for the change @@ -1475,7 +1457,7 @@ LogLevel warn \n\ list_sni_tuple[idx][2], lis, dvsni_key.file) config_text += " \n" - self.dvsni_conf_include_check(self.user_config_file) + self.dvsni_conf_include_check(self.location["default"]) self.register_file_creation(True, CONFIG.APACHE_CHALLENGE_CONF) with open(CONFIG.APACHE_CHALLENGE_CONF, 'w') as new_conf: @@ -1493,7 +1475,7 @@ LogLevel warn \n\ if len(self.find_directive( case_i("Include"), CONFIG.APACHE_CHALLENGE_CONF)) == 0: # print "Including challenge virtual host(s)" - self.add_dir("/files" + main_config, + self.add_dir(get_aug_path(main_config), "Include", CONFIG.APACHE_CHALLENGE_CONF) def dvsni_create_chall_cert(self, name, ext, nonce, dvsni_key): @@ -1652,6 +1634,13 @@ def get_file_path(vhost_path): break return avail_fp +def get_aug_path(file_path): + """Return augeas path for full filepath. + + :param str file_path: Full filepath + + """ + return "/files%s" % file_path def strip_dir(path): """Returns directory of file path. @@ -1716,7 +1705,8 @@ def dvsni_gen_ext(dvsni_r, dvsni_s): :param bytearray dvsni_r: DVSNI r value :param bytearray dvsni_s: DVSNI s value - result: returns z + CONFIG.INVALID_EXT + :returns: z + CONFIG.INVALID_EXT + :rtype: str """ z_base = hashlib.new('sha256') diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py index 3a74d227a..18164e290 100644 --- a/letsencrypt/client/tests/apache_configurator_test.py +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -178,7 +178,7 @@ class TwoVhosts_80(unittest.TestCase): def test_add_name_vhost(self): """test add_name_vhost.""" self.config.add_name_vhost("*:443") - self.config.save(temporary=True) + #self.config.save(temporary=True) self.assertTrue(self.config.find_directive( "NameVirtualHost", re.escape("*:443"))) @@ -193,7 +193,7 @@ class TwoVhosts_80(unittest.TestCase): self.aug_path + "ports.conf", "FakeDirective", "123") matches = self.config.find_directive("FakeDirective", "123") - print matches + self.assertTrue(len(matches) == 1) self.assertTrue("IfModule" in matches[0]) From 1fdf9b39c9be005574fc0c7e57efcab90883089f Mon Sep 17 00:00:00 2001 From: James Kasten Date: Sat, 6 Dec 2014 02:33:06 -0800 Subject: [PATCH 07/22] unittests add_dir and make_vhost_ssl --- letsencrypt/client/apache_configurator.py | 22 +++++----- .../client/tests/apache_configurator_test.py | 43 ++++++++++++++++++- 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/letsencrypt/client/apache_configurator.py b/letsencrypt/client/apache_configurator.py index 9f6da6b33..6f575509a 100644 --- a/letsencrypt/client/apache_configurator.py +++ b/letsencrypt/client/apache_configurator.py @@ -225,7 +225,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): return False logger.info("Deploying Certificate to VirtualHost %s" % vhost.file) - print path self.aug.set(path["cert_file"][0], cert) self.aug.set(path["cert_key"][0], key) @@ -537,14 +536,14 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def make_server_sni_ready(self, vhost, default_addr="*:443"): """Checks to see if the server is ready for SNI challenges. - .. todo:: This should largely depend on the version of Apache - :param vhost: VHost to check SNI compatibility :type vhost: :class:`VH` :param str default_addr: TODO - investigate function further """ + if self.version >= (2, 4): + return # Check for NameVirtualHost # First see if any of the vhost addresses is a _default_ addr for addr in vhost.addrs: @@ -555,7 +554,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "%s to be name based vhosts" % default_addr)) self.add_name_vhost(default_addr) - return True # No default addresses... so set each one individually for addr in vhost.addrs: if not self.is_name_vhost(addr): @@ -563,8 +561,6 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): "to be a name based virtual host" % addr)) self.add_name_vhost(addr) - return True - def _get_ifmod(self, aug_conf_path, mod): """Returns the path to and creates one if it doesn't exist. @@ -585,7 +581,8 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): def add_dir(self, aug_conf_path, directive, arg): """Appends directive to the end fo the file given by aug_conf_path. - Note: Not added to AugeasConfigurator because it may depend on the lens + .. note:: Not added to AugeasConfigurator because it may depend + on the lens :param str aug_conf_path: Augeas configuration path to add directive :param str directive: Directive to add @@ -662,12 +659,12 @@ class ApacheConfigurator(augeas_configurator.AugeasConfigurator): for include in includes: # start[6:] to strip off /files matches.extend(self.find_directive( - directive, arg, self.get_include_path(strip_dir(start[6:]), - self.aug.get(include)))) + directive, arg, self._get_include_path(strip_dir(start[6:]), + self.aug.get(include)))) return matches - def get_include_path(self, cur_dir, arg): + def _get_include_path(self, cur_dir, arg): """Converts an Apache Include directive into Augeas path. Converts an Apache Include directive argument into an Augeas @@ -1387,8 +1384,7 @@ LogLevel warn \n\ return None # TODO - @jdkasten review this code to make sure it makes sense - if not self.make_server_sni_ready(vhost, default_addr): - return None + self.make_server_sni_ready(vhost, default_addr) for addr in vhost.addrs: if "_default_" in addr: @@ -1634,6 +1630,7 @@ def get_file_path(vhost_path): break return avail_fp + def get_aug_path(file_path): """Return augeas path for full filepath. @@ -1642,6 +1639,7 @@ def get_aug_path(file_path): """ return "/files%s" % file_path + def strip_dir(path): """Returns directory of file path. diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py index 18164e290..4189bc2e4 100644 --- a/letsencrypt/client/tests/apache_configurator_test.py +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -134,6 +134,15 @@ class TwoVhosts_80(unittest.TestCase): self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].file)) self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].file)) + def test_add_dir(self): + """test add_dir.""" + aug_default = "/files" + self.config.location["default"] + self.config.add_dir( + aug_default, "AddDirective", "test") + + self.assertTrue( + self.config.find_directive("AddDirective", "test", aug_default)) + def test_deploy_cert(self): """test deploy_cert. @@ -178,7 +187,7 @@ class TwoVhosts_80(unittest.TestCase): def test_add_name_vhost(self): """test add_name_vhost.""" self.config.add_name_vhost("*:443") - #self.config.save(temporary=True) + # self.config.save(temporary=True) self.assertTrue(self.config.find_directive( "NameVirtualHost", re.escape("*:443"))) @@ -190,14 +199,43 @@ class TwoVhosts_80(unittest.TestCase): """ self.config._add_dir_to_ifmodssl( - self.aug_path + "ports.conf", "FakeDirective", "123") + "/files" + self.config.location["default"], "FakeDirective", "123") matches = self.config.find_directive("FakeDirective", "123") self.assertTrue(len(matches) == 1) self.assertTrue("IfModule" in matches[0]) + def test_make_vhost_ssl(self): + """test make_vhost_ssl.""" + ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0]) + + self.assertTrue( + ssl_vhost.file == + os.path.join(self.config_path, "sites-available", + "encryption-example-le-ssl.conf")) + + self.assertTrue(ssl_vhost.path == + "/files" + ssl_vhost.file + "/IfModule/VirtualHost") + self.assertTrue(ssl_vhost.addrs == ["*:443"]) + self.assertTrue(ssl_vhost.names == ["encryption-example.demo"]) + self.assertTrue(ssl_vhost.ssl) + self.assertFalse(ssl_vhost.enabled) + + self.assertTrue(self.config.find_directive( + "SSLCertificateFile", None, ssl_vhost.path)) + self.assertTrue(self.config.find_directive( + "SSLCertificateKeyFile", None, ssl_vhost.path)) + self.assertTrue(self.config.find_directive( + "Include", CONFIG.OPTIONS_SSL_CONF, ssl_vhost.path)) + + self.assertTrue(self.config.is_name_vhost(self.vh_truth[0]) == + self.config.is_name_vhost(ssl_vhost)) + + self.assertTrue(len(self.config.vhosts) == 5) + def _verify_redirect(self, config_path): + """Verifies that the vhost contains the REWRITE.""" with open(config_path, 'r') as config_fd: conf = config_fd.read() @@ -205,6 +243,7 @@ class TwoVhosts_80(unittest.TestCase): def debug_file(filepath): + """Print out the file.""" with open(filepath, 'r')as file_d: print file_d.read() From cc6aca6aa189a496bc5ecee788b0195a0071d5be Mon Sep 17 00:00:00 2001 From: James Kasten Date: Sun, 7 Dec 2014 19:36:46 -0800 Subject: [PATCH 08/22] Move test configs into repo and apply necessary changes. --- .travis.yml | 2 +- .../client/tests/apache_configurator_test.py | 35 +- .../default_vhost/apache2/apache2.conf | 223 +++++ .../apache2/conf-available/charset.conf | 8 + .../conf-available/localized-error-pages.conf | 81 ++ .../other-vhosts-access-log.conf | 4 + .../apache2/conf-available/security.conf | 74 ++ .../apache2/conf-available/serve-cgi-bin.conf | 20 + .../apache2/conf-enabled/charset.conf | 8 + .../conf-enabled/localized-error-pages.conf | 81 ++ .../conf-enabled/other-vhosts-access-log.conf | 4 + .../apache2/conf-enabled/security.conf | 74 ++ .../apache2/conf-enabled/serve-cgi-bin.conf | 20 + .../default_vhost/apache2/envvars | 47 + .../default_vhost/apache2/magic | 935 ++++++++++++++++++ .../apache2/mods-available/ssl.conf | 89 ++ .../apache2/mods-available/ssl.load | 2 + .../default_vhost/apache2/ports.conf | 20 + .../apache2/sites-available/000-default.conf | 32 + .../apache2/sites-available/default-ssl.conf | 136 +++ .../apache2/sites-enabled/000-default.conf | 1 + .../debian_apache_2_4/default_vhost/sites | 1 + .../two_vhost_80/apache2/apache2.conf | 223 +++++ .../apache2/conf-available/charset.conf | 8 + .../conf-available/localized-error-pages.conf | 81 ++ .../other-vhosts-access-log.conf | 4 + .../apache2/conf-available/security.conf | 74 ++ .../apache2/conf-available/serve-cgi-bin.conf | 20 + .../apache2/conf-enabled/charset.conf | 8 + .../conf-enabled/localized-error-pages.conf | 81 ++ .../conf-enabled/other-vhosts-access-log.conf | 4 + .../apache2/conf-enabled/security.conf | 74 ++ .../apache2/conf-enabled/serve-cgi-bin.conf | 20 + .../two_vhost_80/apache2/envvars | 47 + .../two_vhost_80/apache2/magic | 935 ++++++++++++++++++ .../apache2/mods-available/ssl.conf | 89 ++ .../apache2/mods-available/ssl.load | 2 + .../two_vhost_80/apache2/ports.conf | 15 + .../apache2/sites-available/000-default.conf | 32 + .../apache2/sites-available/default-ssl.conf | 136 +++ .../sites-available/encryption-example.conf | 42 + .../apache2/sites-available/letsencrypt.conf | 42 + .../apache2/sites-enabled/000-default.conf | 1 + .../sites-enabled/encryption-example.conf | 1 + .../apache2/sites-enabled/letsencrypt.conf | 1 + .../debian_apache_2_4/two_vhost_80/sites | 2 + 46 files changed, 3815 insertions(+), 24 deletions(-) create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/apache2.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/charset.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/localized-error-pages.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/other-vhosts-access-log.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/security.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/serve-cgi-bin.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/charset.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/localized-error-pages.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/other-vhosts-access-log.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/security.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/serve-cgi-bin.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/envvars create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/magic create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/mods-available/ssl.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/mods-available/ssl.load create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/ports.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/sites-available/000-default.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/sites-available/default-ssl.conf create mode 120000 letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/sites-enabled/000-default.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/default_vhost/sites create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/apache2.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-available/charset.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-available/localized-error-pages.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-available/other-vhosts-access-log.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-available/security.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-available/serve-cgi-bin.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-enabled/charset.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-enabled/localized-error-pages.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-enabled/other-vhosts-access-log.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-enabled/security.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/conf-enabled/serve-cgi-bin.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/envvars create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/magic create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/mods-available/ssl.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/mods-available/ssl.load create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/ports.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-available/000-default.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-available/default-ssl.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-available/encryption-example.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-available/letsencrypt.conf create mode 120000 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-enabled/000-default.conf create mode 120000 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-enabled/encryption-example.conf create mode 120000 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/apache2/sites-enabled/letsencrypt.conf create mode 100644 letsencrypt/client/tests/debian_apache_2_4/two_vhost_80/sites diff --git a/.travis.yml b/.travis.yml index d55cfdf77..3a3a4b256 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ before_install: > python-virtualenv python-dev gcc swig dialog libaugeas0 libssl-dev install: travis_retry python setup.py dev # installs tox -script: travis_retry tox +script: travis_retry sudo tox env: - TOXENV=py${TRAVIS_PYTHON_VERSION//[.]/} diff --git a/letsencrypt/client/tests/apache_configurator_test.py b/letsencrypt/client/tests/apache_configurator_test.py index 4189bc2e4..1c2ecb92a 100644 --- a/letsencrypt/client/tests/apache_configurator_test.py +++ b/letsencrypt/client/tests/apache_configurator_test.py @@ -12,25 +12,21 @@ import re import os import shutil import sys -import tarfile import unittest from letsencrypt.client import apache_configurator from letsencrypt.client import CONFIG from letsencrypt.client import display -from letsencrypt.client import le_util from letsencrypt.client import logger # Some of these will likely go into a letsencrypt.tests.CONFIG file -TESTING_DIR = "/home/ubuntu/testing/" -UBUNTU_CONFIGS = os.path.join(TESTING_DIR, "ubuntu_apache_2_4/") +TESTING_DIR = os.path.dirname(os.path.realpath(__file__)) +UBUNTU_CONFIGS = os.path.join(TESTING_DIR, "debian_apache_2_4/") TEMP_DIR = os.path.join(TESTING_DIR, "temp") -# This will end up going into the repo... -CONFIG_TGZ_URL = "https://jdkasten.com/projects/config.tgz" - def setUpModule(): + """Run once before all unittests.""" logger.setLogger(logger.FileLogger(sys.stdout)) logger.setLogLevel(logger.INFO) display.set_display(display.NcursesDisplay()) @@ -42,12 +38,15 @@ def setUpModule(): def tearDownModule(): + """Run once after all unittests.""" shutil.rmtree(TEMP_DIR) -class TwoVhosts_80(unittest.TestCase): +class TwoVhost80(unittest.TestCase): + """Standard two http vhosts that are well configured.""" def setUp(self): + """Run before each and every tests.""" # Final slash is currently important self.config_path = os.path.join(TEMP_DIR, "two_vhost_80/apache2/") @@ -234,12 +233,12 @@ class TwoVhosts_80(unittest.TestCase): self.assertTrue(len(self.config.vhosts) == 5) - def _verify_redirect(self, config_path): - """Verifies that the vhost contains the REWRITE.""" - with open(config_path, 'r') as config_fd: - conf = config_fd.read() + # def _verify_redirect(self, config_path): + # """Verifies that the vhost contains the REWRITE.""" + # with open(config_path, 'r') as config_fd: + # conf = config_fd.read() - return CONFIG.REWRITE_HTTPS_ARGS[1] in conf + # return CONFIG.REWRITE_HTTPS_ARGS[1] in conf def debug_file(filepath): @@ -249,13 +248,3 @@ def debug_file(filepath): if __name__ == '__main__': unittest.main() -# def download_unpack_tests(url=CONFIG_TGZ_URL): -# r = requests.get(url) -# local_tgz_file = os.path.join(TESTING_DIR, 'ubuntu_2_4.tgz') -# with open(local_tgz_file, 'w') as tgz_file: -# tgz_file.write(r.content) - -# if tarfile.is_tarfile(local_tgz_file): -# tar = tarfile.open(local_tgz_file) -# tar.extractall() -# tar.close() diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/apache2.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/apache2.conf new file mode 100644 index 000000000..b9f7bc0b6 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/apache2.conf @@ -0,0 +1,223 @@ +# This is the main Apache server configuration file. It contains the +# configuration directives that give the server its instructions. +# See http://httpd.apache.org/docs/2.4/ for detailed information about +# the directives and /usr/share/doc/apache2/README.Debian about Debian specific +# hints. +# +# +# Summary of how the Apache 2 configuration works in Debian: +# The Apache 2 web server configuration in Debian is quite different to +# upstream's suggested way to configure the web server. This is because Debian's +# default Apache2 installation attempts to make adding and removing modules, +# virtual hosts, and extra configuration directives as flexible as possible, in +# order to make automating the changes and administering the server as easy as +# possible. + +# It is split into several files forming the configuration hierarchy outlined +# below, all located in the /etc/apache2/ directory: +# +# /etc/apache2/ +# |-- apache2.conf +# | `-- ports.conf +# |-- mods-enabled +# | |-- *.load +# | `-- *.conf +# |-- conf-enabled +# | `-- *.conf +# `-- sites-enabled +# `-- *.conf +# +# +# * apache2.conf is the main configuration file (this file). It puts the pieces +# together by including all remaining configuration files when starting up the +# web server. +# +# * ports.conf is always included from the main configuration file. It is +# supposed to determine listening ports for incoming connections which can be +# customized anytime. +# +# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/ +# directories contain particular configuration snippets which manage modules, +# global configuration fragments, or virtual host configurations, +# respectively. +# +# They are activated by symlinking available configuration files from their +# respective *-available/ counterparts. These should be managed by using our +# helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See +# their respective man pages for detailed information. +# +# * The binary is called apache2. Due to the use of environment variables, in +# the default configuration, apache2 needs to be started/stopped with +# /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not +# work with the default configuration. + + +# Global configuration +# + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# NOTE! If you intend to place this on an NFS (or otherwise network) +# mounted filesystem then please read the Mutex documentation (available +# at ); +# you will save yourself a lot of trouble. +# +# Do NOT add a slash at the end of the directory path. +# +#ServerRoot "/etc/apache2" + +# +# The accept serialization lock file MUST BE STORED ON A LOCAL DISK. +# +Mutex file:${APACHE_LOCK_DIR} default + +# +# PidFile: The file in which the server should record its process +# identification number when it starts. +# This needs to be set in /etc/apache2/envvars +# +PidFile ${APACHE_PID_FILE} + +# +# Timeout: The number of seconds before receives and sends time out. +# +Timeout 300 + +# +# KeepAlive: Whether or not to allow persistent connections (more than +# one request per connection). Set to "Off" to deactivate. +# +KeepAlive On + +# +# MaxKeepAliveRequests: The maximum number of requests to allow +# during a persistent connection. Set to 0 to allow an unlimited amount. +# We recommend you leave this number high, for maximum performance. +# +MaxKeepAliveRequests 100 + +# +# KeepAliveTimeout: Number of seconds to wait for the next request from the +# same client on the same connection. +# +KeepAliveTimeout 5 + + +# These need to be set in /etc/apache2/envvars +User ${APACHE_RUN_USER} +Group ${APACHE_RUN_GROUP} + +# +# HostnameLookups: Log the names of clients or just their IP addresses +# e.g., www.apache.org (on) or 204.62.129.132 (off). +# The default is off because it'd be overall better for the net if people +# had to knowingly turn this feature on, since enabling it means that +# each client request will result in AT LEAST one lookup request to the +# nameserver. +# +HostnameLookups Off + +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog ${APACHE_LOG_DIR}/error.log + +# +# LogLevel: Control the severity of messages logged to the error_log. +# Available values: trace8, ..., trace1, debug, info, notice, warn, +# error, crit, alert, emerg. +# It is also possible to configure the log level for particular modules, e.g. +# "LogLevel info ssl:warn" +# +LogLevel warn + +# Include module configuration: +IncludeOptional mods-enabled/*.load +IncludeOptional mods-enabled/*.conf + +# Include list of ports to listen on +Include ports.conf + + +# Sets the default security model of the Apache2 HTTPD server. It does +# not allow access to the root filesystem outside of /usr/share and /var/www. +# The former is used by web applications packaged in Debian, +# the latter may be used for local directories served by the web server. If +# your system is serving content from a sub-directory in /srv you must allow +# access here, or in any related virtual host. + + Options FollowSymLinks + AllowOverride None + Require all denied + + + + AllowOverride None + Require all granted + + +# + + Options Indexes FollowSymLinks + AllowOverride None + Require all granted + + +# +# Options Indexes FollowSymLinks +# AllowOverride None +# Require all granted +# + + + + +# AccessFileName: The name of the file to look for in each directory +# for additional configuration directives. See also the AllowOverride +# directive. +# +AccessFileName .htaccess + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + + +# +# The following directives define some format nicknames for use with +# a CustomLog directive. +# +# These deviate from the Common Log Format definitions in that they use %O +# (the actual bytes sent including headers) instead of %b (the size of the +# requested file), because the latter makes it impossible to detect partial +# requests. +# +# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended. +# Use mod_remoteip instead. +# +LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined +LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined +LogFormat "%h %l %u %t \"%r\" %>s %O" common +LogFormat "%{Referer}i -> %U" referer +LogFormat "%{User-agent}i" agent + +# Include of directories ignores editors' and dpkg's backup files, +# see README.Debian for details. + +# Include generic snippets of statements +IncludeOptional conf-enabled/*.conf + +# Include the virtual host configurations: +IncludeOptional sites-enabled/*.conf + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet +ServerName 127.0.1.1 diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/charset.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/charset.conf new file mode 100644 index 000000000..8b0f41594 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/charset.conf @@ -0,0 +1,8 @@ +# Read the documentation before enabling AddDefaultCharset. +# In general, it is only a good idea if you know that all your files +# have this encoding. It will override any encoding given in the files +# in meta http-equiv or xml encoding tags. + +#AddDefaultCharset UTF-8 + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/localized-error-pages.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/localized-error-pages.conf new file mode 100644 index 000000000..f188d806f --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/localized-error-pages.conf @@ -0,0 +1,81 @@ +# Customizable error responses come in three flavors: +# 1) plain text +# 2) local redirects +# 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# Putting this all together, we can internationalize error responses. +# +# We use Alias to redirect any /error/HTTP_.html.var response to +# our collection of by-error message multi-language collections. We use +# includes to substitute the appropriate text. +# +# You can modify the messages' appearance without changing any of the +# default HTTP_.html.var files by adding the line: +# +#Alias /error/include/ "/your/include/path/" +# +# which allows you to create your own set of files by starting with the +# /usr/share/apache2/error/include/ files and copying them to /your/include/path/, +# even on a per-VirtualHost basis. If you include the Alias in the global server +# context, is has to come _before_ the 'Alias /error/ ...' line. +# +# The default include files will display your Apache version number and your +# ServerAdmin email address regardless of the setting of ServerSignature. +# +# WARNING: The configuration below will NOT work out of the box if you have a +# SetHandler directive in a context somewhere. Adding +# the following three lines AFTER the context should +# make it work in most cases: +# +# SetHandler none +# +# +# The internationalized error documents require mod_alias, mod_include +# and mod_negotiation. To activate them, uncomment the following 37 lines. + +# +# +# +# +# Alias /error/ "/usr/share/apache2/error/" +# +# +# Options IncludesNoExec +# AddOutputFilter Includes html +# AddHandler type-map var +# Order allow,deny +# Allow from all +# LanguagePriority en cs de es fr it nl sv pt-br ro +# ForceLanguagePriority Prefer Fallback +# +# +# ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var +# ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var +# ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var +# ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var +# ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var +# ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var +# ErrorDocument 410 /error/HTTP_GONE.html.var +# ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var +# ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var +# ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var +# ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var +# ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var +# ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var +# ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var +# ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var +# ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var +# ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var +# +# +# + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/other-vhosts-access-log.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/other-vhosts-access-log.conf new file mode 100644 index 000000000..5e9f5e9e7 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/other-vhosts-access-log.conf @@ -0,0 +1,4 @@ +# Define an access log for VirtualHosts that don't define their own logfile +CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log vhost_combined + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/security.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/security.conf new file mode 100644 index 000000000..599333b14 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/security.conf @@ -0,0 +1,74 @@ +# +# Disable access to the entire file system except for the directories that +# are explicitly allowed later. +# +# This currently breaks the configurations that come with some web application +# Debian packages. +# +# +# AllowOverride None +# Order Deny,Allow +# Deny from all +# + + +# Changing the following options will not really affect the security of the +# server, but might make attacks slightly more difficult in some cases. + +# +# ServerTokens +# This directive configures what you return as the Server HTTP response +# Header. The default is 'Full' which sends information about the OS-Type +# and compiled in modules. +# Set to one of: Full | OS | Minimal | Minor | Major | Prod +# where Full conveys the most information, and Prod the least. +#ServerTokens Minimal +ServerTokens OS +#ServerTokens Full + +# +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (internal error documents, FTP directory +# listings, mod_status and mod_info output etc., but not CGI generated +# documents or custom error documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +#ServerSignature Off +ServerSignature On + +# +# Allow TRACE method +# +# Set to "extended" to also reflect the request body (only for testing and +# diagnostic purposes). +# +# Set to one of: On | Off | extended +TraceEnable Off +#TraceEnable On + +# +# Forbid access to version control directories +# +# If you use version control systems in your document root, you should +# probably deny access to their directories. For example, for subversion: +# +# +# Require all denied +# + +# +# Setting this header will prevent MSIE from interpreting files as something +# else than declared by the content type in the HTTP headers. +# Requires mod_headers to be enabled. +# +#Header set X-Content-Type-Options: "nosniff" + +# +# Setting this header will prevent other sites from embedding pages from this +# site as frames. This defends against clickjacking attacks. +# Requires mod_headers to be enabled. +# +#Header set X-Frame-Options: "sameorigin" + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/serve-cgi-bin.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/serve-cgi-bin.conf new file mode 100644 index 000000000..b02782dab --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-available/serve-cgi-bin.conf @@ -0,0 +1,20 @@ + + + Define ENABLE_USR_LIB_CGI_BIN + + + + Define ENABLE_USR_LIB_CGI_BIN + + + + ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/ + + AllowOverride None + Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch + Require all granted + + + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/charset.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/charset.conf new file mode 100644 index 000000000..8b0f41594 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/charset.conf @@ -0,0 +1,8 @@ +# Read the documentation before enabling AddDefaultCharset. +# In general, it is only a good idea if you know that all your files +# have this encoding. It will override any encoding given in the files +# in meta http-equiv or xml encoding tags. + +#AddDefaultCharset UTF-8 + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/localized-error-pages.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/localized-error-pages.conf new file mode 100644 index 000000000..f188d806f --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/localized-error-pages.conf @@ -0,0 +1,81 @@ +# Customizable error responses come in three flavors: +# 1) plain text +# 2) local redirects +# 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# Putting this all together, we can internationalize error responses. +# +# We use Alias to redirect any /error/HTTP_.html.var response to +# our collection of by-error message multi-language collections. We use +# includes to substitute the appropriate text. +# +# You can modify the messages' appearance without changing any of the +# default HTTP_.html.var files by adding the line: +# +#Alias /error/include/ "/your/include/path/" +# +# which allows you to create your own set of files by starting with the +# /usr/share/apache2/error/include/ files and copying them to /your/include/path/, +# even on a per-VirtualHost basis. If you include the Alias in the global server +# context, is has to come _before_ the 'Alias /error/ ...' line. +# +# The default include files will display your Apache version number and your +# ServerAdmin email address regardless of the setting of ServerSignature. +# +# WARNING: The configuration below will NOT work out of the box if you have a +# SetHandler directive in a context somewhere. Adding +# the following three lines AFTER the context should +# make it work in most cases: +# +# SetHandler none +# +# +# The internationalized error documents require mod_alias, mod_include +# and mod_negotiation. To activate them, uncomment the following 37 lines. + +# +# +# +# +# Alias /error/ "/usr/share/apache2/error/" +# +# +# Options IncludesNoExec +# AddOutputFilter Includes html +# AddHandler type-map var +# Order allow,deny +# Allow from all +# LanguagePriority en cs de es fr it nl sv pt-br ro +# ForceLanguagePriority Prefer Fallback +# +# +# ErrorDocument 400 /error/HTTP_BAD_REQUEST.html.var +# ErrorDocument 401 /error/HTTP_UNAUTHORIZED.html.var +# ErrorDocument 403 /error/HTTP_FORBIDDEN.html.var +# ErrorDocument 404 /error/HTTP_NOT_FOUND.html.var +# ErrorDocument 405 /error/HTTP_METHOD_NOT_ALLOWED.html.var +# ErrorDocument 408 /error/HTTP_REQUEST_TIME_OUT.html.var +# ErrorDocument 410 /error/HTTP_GONE.html.var +# ErrorDocument 411 /error/HTTP_LENGTH_REQUIRED.html.var +# ErrorDocument 412 /error/HTTP_PRECONDITION_FAILED.html.var +# ErrorDocument 413 /error/HTTP_REQUEST_ENTITY_TOO_LARGE.html.var +# ErrorDocument 414 /error/HTTP_REQUEST_URI_TOO_LARGE.html.var +# ErrorDocument 415 /error/HTTP_UNSUPPORTED_MEDIA_TYPE.html.var +# ErrorDocument 500 /error/HTTP_INTERNAL_SERVER_ERROR.html.var +# ErrorDocument 501 /error/HTTP_NOT_IMPLEMENTED.html.var +# ErrorDocument 502 /error/HTTP_BAD_GATEWAY.html.var +# ErrorDocument 503 /error/HTTP_SERVICE_UNAVAILABLE.html.var +# ErrorDocument 506 /error/HTTP_VARIANT_ALSO_VARIES.html.var +# +# +# + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/other-vhosts-access-log.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/other-vhosts-access-log.conf new file mode 100644 index 000000000..5e9f5e9e7 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/other-vhosts-access-log.conf @@ -0,0 +1,4 @@ +# Define an access log for VirtualHosts that don't define their own logfile +CustomLog ${APACHE_LOG_DIR}/other_vhosts_access.log vhost_combined + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/security.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/security.conf new file mode 100644 index 000000000..599333b14 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/security.conf @@ -0,0 +1,74 @@ +# +# Disable access to the entire file system except for the directories that +# are explicitly allowed later. +# +# This currently breaks the configurations that come with some web application +# Debian packages. +# +# +# AllowOverride None +# Order Deny,Allow +# Deny from all +# + + +# Changing the following options will not really affect the security of the +# server, but might make attacks slightly more difficult in some cases. + +# +# ServerTokens +# This directive configures what you return as the Server HTTP response +# Header. The default is 'Full' which sends information about the OS-Type +# and compiled in modules. +# Set to one of: Full | OS | Minimal | Minor | Major | Prod +# where Full conveys the most information, and Prod the least. +#ServerTokens Minimal +ServerTokens OS +#ServerTokens Full + +# +# Optionally add a line containing the server version and virtual host +# name to server-generated pages (internal error documents, FTP directory +# listings, mod_status and mod_info output etc., but not CGI generated +# documents or custom error documents). +# Set to "EMail" to also include a mailto: link to the ServerAdmin. +# Set to one of: On | Off | EMail +#ServerSignature Off +ServerSignature On + +# +# Allow TRACE method +# +# Set to "extended" to also reflect the request body (only for testing and +# diagnostic purposes). +# +# Set to one of: On | Off | extended +TraceEnable Off +#TraceEnable On + +# +# Forbid access to version control directories +# +# If you use version control systems in your document root, you should +# probably deny access to their directories. For example, for subversion: +# +# +# Require all denied +# + +# +# Setting this header will prevent MSIE from interpreting files as something +# else than declared by the content type in the HTTP headers. +# Requires mod_headers to be enabled. +# +#Header set X-Content-Type-Options: "nosniff" + +# +# Setting this header will prevent other sites from embedding pages from this +# site as frames. This defends against clickjacking attacks. +# Requires mod_headers to be enabled. +# +#Header set X-Frame-Options: "sameorigin" + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/serve-cgi-bin.conf b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/serve-cgi-bin.conf new file mode 100644 index 000000000..b02782dab --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/conf-enabled/serve-cgi-bin.conf @@ -0,0 +1,20 @@ + + + Define ENABLE_USR_LIB_CGI_BIN + + + + Define ENABLE_USR_LIB_CGI_BIN + + + + ScriptAlias /cgi-bin/ /usr/lib/cgi-bin/ + + AllowOverride None + Options +ExecCGI -MultiViews +SymLinksIfOwnerMatch + Require all granted + + + + +# vim: syntax=apache ts=4 sw=4 sts=4 sr noet diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/envvars b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/envvars new file mode 100644 index 000000000..91328ac72 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/envvars @@ -0,0 +1,47 @@ +# envvars - default environment variables for apache2ctl + +# this won't be correct after changing uid +unset HOME + +# for supporting multiple apache2 instances +if [ "${APACHE_CONFDIR##/etc/apache2-}" != "${APACHE_CONFDIR}" ] ; then + SUFFIX="-${APACHE_CONFDIR##/etc/apache2-}" +else + SUFFIX= +fi + +# Since there is no sane way to get the parsed apache2 config in scripts, some +# settings are defined via environment variables and then used in apache2ctl, +# /etc/init.d/apache2, /etc/logrotate.d/apache2, etc. +export APACHE_RUN_USER=www-data +export APACHE_RUN_GROUP=www-data +# temporary state file location. This might be changed to /run in Wheezy+1 +export APACHE_PID_FILE=/var/run/apache2/apache2$SUFFIX.pid +export APACHE_RUN_DIR=/var/run/apache2$SUFFIX +export APACHE_LOCK_DIR=/var/lock/apache2$SUFFIX +# Only /var/log/apache2 is handled by /etc/logrotate.d/apache2. +export APACHE_LOG_DIR=/var/log/apache2$SUFFIX + +## The locale used by some modules like mod_dav +export LANG=C +## Uncomment the following line to use the system default locale instead: +#. /etc/default/locale + +export LANG + +## The command to get the status for 'apache2ctl status'. +## Some packages providing 'www-browser' need '--dump' instead of '-dump'. +#export APACHE_LYNX='www-browser -dump' + +## If you need a higher file descriptor limit, uncomment and adjust the +## following line (default is 8192): +#APACHE_ULIMIT_MAX_FILES='ulimit -n 65536' + +## If you would like to pass arguments to the web server, add them below +## to the APACHE_ARGUMENTS environment. +#export APACHE_ARGUMENTS='' + +## Enable the debug mode for maintainer scripts. +## This will produce a verbose output on package installations of web server modules and web application +## installations which interact with Apache +#export APACHE2_MAINTSCRIPT_DEBUG=1 diff --git a/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/magic b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/magic new file mode 100644 index 000000000..cdf9ac516 --- /dev/null +++ b/letsencrypt/client/tests/debian_apache_2_4/default_vhost/apache2/magic @@ -0,0 +1,935 @@ +# Magic data for mod_mime_magic (originally for file(1) command) +# +# The format is 4-5 columns: +# Column #1: byte number to begin checking from, ">" indicates continuation +# Column #2: type of data to match +# Column #3: contents of data to match +# Column #4: MIME type of result +# Column #5: MIME encoding of result (optional) + +#------------------------------------------------------------------------------ +# Localstuff: file(1) magic for locally observed files +# Add any locally observed files here. + +# Real Audio (Magic .ra\0375) +0 belong 0x2e7261fd audio/x-pn-realaudio +0 string .RMF application/vnd.rn-realmedia + +#video/x-pn-realvideo +#video/vnd.rn-realvideo +#application/vnd.rn-realmedia +# sigh, there are many mimes for that but the above are the most common. + +# Taken from magic, converted to magic.mime +# mime types according to http://www.geocities.com/nevilo/mod.htm: +# audio/it .it +# audio/x-zipped-it .itz +# audio/xm fasttracker modules +# audio/x-s3m screamtracker modules +# audio/s3m screamtracker modules +# audio/x-zipped-mod mdz +# audio/mod mod +# audio/x-mod All modules (mod, s3m, 669, mtm, med, xm, it, mdz, stm, itz, xmz, s3z) + +# Taken from loader code from mikmod version 2.14 +# by Steve McIntyre (stevem@chiark.greenend.org.uk) +# added title printing on 2003-06-24 +0 string MAS_UTrack_V00 +>14 string >/0 audio/x-mod +#audio/x-tracker-module + +#0 string UN05 MikMod UNI format module sound data + +0 string Extended\ Module: audio/x-mod +#audio/x-tracker-module +##>17 string >\0 Title: "%s" + +21 string/c \!SCREAM! audio/x-mod +#audio/x-screamtracker-module +21 string BMOD2STM audio/x-mod +#audio/x-screamtracker-module +1080 string M.K. audio/x-mod +#audio/x-protracker-module +#>0 string >\0 Title: "%s" +1080 string M!K! audio/x-mod +#audio/x-protracker-module +#>0 string >\0 Title: "%s" +1080 string FLT4 audio/x-mod +#audio/x-startracker-module +#>0 string >\0 Title: "%s" +1080 string FLT8 audio/x-mod +#audio/x-startracker-module +#>0 string >\0 Title: "%s" +1080 string 4CHN audio/x-mod +#audio/x-fasttracker-module +#>0 string >\0 Title: "%s" +1080 string 6CHN audio/x-mod +#audio/x-fasttracker-module +#>0 string >\0 Title: "%s" +1080 string 8CHN audio/x-mod +#audio/x-fasttracker-module +#>0 string >\0 Title: "%s" +1080 string CD81 audio/x-mod +#audio/x-oktalyzer-tracker-module +#>0 string >\0 Title: "%s" +1080 string OKTA audio/x-mod +#audio/x-oktalyzer-tracker-module +#>0 string >\0 Title: "%s" +# Not good enough. +#1082 string CH +#>1080 string >/0 %.2s-channel Fasttracker "oktalyzer" module sound data +1080 string 16CN audio/x-mod +#audio/x-taketracker-module +#>0 string >\0 Title: "%s" +1080 string 32CN audio/x-mod +#audio/x-taketracker-module +#>0 string >\0 Title: "%s" + +# Impuse tracker module (it) +0 string IMPM audio/x-mod +#>4 string >\0 "%s" +#>40 leshort !0 compatible w/ITv%x +#>42 leshort !0 created w/ITv%x + +#------------------------------------------------------------------------------ +# end local stuff +#------------------------------------------------------------------------------ + +# xml based formats! + +# svg + +0 string \38 string \<\!DOCTYPE\040svg image/svg+xml + + +# xml +0 string \2 short 0xbabe application/java + +#------------------------------------------------------------------------------ +# audio: file(1) magic for sound formats +# +# from Jan Nicolai Langfeldt , +# + +# Sun/NeXT audio data +0 string .snd +>12 belong 1 audio/basic +>12 belong 2 audio/basic +>12 belong 3 audio/basic +>12 belong 4 audio/basic +>12 belong 5 audio/basic +>12 belong 6 audio/basic +>12 belong 7 audio/basic + +>12 belong 23 audio/x-adpcm + +# DEC systems (e.g. DECstation 5000) use a variant of the Sun/NeXT format +# that uses little-endian encoding and has a different magic number +# (0x0064732E in little-endian encoding). +0 lelong 0x0064732E +>12 lelong 1 audio/x-dec-basic +>12 lelong 2 audio/x-dec-basic +>12 lelong 3 audio/x-dec-basic +>12 lelong 4 audio/x-dec-basic +>12 lelong 5 audio/x-dec-basic +>12 lelong 6 audio/x-dec-basic +>12 lelong 7 audio/x-dec-basic +# compressed (G.721 ADPCM) +>12 lelong 23 audio/x-dec-adpcm + +# Bytes 0-3 of AIFF, AIFF-C, & 8SVX audio files are "FORM" +# AIFF audio data +8 string AIFF audio/x-aiff +# AIFF-C audio data +8 string AIFC audio/x-aiff +# IFF/8SVX audio data +8 string 8SVX audio/x-aiff + + + +# Creative Labs AUDIO stuff +# Standard MIDI data +0 string MThd audio/unknown +#>9 byte >0 (format %d) +#>11 byte >1 using %d channels +# Creative Music (CMF) data +0 string CTMF audio/unknown +# SoundBlaster instrument data +0 string SBI audio/unknown +# Creative Labs voice data +0 string Creative\ Voice\ File audio/unknown +## is this next line right? it came this way... +#>19 byte 0x1A +#>23 byte >0 - version %d +#>22 byte >0 \b.%d + +# [GRR 950115: is this also Creative Labs? Guessing that first line +# should be string instead of unknown-endian long...] +#0 long 0x4e54524b MultiTrack sound data +#0 string NTRK MultiTrack sound data +#>4 long x - version %ld + +# Microsoft WAVE format (*.wav) +# [GRR 950115: probably all of the shorts and longs should be leshort/lelong] +# Microsoft RIFF +0 string RIFF +# - WAVE format +>8 string WAVE audio/x-wav +>8 string/B AVI video/x-msvideo +# +>8 string CDRA image/x-coreldraw + +# AAC (aka MPEG-2 NBC) +0 beshort&0xfff6 0xfff0 audio/X-HX-AAC-ADTS +0 string ADIF audio/X-HX-AAC-ADIF +0 beshort&0xffe0 0x56e0 audio/MP4A-LATM +0 beshort 0x4De1 audio/MP4A-LATM + +# MPEG Layer 3 sound files +0 beshort&0xfffe =0xfffa audio/mpeg +#MP3 with ID3 tag +0 string ID3 audio/mpeg +# Ogg/Vorbis +0 string OggS application/ogg + +#------------------------------------------------------------------------------ +# c-lang: file(1) magic for C programs or various scripts +# + +# XPM icons (Greg Roelofs, newt@uchicago.edu) +# ideally should go into "images", but entries below would tag XPM as C source +0 string /*\ XPM image/x-xpmi 7bit + +# 3DS (3d Studio files) +#16 beshort 0x3d3d image/x-3ds + +# this first will upset you if you're a PL/1 shop... (are there any left?) +# in which case rm it; ascmagic will catch real C programs +# C or REXX program text +#0 string /* text/x-c +# C++ program text +#0 string // text/x-c++ + +#------------------------------------------------------------------------------ +# commands: file(1) magic for various shells and interpreters +# +#0 string :\ shell archive or commands for antique kernel text +0 string #!/bin/sh application/x-shellscript +0 string #!\ /bin/sh application/x-shellscript +0 string #!/bin/csh application/x-shellscript +0 string #!\ /bin/csh application/x-shellscript +# korn shell magic, sent by George Wu, gwu@clyde.att.com +0 string #!/bin/ksh application/x-shellscript +0 string #!\ /bin/ksh application/x-shellscript +0 string #!/bin/tcsh application/x-shellscript +0 string #!\ /bin/tcsh application/x-shellscript +0 string #!/usr/local/tcsh application/x-shellscript +0 string #!\ /usr/local/tcsh application/x-shellscript +0 string #!/usr/local/bin/tcsh application/x-shellscript +0 string #!\ /usr/local/bin/tcsh application/x-shellscript +# bash shell magic, from Peter Tobias (tobias@server.et-inf.fho-emden.de) +0 string #!/bin/bash application/x-shellscript +0 string #!\ /bin/bash application/x-shellscript +0 string #!/usr/local/bin/bash application/x-shellscript +0 string #!\ /usr/local/bin/bash application/x-shellscript + +# +# zsh/ash/ae/nawk/gawk magic from cameron@cs.unsw.oz.au (Cameron Simpson) +0 string #!/bin/zsh application/x-shellscript +0 string #!/usr/bin/zsh application/x-shellscript +0 string #!/usr/local/bin/zsh application/x-shellscript +0 string #!\ /usr/local/bin/zsh application/x-shellscript +0 string #!/usr/local/bin/ash application/x-shellscript +0 string #!\ /usr/local/bin/ash application/x-shellscript +#0 string #!/usr/local/bin/ae Neil Brown's ae +#0 string #!\ /usr/local/bin/ae Neil Brown's ae +0 string #!/bin/nawk application/x-nawk +0 string #!\ /bin/nawk application/x-nawk +0 string #!/usr/bin/nawk application/x-nawk +0 string #!\ /usr/bin/nawk application/x-nawk +0 string #!/usr/local/bin/nawk application/x-nawk +0 string #!\ /usr/local/bin/nawk application/x-nawk +0 string #!/bin/gawk application/x-gawk +0 string #!\ /bin/gawk application/x-gawk +0 string #!/usr/bin/gawk application/x-gawk +0 string #!\ /usr/bin/gawk application/x-gawk +0 string #!/usr/local/bin/gawk application/x-gawk +0 string #!\ /usr/local/bin/gawk application/x-gawk +# +0 string #!/bin/awk application/x-awk +0 string #!\ /bin/awk application/x-awk +0 string #!/usr/bin/awk application/x-awk +0 string #!\ /usr/bin/awk application/x-awk +# update to distinguish from *.vcf files by Joerg Jenderek: joerg dot jenderek at web dot de +#0 regex BEGIN[[:space:]]*[{] application/x-awk + +# For Larry Wall's perl language. The ``eval'' line recognizes an +# outrageously clever hack for USG systems. +# Keith Waclena +0 string #!/bin/perl application/x-perl +0 string #!\ /bin/perl application/x-perl +0 string eval\ "exec\ /bin/perl application/x-perl +0 string #!/usr/bin/perl application/x-perl +0 string #!\ /usr/bin/perl application/x-perl +0 string eval\ "exec\ /usr/bin/perl application/x-perl +0 string #!/usr/local/bin/perl application/x-perl +0 string #!\ /usr/local/bin/perl application/x-perl +0 string eval\ "exec\ /usr/local/bin/perl application/x-perl + +#------------------------------------------------------------------------------ +# compress: file(1) magic for pure-compression formats (no archives) +# +# compress, gzip, pack, compact, huf, squeeze, crunch, freeze, yabba, whap, etc. +# +# Formats for various forms of compressed data +# Formats for "compress" proper have been moved into "compress.c", +# because it tries to uncompress it to figure out what's inside. + +# standard unix compress +#0 string \037\235 application/x-compress + +# gzip (GNU zip, not to be confused with [Info-ZIP/PKWARE] zip archiver) +#0 string \037\213 application/x-gzip + +0 string PK\003\004 application/x-zip + +# RAR archiver (Greg Roelofs, newt@uchicago.edu) +0 string Rar! application/x-rar + +# According to gzip.h, this is the correct byte order for packed data. +0 string \037\036 application/octet-stream +# +# This magic number is byte-order-independent. +# +0 short 017437 application/octet-stream + +# XXX - why *two* entries for "compacted data", one of which is +# byte-order independent, and one of which is byte-order dependent? +# +# compacted data +0 short 0x1fff application/octet-stream +0 string \377\037 application/octet-stream +# huf output +0 short 0145405 application/octet-stream + +# Squeeze and Crunch... +# These numbers were gleaned from the Unix versions of the programs to +# handle these formats. Note that I can only uncrunch, not crunch, and +# I didn't have a crunched file handy, so the crunch number is untested. +# Keith Waclena +#0 leshort 0x76FF squeezed data (CP/M, DOS) +#0 leshort 0x76FE crunched data (CP/M, DOS) + +# Freeze +#0 string \037\237 Frozen file 2.1 +#0 string \037\236 Frozen file 1.0 (or gzip 0.5) + +# lzh? +#0 string \037\240 LZH compressed data + +257 string ustar\0 application/x-tar posix +257 string ustar\040\040\0 application/x-tar gnu + +0 short 070707 application/x-cpio +0 short 0143561 application/x-cpio swapped + +0 string = application/x-archive +0 string \! application/x-archive +>8 string debian application/x-debian-package + +#------------------------------------------------------------------------------ +# +# RPM: file(1) magic for Red Hat Packages Erik Troan (ewt@redhat.com) +# +0 beshort 0xedab +>2 beshort 0xeedb application/x-rpm + +0 lelong&0x8080ffff 0x0000081a application/x-arc lzw +0 lelong&0x8080ffff 0x0000091a application/x-arc squashed +0 lelong&0x8080ffff 0x0000021a application/x-arc uncompressed +0 lelong&0x8080ffff 0x0000031a application/x-arc packed +0 lelong&0x8080ffff 0x0000041a application/x-arc squeezed +0 lelong&0x8080ffff 0x0000061a application/x-arc crunched + +0 leshort 0xea60 application/x-arj + +# LHARC/LHA archiver (Greg Roelofs, newt@uchicago.edu) +2 string -lh0- application/x-lharc lh0 +2 string -lh1- application/x-lharc lh1 +2 string -lz4- application/x-lharc lz4 +2 string -lz5- application/x-lharc lz5 +# [never seen any but the last; -lh4- reported in comp.compression:] +2 string -lzs- application/x-lha lzs +2 string -lh\ - application/x-lha lh +2 string -lhd- application/x-lha lhd +2 string -lh2- application/x-lha lh2 +2 string -lh3- application/x-lha lh3 +2 string -lh4- application/x-lha lh4 +2 string -lh5- application/x-lha lh5 +2 string -lh6- application/x-lha lh6 +2 string -lh7- application/x-lha lh7 +# Shell archives +10 string #\ This\ is\ a\ shell\ archive application/octet-stream x-shell + +#------------------------------------------------------------------------------ +# frame: file(1) magic for FrameMaker files +# +# This stuff came on a FrameMaker demo tape, most of which is +# copyright, but this file is "published" as witness the following: +# +0 string \ +# +0 string/cB \14 byte 12 (OS/2 1.x format) +#>14 byte 64 (OS/2 2.x format) +#>14 byte 40 (Windows 3.x format) +#0 string IC icon +#0 string PI pointer +#0 string CI color icon +#0 string CP color pointer +#0 string BA bitmap array + +# CDROM Filesystems +32769 string CD001 application/x-iso9660 + +# Newer StuffIt archives (grant@netbsd.org) +0 string StuffIt application/x-stuffit +#>162 string >0 : %s + +# BinHex is the Macintosh ASCII-encoded file format (see also "apple") +# Daniel Quinlan, quinlan@yggdrasil.com +11 string must\ be\ converted\ with\ BinHex\ 4 application/mac-binhex40 +##>41 string x \b, version %.3s + + +#------------------------------------------------------------------------------ +# lisp: file(1) magic for lisp programs +# +# various lisp types, from Daniel Quinlan (quinlan@yggdrasil.com) +0 string ;; text/plain 8bit +# Emacs 18 - this is always correct, but not very magical. +0 string \012( application/x-elc +# Emacs 19 +0 string ;ELC\023\000\000\000 application/x-elc + +#------------------------------------------------------------------------------ +# mail.news: file(1) magic for mail and news +# +# There are tests to ascmagic.c to cope with mail and news. +0 string Relay-Version: message/rfc822 7bit +0 string #!\ rnews message/rfc822 7bit +0 string N#!\ rnews message/rfc822 7bit +0 string Forward\ to message/rfc822 7bit +0 string Pipe\ to message/rfc822 7bit +0 string Return-Path: message/rfc822 7bit +0 string Received: message/rfc822 +0 string Path: message/news 8bit +0 string Xref: message/news 8bit +0 string From: message/rfc822 7bit +0 string Article message/news 8bit +#------------------------------------------------------------------------------ +# msword: file(1) magic for MS Word files +# +# Contributor claims: +# Reversed-engineered MS Word magic numbers +# + +0 string \376\067\0\043 application/msword +0 string \320\317\021\340\241\261 application/msword +0 string \333\245-\0\0\0 application/msword + + + +#------------------------------------------------------------------------------ +# printer: file(1) magic for printer-formatted files +# + +# PostScript +0 string %! application/postscript +0 string \004%! application/postscript + +# Acrobat +# (due to clamen@cs.cmu.edu) +0 string %PDF- application/pdf + +#------------------------------------------------------------------------------ +# sc: file(1) magic for "sc" spreadsheet +# +38 string Spreadsheet application/x-sc + +#------------------------------------------------------------------------------ +# tex: file(1) magic for TeX files +# +# XXX - needs byte-endian stuff (big-endian and little-endian DVI?) +# +# From + +# Although we may know the offset of certain text fields in TeX DVI +# and font files, we can't use them reliably because they are not +# zero terminated. [but we do anyway, christos] +0 string \367\002 application/x-dvi +#0 string \367\203 TeX generic font data +#0 string \367\131 TeX packed font data +#0 string \367\312 TeX virtual font data +#0 string This\ is\ TeX, TeX transcript text +#0 string This\ is\ METAFONT, METAFONT transcript text + +# There is no way to detect TeX Font Metric (*.tfm) files without +# breaking them apart and reading the data. The following patterns +# match most *.tfm files generated by METAFONT or afm2tfm. +2 string \000\021 application/x-tex-tfm +2 string \000\022 application/x-tex-tfm +#>34 string >\0 (%s) + +# Texinfo and GNU Info, from Daniel Quinlan (quinlan@yggdrasil.com) +0 string \\input\ texinfo text/x-texinfo +0 string This\ is\ Info\ file text/x-info + +# correct TeX magic for Linux (and maybe more) +# from Peter Tobias (tobias@server.et-inf.fho-emden.de) +# +0 leshort 0x02f7 application/x-dvi + +# RTF - Rich Text Format +0 string {\\rtf text/rtf + +#------------------------------------------------------------------------------ +# animation: file(1) magic for animation/movie formats +# +# animation formats, originally from vax@ccwf.cc.utexas.edu (VaX#n8) +# MPEG file +# MPEG sequences +0 belong 0x000001BA +>4 byte &0x40 video/mp2p +>4 byte ^0x40 video/mpeg +0 belong 0x000001BB video/mpeg +0 belong 0x000001B0 video/mp4v-es +0 belong 0x000001B5 video/mp4v-es +0 belong 0x000001B3 video/mpv +0 belong&0xFF5FFF1F 0x47400010 video/mp2t +0 belong 0x00000001 +>4 byte&0x1F 0x07 video/h264 + +# FLI animation format +0 leshort 0xAF11 video/fli +# FLC animation format +0 leshort 0xAF12 video/flc +# +# SGI and Apple formats +# Added ISO mimes +0 string MOVI video/sgi +4 string moov video/quicktime +4 string mdat video/quicktime +4 string wide video/quicktime +4 string skip video/quicktime +4 string free video/quicktime +4 string idsc image/x-quicktime +4 string idat image/x-quicktime +4 string pckg application/x-quicktime +4 string/B jP image/jp2 +4 string ftyp +>8 string isom video/mp4 +>8 string mp41 video/mp4 +>8 string mp42 video/mp4 +>8 string/B jp2 image/jp2 +>8 string 3gp video/3gpp +>8 string avc1 video/3gpp +>8 string mmp4 video/mp4 +>8 string/B M4A audio/mp4 +>8 string/B qt video/quicktime +# The contributor claims: +# I couldn't find a real magic number for these, however, this +# -appears- to work. Note that it might catch other files, too, +# so BE CAREFUL! +# +# Note that title and author appear in the two 20-byte chunks +# at decimal offsets 2 and 22, respectively, but they are XOR'ed with +# 255 (hex FF)! DL format SUCKS BIG ROCKS. +# +# DL file version 1 , medium format (160x100, 4 images/screen) +0 byte 1 video/unknown +0 byte 2 video/unknown +# +# Databases +# +# GDBM magic numbers +# Will be maintained as part of the GDBM distribution in the future. +# +0 belong 0x13579ace application/x-gdbm +0 lelong 0x13579ace application/x-gdbm +0 string GDBM application/x-gdbm +# +0 belong 0x061561 application/x-dbm +# +# Executables +# +0 string \177ELF +>16 leshort 0 application/octet-stream +>16 leshort 1 application/x-object +>16 leshort 2 application/x-executable +>16 leshort 3 application/x-sharedlib +>16 leshort 4 application/x-coredump +>16 beshort 0 application/octet-stream +>16 beshort 1 application/x-object +>16 beshort 2 application/x-executable +>16 beshort 3 application/x-sharedlib +>16 beshort 4 application/x-coredump +# +# DOS +0 string MZ application/x-dosexec +# +# KDE +0 string [KDE\ Desktop\ Entry] application/x-kdelnk +0 string \#\ KDE\ Config\ File application/x-kdelnk +# xmcd database file for kscd +0 string \#\ xmcd text/xmcd + +#------------------------------------------------------------------------------ +# pkgadd: file(1) magic for SysV R4 PKG Datastreams +# +0 string #\ PaCkAgE\ DaTaStReAm application/x-svr4-package + +#PNG Image Format +0 string \x89PNG image/png + +# MNG Video Format, +0 string \x8aMNG video/x-mng +0 string \x8aJNG video/x-jng + +#------------------------------------------------------------------------------ +# Hierarchical Data Format, used to facilitate scientific data exchange +# specifications at http://hdf.ncsa.uiuc.edu/ +#Hierarchical Data Format (version 4) data +0 belong 0x0e031301 application/x-hdf +#Hierarchical Data Format (version 5) data +0 string \211HDF\r\n\032 application/x-hdf + +# Adobe Photoshop +0 string 8BPS image/x-photoshop + +# Felix von Leitner +0 string d8:announce application/x-bittorrent + + +# lotus 1-2-3 document +0 belong 0x00001a00 application/x-123 +0 belong 0x00000200 application/x-123 + +# MS Access database +4 string Standard\ Jet\ DB application/msaccess + +## magic for XBase files +#0 byte 0x02 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x03 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x04 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x05 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x30 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x43 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x7b +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x83 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x8b +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0x8e +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0xb3 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 byte 0xf5 +#>8 leshort >0 +#>>12 leshort 0 application/x-dbf +# +#0 leshort 0x0006 application/x-dbt + +# Debian has entries for the old PGP formats: +# pgp: file(1) magic for Pretty Good Privacy +# see http://lists.gnupg.org/pipermail/gnupg-devel/1999-September/016052.html +#text/PGP key public ring +0 beshort 0x9900 application/pgp +#text/PGP key security ring +0 beshort 0x9501 application/pgp +#text/PGP key security ring +0 beshort 0x9500 application/pgp +#text/PGP encrypted data +0 beshort 0xa600 application/pgp-encrypted +#text/PGP armored data +##public key block +2 string ---BEGIN\ PGP\ PUBLIC\ KEY\ BLOCK- application/pgp-keys +0 string -----BEGIN\040PGP\40MESSAGE- application/pgp +0 string -----BEGIN\040PGP\40SIGNATURE- application/pgp-signature +# +# GnuPG Magic: +# +# +#text/GnuPG key public ring +0 beshort 0x9901 application/pgp +#text/OpenPGP data +0 beshort 0x8501 application/pgp-encrypted + +# flash: file(1) magic for Macromedia Flash file format +# +# See +# +# http://www.macromedia.com/software/flash/open/ +# +0 string FWS +>3 byte x application/x-shockwave-flash + +# The following paramaters are created for Namazu. +# +# +# 1999/08/13 +#0 string \