Tests update

tests/device/Makefile

@@ -71,7 +71,8 @@ ifneq ("$(NO_RUN)","1")
 			$(RUNNER_DEBUG_FLAG) \
 			-p $(UPLOAD_PORT) \
 			-n $(basename $(notdir $@)) \
-			-o $(LOCAL_BUILD_DIR)/test_result.xml
+			-o $(LOCAL_BUILD_DIR)/test_result.xml \
+			`test -f $(addsuffix .py, $(basename $@)) && echo "-m $(addsuffix .py, $(basename $@))" || echo ""`
 endif
 
 $(BUILD_DIR):

tests/device/libraries/BSTest/mock_decorators.py

@@ -0,0 +1,28 @@
+
+env = dict()
+
+def setup(test_name):
+	global env
+	if not test_name in env:
+		env[test_name] = dict()
+	func_env = env[test_name]
+	def decorator(func):
+		def func_wrapper():
+			return func(env[test_name])
+		func_env['setup'] = func_wrapper
+		return func_wrapper
+	return decorator
+
+
+
+def teardown(test_name):
+	global env
+	if not test_name in env:
+		env[test_name] = dict()
+	func_env = env[test_name]
+	def decorator(func):
+		def func_wrapper():
+			return func(env[test_name])
+		func_env['teardown'] = func_wrapper
+		return func_wrapper
+	return decorator

tests/device/libraries/BSTest/requirements.txt

@@ -1,6 +1,12 @@
+Flask==0.10.1
+itsdangerous==0.24
+Jinja2==2.8
 junit-xml==1.6
+MarkupSafe==0.23
 pexpect==4.0.1
 ptyprocess==0.5.1
 pyserial==3.0.1
+PyYAML==3.11
 six==1.10.0
+Werkzeug==0.11.9
 wheel==0.24.0

tests/device/libraries/BSTest/runner.py

@@ -8,15 +8,19 @@ import time
 import argparse
 import serial
 import subprocess
+import imp
 from urlparse import urlparse
 from junit_xml import TestSuite, TestCase
 try:
     from cStringIO import StringIO
 except:
     from StringIO import StringIO
+import mock_decorators
 
 debug = False
 
+sys.path.append(os.path.abspath(__file__))
+
 def debug_print(*args, **kwargs):
     if not debug:
         return
@@ -29,17 +33,18 @@ class BSTestRunner(object):
     TIMEOUT = 2
     CRASH = 3
 
-    def __init__(self, spawn_obj, name):
+    def __init__(self, spawn_obj, name, mocks):
         self.sp = spawn_obj
         self.tests = []
         self.reset_timeout = 2
         self.name = name
+        self.mocks = mocks
 
     def get_test_list(self):
         self.sp.sendline('-1')
         timeout = 10
         while timeout > 0:
-            res = self.sp.expect(['>>>>>bs_test_menu_begin', EOF])
+            res = self.sp.expect(['>>>>>bs_test_menu_begin', EOF, TIMEOUT])
             if res == 0:
                 break
             timeout-=1
@@ -49,7 +54,7 @@ class BSTestRunner(object):
         while True:
             res = self.sp.expect(['>>>>>bs_test_item id\=(\d+) name\="([^\"]*?)" desc="([^"]*?)"',
                                   '>>>>>bs_test_menu_end',
-                                  EOF])
+                                  EOF, TIMEOUT])
             if res == 0:
                 m = self.sp.match
                 t = {'id': m.group(1), 'name': m.group(2), 'desc': m.group(3)}
@@ -57,7 +62,7 @@ class BSTestRunner(object):
                 debug_print('added test', t)
             elif res == 1:
                 break
-            elif res == 2:
+            elif res >= 2:
                 time.sleep(0.1)
 
         debug_print('got {} tests'.format(len(self.tests)))
@@ -75,8 +80,14 @@ class BSTestRunner(object):
             else:
                 test_output = StringIO()
                 self.sp.logfile = test_output
+                if name in self.mocks:
+                    print('setting up mocks')
+                    self.mocks[name]['setup']()
                 t_start = time.time()
                 result = self.run_test(index)
+                if name in self.mocks:
+                    print('tearing down mocks')
+                    self.mocks[name]['teardown']()
                 t_stop = time.time()
                 self.sp.logfile = None
                 test_case.elapsed_sec = t_stop - t_start
@@ -96,7 +107,7 @@ class BSTestRunner(object):
         self.sp.sendline('{}'.format(index))
         timeout = 10
         while timeout > 0:
-            res = self.sp.expect(['>>>>>bs_test_start', EOF])
+            res = self.sp.expect(['>>>>>bs_test_start', EOF, TIMEOUT])
             if res == 0:
                 break
             time.sleep(0.1)
@@ -147,8 +158,8 @@ def spawn_port(port_name, baudrate=115200):
 def spawn_exec(name):
     return pexpect.spawn(name, timeout=0)
 
-def run_tests(spawn, name):
+def run_tests(spawn, name, mocks):
-    tw = BSTestRunner(spawn, name)
+    tw = BSTestRunner(spawn, name, mocks)
     tw.get_test_list()
     return tw.run_tests()
 
@@ -159,6 +170,7 @@ def parse_args():
     parser.add_argument('-e', '--executable', help='Talk to the test executable')
     parser.add_argument('-n', '--name', help='Test run name')
     parser.add_argument('-o', '--output', help='Output JUnit format test report')
+    parser.add_argument('-m', '--mock', help='Set python script to use for mocking purposes')
     return parser.parse_args()
 
 def main():
@@ -178,8 +190,12 @@ def main():
     if spawn_func is None:
         debug_print("Please specify port or executable", file=sys.stderr)
         return 1
+    mocks = {}
+    if args.mock is not None:
+        mocks_mod = imp.load_source('mocks', args.mock)
+        mocks = mock_decorators.env
     with spawn_func(spawn_arg) as sp:
-        ts = run_tests(sp, name)
+        ts = run_tests(sp, name, mocks)
         if args.output:
             with open(args.output, "w") as f:
                 TestSuite.to_file(f, [ts])

tests/device/test_FS/test_FS.ino

@@ -90,7 +90,7 @@ TEST_CASE("files can be renamed", "[fs]")
     }
 }
 
-TEST_CASE("FS::info works")
+TEST_CASE("FS::info works","[fs]")
 {
     REQUIRE(SPIFFS.begin());
     FSInfo info;

tests/device/test_http_client/server.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIICzzCCAbcCCQCUajf39FoF8jANBgkqhkiG9w0BAQsFADAWMRQwEgYDVQQDDAtj
+YS50bHMudGVzdDAeFw0xNjA0MjIxMTU1MTlaFw0xNzA0MjIxMTU1MTlaMD0xITAf
+BgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEYMBYGA1UEAwwPc2VydmVy
+LnRscy50ZXN0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwhyGSO13
+/ozG0rL6FxDaaxYgMyPVK3hIhTHfKOK/eRZFRDFAhYudz9NbkvUMIF1SToED9FBh
+MbJrkPzCU3cLC1zgnTLseoOS3FKC8xVfJVwsWD8YjMvyQUV/Uo9TAHQSA4SfOB4W
+JjRBMX2GCoWLK5wVzxX+XGd5DnqME4n/CG+Il1t8mB+ACeA1FKwVPTi1wGvDzuCo
+swSEX3J08JB9rP5dix7t+Fezgr6PkvUcnAcu4utAw6f6c0LoHk0SnyXJg1jTdKrl
+J8dIyWQR0cvaxbF+04hvpDepx/62CP+aRs1zcC37eEQ3BVf4phJXbwuXbT7a1CUf
+XQ0cWnJg8mdfcQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBV7ZiwVM8alxPN9vd/
+Ze8EiidSuztkbtZX3mhHfkcJ2Q5c+tZ0Oxy4/g6oISe1+NOQJDMrsARrdWVPmmsb
+ihSHNdS6t0DTcWR5h+i/rnGh+cGiWOJKfhNWvpCVuU6YRZWhYOztL8p6iHoG3ZBi
++zo9GD71FNSRQxMois9sR9q/IgOmkm8CjQgYsv9bQ+Le4mRqfaOCOSoQvsRyLoam
+lNl85gzvVygHYPP9ypiia8btyOHwDSHiV8UhKaERSGKFqznPmTTwTQNXEtQylCXG
+C+13mMGVr49yP4cuYaM8mfL8Rg7Im8Mfa0GXq5PBwEFFYpR9xnbBouQv5erYTQdl
+Oxqk
+-----END CERTIFICATE-----

tests/device/test_http_client/server.csr

@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICgjCCAWoCAQAwPTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRk
+MRgwFgYDVQQDDA9zZXJ2ZXIudGxzLnRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDCHIZI7Xf+jMbSsvoXENprFiAzI9UreEiFMd8o4r95FkVEMUCF
+i53P01uS9QwgXVJOgQP0UGExsmuQ/MJTdwsLXOCdMux6g5LcUoLzFV8lXCxYPxiM
+y/JBRX9Sj1MAdBIDhJ84HhYmNEExfYYKhYsrnBXPFf5cZ3kOeowTif8Ib4iXW3yY
+H4AJ4DUUrBU9OLXAa8PO4KizBIRfcnTwkH2s/l2LHu34V7OCvo+S9RycBy7i60DD
+p/pzQugeTRKfJcmDWNN0quUnx0jJZBHRy9rFsX7TiG+kN6nH/rYI/5pGzXNwLft4
+RDcFV/imEldvC5dtPtrUJR9dDRxacmDyZ19xAgMBAAGgADANBgkqhkiG9w0BAQsF
+AAOCAQEAE2xZjAkmVr7/p7LkV3UV3Y0wzeVP7kDBN7VnhNmEMw8xTKwXKsc6z5pY
+fa4/lmMMCIJ04dUPIgPC2qiYXJ1AXevLm0A7Blpay6HJilw51NMjhF6SLUkShl6e
+k3zyj9LnA5TxijsTrFy/km7qIoo6l7sR7+DwTOIlO/Sj/SDNJn+GAVL153zjxYCy
+eBYdEz07kxkeONLUjAW2bV+TkliqTdM6meRpf9GQGQksQrkHrudl7JyyKZTf2m6Z
+U33AnUp2SRIXESJmst824LKwkjLMYiXgxRK8ZSSP7iiNBj71DK1vQOZQwEZ2cCjW
+/pQxKJnHnRvZ3rityuShtHTUF+C39w==
+-----END CERTIFICATE REQUEST-----

tests/device/test_http_client/server.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAwhyGSO13/ozG0rL6FxDaaxYgMyPVK3hIhTHfKOK/eRZFRDFA
+hYudz9NbkvUMIF1SToED9FBhMbJrkPzCU3cLC1zgnTLseoOS3FKC8xVfJVwsWD8Y
+jMvyQUV/Uo9TAHQSA4SfOB4WJjRBMX2GCoWLK5wVzxX+XGd5DnqME4n/CG+Il1t8
+mB+ACeA1FKwVPTi1wGvDzuCoswSEX3J08JB9rP5dix7t+Fezgr6PkvUcnAcu4utA
+w6f6c0LoHk0SnyXJg1jTdKrlJ8dIyWQR0cvaxbF+04hvpDepx/62CP+aRs1zcC37
+eEQ3BVf4phJXbwuXbT7a1CUfXQ0cWnJg8mdfcQIDAQABAoIBAQCKfl2VFNEjbf/B
+fu8Om/iqpOuucSd7z2M2nSGSg02HsEsVX2qgnb+n8y4ICQxw3wSFfGl83Aiss5cp
+qB1h/bKEleywXA/7TXrv9XL2ys3K0xvUjpgOhMjApzwzhIz86tObbPlIybaV5XyK
+Ofvz79oe8EtjKFcGSNFHckoU+sValzH/gyaeIFuPFlJdsW9MOgN5lcyLDtddg2lr
+Z9k+OEEob9BC1dDmc5hxtnClfyiXW1ugjZovOBv79Mq2N8G/V8JZaTHxzEFdOGy3
+lUQ1B/s2xPDejQYGddc3N5+wF4ZuGHAYOmcv8IRaMvLdBMPmVnKdgDJeY4HheJY0
+z0t4BEYtAoGBAPL1exWZputT8GitBxTJcJB1crMzxvlSplS5mGxM36GgZxf2ql72
+8Ufa1hCAQLP7keKPWWl26Bt6Bm4uv/jsB/F8ezMloPvwwQ3nQrN8QY4ihtLLFcvK
+OMaErweMEVvQ6GdQHPGow5RqAkYj/X6jJjoAI88uV1yOx+TucmDneEnbAoGBAMyH
+04ZR3VNvzHSjDx2wo48FP7Cs/uCynOEd/bRb1I8n+fazIXIlWdWX2drFr2YRM+Uy
+UzAadgqFOu3Bvuk64UKwAlxX9jCIm1Ed5O1MUGPj+6OnW1oq/eYIbJ9RlFDyIEBv
+Ic+jV/zxojgpP38jpTzn2oC5jWWS+CjHpZNThtujAoGBAOquRB4xNRLLGcWCnPxv
+N9PSHuEKeoAKXbApNhomhz1P+0Uidp8UWSvXLj7yI422ysvVO8Crorgnvl5fuf9v
+vpx8aWSWTFIP0+riH4PP2mK45xJmKL+Yrg7Ty1225m2R9WsV719ebMzHOTsXOJ1C
+aoJL8EsHsEsvf9aanNENxRtRAoGAVIP34zQm2diDytqqX2FjZLENjWse8yi7bMag
+1ItxvSoOv2Nr+af3hCx4aE9x2CJZqGbwOxtkFZrSK/b4dZXQCWeDwjbS02FPlOhe
+dbQoL+7AR/La69qCCjEG+ZqTSBOVQirp9MwRisMqfjyFMXtAR3ejMbf69rMReoBt
+KgDE5DMCgYAkcBd7XyPjR6t09BXAcz8n5J8QCsoY4taNVeCmCCkl8rUt9y8tjQah
+5NtBk6i834p5/GbGhI4en/sisVUWzpOgc9G80lUAFQSR3weR6lOL1T+/cy691GqO
+wKAWbpOH01fJYmnbUjC7TC2idx033ZA9vJq/s9sNfG2CS2w3egcxag==
+-----END RSA PRIVATE KEY-----

tests/device/test_http_client/test_http_client.ino

@@ -10,6 +10,7 @@ BS_ENV_DECLARE();
 void setup()
 {
     Serial.begin(115200);
+    Serial.setDebugOutput(true);
     WiFi.persistent(false);
     WiFi.begin(STA_SSID, STA_PASS);
     while (WiFi.status() != WL_CONNECTED) {
@@ -18,121 +19,117 @@ void setup()
     BS_RUN(Serial);
 }
 
-// json doesn't allow newlines within a string
+const char* fp = "40 A3 6C E3 8A DF A2 D4 13 B0 32 5C 87 44 54 28 0B CE C5 A4";
-auto test_payload = (const __FlashStringHelper *) "\"\
-0000000 53 48 45 4c 4c 20 3a 3d 20 2f 62 69 6e 2f 62 61\\n\
-0000010 73 68 0a 56 20 3f 3d 20 30 0a 54 45 53 54 5f 4c\\n\
-0000020 49 53 54 20 3f 3d 20 24 28 77 69 6c 64 63 61 72\\n\
-0000030 64 20 74 65 73 74 5f 2a 2f 2a 2e 69 6e 6f 29 0a\\n\
-0000040 45 53 50 38 32 36 36 5f 43 4f 52 45 5f 50 41 54\\n\
-0000050 48 20 3f 3d 20 2e 2e 2f 2e 2e 0a 42 55 49 4c 44\\n\
-0000060 5f 44 49 52 20 3f 3d 20 24 28 50 57 44 29 2f 2e\\n\
-0000070 62 75 69 6c 64 0a 48 41 52 44 57 41 52 45 5f 44\\n\
-0000080 49 52 20 3f 3d 20 24 28 50 57 44 29 2f 2e 68 61\\n\
-0000090 72 64 77 61 72 65 0a 45 53 50 54 4f 4f 4c 20 3f\\n\
-00000a0 3d 20 24 28 45 53 50 38 32 36 36 5f 43 4f 52 45\\n\
-123\"";
-
-auto bin_template_1 = (const __FlashStringHelper *) R"({
-  "status": 200,
-  "statusText": "OK",
-  "httpVersion": "HTTP/1.1",
-  "headers": [],
-  "cookies": [],
-  "content": {
-    "mimeType": "text/plain",
-    "text":
-)";
-
-auto bin_template_2 = (const __FlashStringHelper *) R"(
-  }
-})";
-
-const char *mockbin_fingerprint = "F0 C1 7B 6A 89 4F AA 67 86 15 4A 17 6C 25 52 8E 2F F6 0F 34";
-
-String createBin(bool https)
-{
-    String payload;
-    payload.reserve(
-        strlen_P((PGM_P) bin_template_1) + 
-        strlen_P((PGM_P) test_payload) + 
-        strlen_P((PGM_P) bin_template_2) + 1);
-    payload += bin_template_1;
-    payload += test_payload;
-    payload += bin_template_2;
-
-    HTTPClient http;
-    if (https) {
-        http.begin("https://mockbin.org/bin/create", mockbin_fingerprint);
-    }
-    else {
-        http.begin("http://mockbin.org/bin/create");
-    }
-    const char* location = "Location";
-    http.collectHeaders(&location, 1);
-    http.addHeader("Content-Type", "application/json");
-    auto result = http.POST(payload);
-    Serial.println(result);
-    Serial.println("----");
-    Serial.println(http.getString());
-    Serial.println("----");
-    REQUIRE(result == 201);
-    String url = "http://mockbin.org" + http.header(location);
-    http.end();
-    return url;
-}
 
 TEST_CASE("HTTP GET request", "[HTTPClient]")
 {
-    const int repeatCount = 10;
+    {
-
+        // small request
-    String url = createBin(false);
-    int heapBefore = ESP.getFreeHeap();
-    for (int i = 0; i < repeatCount; ++i) {
         HTTPClient http;
-        http.begin(url);
+        http.begin(SERVER_IP, 8088, "/");
         auto httpCode = http.GET();
         REQUIRE(httpCode == HTTP_CODE_OK);
         String payload = http.getString();
-        payload.replace("\n", "\\n");
+        REQUIRE(payload == "hello!!!");
-        String quotedPayload;
-        quotedPayload.reserve(payload.length() + 3);
-        quotedPayload += "\"";
-        quotedPayload += payload;
-        quotedPayload += "\"";
-        Serial.println("----payload:");
-        Serial.println(quotedPayload);
-        Serial.println("----");
-        Serial.println("----test_payload:");
-        Serial.println(test_payload);
-        Serial.println("----");
-        CHECK(quotedPayload == test_payload);
-        http.end();
-        delay(100);
     }
-    int heapAfter = ESP.getFreeHeap();
+    {
-    CHECK(heapBefore - heapAfter <= 8);
+        // request which returns 8000 bytes
+        HTTPClient http;
+        http.begin(SERVER_IP, 8088, "/data?size=8000");
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == HTTP_CODE_OK);
+        String payload = http.getString();
+        auto len = payload.length();
+        REQUIRE(len == 8000);
+        for (int i = 0; i < len; ++i) {
+            if (payload[i] != 'a') {
+                REQUIRE(false);
+            }
+        }
+    }
+
 }
 
+
 TEST_CASE("HTTPS GET request", "[HTTPClient]")
 {
-    const int repeatCount = 10;
+    {
-
+        // small request
-    String url = createBin(true);
-    int heapBefore = ESP.getFreeHeap();
-    for (int i = 0; i < repeatCount; ++i) {
         HTTPClient http;
-        http.begin(url, mockbin_fingerprint);
+        http.begin(SERVER_IP, 8088, "/", fp);
         auto httpCode = http.GET();
         REQUIRE(httpCode == HTTP_CODE_OK);
         String payload = http.getString();
-        CHECK(payload == test_payload);
+        REQUIRE(payload == "hello!!!");
-        http.end();
-        delay(100);
     }
-    int heapAfter = ESP.getFreeHeap();
+    {
-    CHECK(heapBefore - heapAfter <= 8);
+        // request which returns 8000 bytes
+        HTTPClient http;
+        http.begin(SERVER_IP, 8088, "/data?size=8000", fp);
+        auto httpCode = http.GET();
+        REQUIRE(httpCode == HTTP_CODE_OK);
+        String payload = http.getString();
+        auto len = payload.length();
+        REQUIRE(len == 8000);
+        for (int i = 0; i < len; ++i) {
+            if (payload[i] != 'a') {
+                REQUIRE(false);
+            }
+        }
+    }
+
 }
+// TEST_CASE("HTTP GET request", "[HTTPClient]")
+// {
+//     const int repeatCount = 10;
+
+//     String url = createBin(false);
+//     int heapBefore = ESP.getFreeHeap();
+//     for (int i = 0; i < repeatCount; ++i) {
+//         HTTPClient http;
+//         http.begin(url);
+//         auto httpCode = http.GET();
+//         REQUIRE(httpCode == HTTP_CODE_OK);
+//         String payload = http.getString();
+//         payload.replace("\n", "\\n");
+//         String quotedPayload;
+//         quotedPayload.reserve(payload.length() + 3);
+//         quotedPayload += "\"";
+//         quotedPayload += payload;
+//         quotedPayload += "\"";
+//         Serial.println("----payload:");
+//         Serial.println(quotedPayload);
+//         Serial.println("----");
+//         Serial.println("----test_payload:");
+//         Serial.println(test_payload);
+//         Serial.println("----");
+//         CHECK(quotedPayload == test_payload);
+//         http.end();
+//         delay(100);
+//     }
+//     int heapAfter = ESP.getFreeHeap();
+//     CHECK(heapBefore - heapAfter <= 8);
+// }
+
+// TEST_CASE("HTTPS GET request", "[HTTPClient]")
+// {
+//     const int repeatCount = 10;
+
+//     String url = createBin(true);
+//     int heapBefore = ESP.getFreeHeap();
+//     for (int i = 0; i < repeatCount; ++i) {
+//         HTTPClient http;
+//         http.begin(url, mockbin_fingerprint);
+//         auto httpCode = http.GET();
+//         REQUIRE(httpCode == HTTP_CODE_OK);
+//         String payload = http.getString();
+//         CHECK(payload == test_payload);
+//         http.end();
+//         delay(100);
+//     }
+//     int heapAfter = ESP.getFreeHeap();
+//     CHECK(heapBefore - heapAfter <= 8);
+// }
 
 void loop()
 {