From ab7af890021a8a78ad703baf62748ba0a5ca4d4e Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Thu, 10 Mar 2016 09:12:11 +0300 Subject: [PATCH] Device side test library and test runner --- tests/.gitignore | 1 - tests/FSWrapper/FSWrapper.ino | 155 ------------- tests/Time/Time.ino | 35 --- tests/device/.gitignore | 2 + tests/device/Makefile | 98 ++++++++ tests/device/libraries/BSTest/.gitignore | 3 + tests/device/libraries/BSTest/Makefile | 22 ++ .../libraries/BSTest/library.properties | 9 + .../device/libraries/BSTest/requirements.txt | 6 + tests/device/libraries/BSTest/runner.py | 189 +++++++++++++++ tests/device/libraries/BSTest/src/BSArduino.h | 57 +++++ .../device/libraries/BSTest/src/BSProtocol.h | 55 +++++ tests/device/libraries/BSTest/src/BSStdio.h | 39 ++++ tests/device/libraries/BSTest/src/BSTest.h | 217 ++++++++++++++++++ tests/device/libraries/BSTest/test/test.cpp | 51 ++++ tests/device/libraries/test_config/.gitignore | 1 + .../libraries/test_config/library.properties | 9 + .../test_config/test_config.h.template | 8 + tests/device/test_FS/test_FS.ino | 136 +++++++++++ .../test_Print_printf/test_Print_printf.ino | 38 +++ .../test_http_client/test_http_client.ino | 35 +++ .../device/test_overrides/test_overrides.ino | 38 +++ tests/device/test_tests/test_tests.ino | 47 ++++ tests/device/test_time/test_time.ino | 46 ++++ .../test_umm_malloc/test_umm_malloc.ino | 16 +- tests/test_overrides/test_overrides.ino | 11 - 26 files changed, 1118 insertions(+), 206 deletions(-) delete mode 100644 tests/FSWrapper/FSWrapper.ino delete mode 100644 tests/Time/Time.ino create mode 100644 tests/device/.gitignore create mode 100644 tests/device/Makefile create mode 100644 tests/device/libraries/BSTest/.gitignore create mode 100644 tests/device/libraries/BSTest/Makefile create mode 100644 tests/device/libraries/BSTest/library.properties create mode 100644 tests/device/libraries/BSTest/requirements.txt create mode 100644 tests/device/libraries/BSTest/runner.py create mode 100644 tests/device/libraries/BSTest/src/BSArduino.h create mode 100644 tests/device/libraries/BSTest/src/BSProtocol.h create mode 100644 tests/device/libraries/BSTest/src/BSStdio.h create mode 100644 tests/device/libraries/BSTest/src/BSTest.h create mode 100644 tests/device/libraries/BSTest/test/test.cpp create mode 100644 tests/device/libraries/test_config/.gitignore create mode 100644 tests/device/libraries/test_config/library.properties create mode 100644 tests/device/libraries/test_config/test_config.h.template create mode 100644 tests/device/test_FS/test_FS.ino create mode 100644 tests/device/test_Print_printf/test_Print_printf.ino create mode 100644 tests/device/test_http_client/test_http_client.ino create mode 100644 tests/device/test_overrides/test_overrides.ino create mode 100644 tests/device/test_tests/test_tests.ino create mode 100644 tests/device/test_time/test_time.ino rename tests/{ => device}/test_umm_malloc/test_umm_malloc.ino (51%) delete mode 100644 tests/test_overrides/test_overrides.ino diff --git a/tests/.gitignore b/tests/.gitignore index 7bce08d60..2a80ca891 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,2 @@ hardware tmp -.env diff --git a/tests/FSWrapper/FSWrapper.ino b/tests/FSWrapper/FSWrapper.ino deleted file mode 100644 index 099065f4f..000000000 --- a/tests/FSWrapper/FSWrapper.ino +++ /dev/null @@ -1,155 +0,0 @@ -#include -#include "FS.h" - - -void fail(const char* msg) { - Serial.println(msg); - while (true) { - yield(); - } -} - -void setup() { - Serial.begin(115200); - Serial.setDebugOutput(true); - WiFi.mode(WIFI_OFF); - Serial.println("\n\nFS test\n"); - - { - if (!SPIFFS.format()) { - fail("format failed"); - } - Dir root = SPIFFS.openDir("/"); - int count = 0; - while (root.next()) { - ++count; - } - if (count > 0) { - fail("some files left after format"); - } - } - - - if (!SPIFFS.begin()) { - fail("SPIFFS init failed"); - } - - String text = "write test"; - { - File out = SPIFFS.open("/tmp.txt", "w"); - if (!out) { - fail("failed to open tmp.txt for writing"); - } - out.print(text); - } - - { - File in = SPIFFS.open("/tmp.txt", "r"); - if (!in) { - fail("failed to open tmp.txt for reading"); - } - Serial.printf("size=%d\r\n", in.size()); - if (in.size() != text.length()) { - fail("invalid size of tmp.txt"); - } - Serial.print("Reading data: "); - in.setTimeout(0); - String result = in.readString(); - Serial.println(result); - if (result != text) { - fail("invalid data in tmp.txt"); - } - } - - { - for (int i = 0; i < 10; ++i) { - String name = "seq_"; - name += i; - name += ".txt"; - - File out = SPIFFS.open(name, "w"); - if (!out) { - fail("can't open seq_ file"); - } - - out.println(i); - } - } - { - Dir root = SPIFFS.openDir("/"); - while (root.next()) { - String fileName = root.fileName(); - File f = root.openFile("r"); - Serial.printf("%s: %d\r\n", fileName.c_str(), f.size()); - } - } - - { - Dir root = SPIFFS.openDir("/"); - while (root.next()) { - String fileName = root.fileName(); - Serial.print("deleting "); - Serial.println(fileName); - if (!SPIFFS.remove(fileName)) { - fail("remove failed"); - } - } - } - - { - File tmp = SPIFFS.open("/tmp1.txt", "w"); - tmp.println("rename test"); - } - - { - if (!SPIFFS.rename("/tmp1.txt", "/tmp2.txt")) { - fail("rename failed"); - } - File tmp2 = SPIFFS.open("/tmp2.txt", "r"); - if (!tmp2) { - fail("open tmp2 failed"); - } - } - - { - FSInfo info; - if (!SPIFFS.info(info)) { - fail("info failed"); - } - Serial.printf("Total: %u\nUsed: %u\nBlock: %u\nPage: %u\nMax open files: %u\nMax path len: %u\n", - info.totalBytes, - info.usedBytes, - info.blockSize, - info.pageSize, - info.maxOpenFiles, - info.maxPathLength - ); - } - - { - if (!SPIFFS.format()) { - fail("format failed"); - } - Dir root = SPIFFS.openDir("/"); - int count = 0; - while (root.next()) { - ++count; - } - if (count > 0) { - fail("some files left after format"); - } - } - { - File tmp = SPIFFS.open("/tmp.txt", "w"); - } - { - File tmp = SPIFFS.open("/tmp.txt", "w"); - if (!tmp) { - fail("failed to re-open empty file"); - } - } - Serial.println("success"); -} - -void loop() { -} diff --git a/tests/Time/Time.ino b/tests/Time/Time.ino deleted file mode 100644 index e4078534a..000000000 --- a/tests/Time/Time.ino +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - -const char* ssid = ".........."; -const char* password = ".........."; - -int timezone = 3; -int dst = 0; - -void setup() { - Serial.begin(115200); - Serial.setDebugOutput(true); - - WiFi.mode(WIFI_STA); - WiFi.begin(ssid, password); - Serial.println("\nConnecting to WiFi"); - while (WiFi.status() != WL_CONNECTED) { - Serial.print("."); - delay(1000); - } - - configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov"); - Serial.println("\nWaiting for time"); - while (!time(nullptr)) { - Serial.print("."); - delay(1000); - } - Serial.println(""); -} - -void loop() { - time_t now = time(nullptr); - Serial.println(ctime(&now)); - delay(1000); -} diff --git a/tests/device/.gitignore b/tests/device/.gitignore new file mode 100644 index 000000000..c1ffd0931 --- /dev/null +++ b/tests/device/.gitignore @@ -0,0 +1,2 @@ +.build +.hardware diff --git a/tests/device/Makefile b/tests/device/Makefile new file mode 100644 index 000000000..e21526e1f --- /dev/null +++ b/tests/device/Makefile @@ -0,0 +1,98 @@ +SHELL := /bin/bash +V ?= 0 +TEST_LIST ?= $(wildcard test_*/*.ino) +ESP8266_CORE_PATH ?= ../.. +BUILD_DIR ?= $(PWD)/.build +HARDWARE_DIR ?= $(PWD)/.hardware +ESPTOOL ?= $(ESP8266_CORE_PATH)/tools/esptool/esptool +UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB) +UPLOAD_BAUD ?= 921600 +UPLOAD_BOARD ?= nodemcu +BS_DIR ?= libraries/BSTest +DEBUG_LEVEL ?= DebugLevel=None____ +FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=DIO,UploadSpeed=115200,FlashSize=4M1M,ResetMethod=none,Debug=Serial$(DEBUG_LEVEL) +BUILD_TOOL = $(ARDUINO_IDE_PATH)/arduino-builder +TEST_CONFIG = libraries/test_config/test_config.h + +ifeq ("$(UPLOAD_PORT)","") +$(error "Failed to detect upload port, please export UPLOAD_PORT manually") +endif + +ifeq ("$(ARDUINO_IDE_PATH)","") +$(error "Please export ARDUINO_IDE_PATH") +endif + +ifneq ("$(V)","1") + SILENT = @ +else + BUILDER_DEBUG_FLAG = -verbose + RUNNER_DEBUG_FLAG = -d + UPLOAD_VERBOSE_FLAG = -v +endif + + +all: count tests + +count: + @echo Running $(words $(TEST_LIST)) tests + +tests: $(BUILD_DIR) $(HARDWARE_DIR) virtualenv $(TEST_CONFIG) $(TEST_LIST) + +$(TEST_LIST): LOCAL_BUILD_DIR=$(BUILD_DIR)/$(notdir $@) + +$(TEST_LIST): + $(SILENT)mkdir -p $(LOCAL_BUILD_DIR) +ifneq ("$(NO_BUILD)","1") + @echo Compiling $(notdir $@) + $(SILENT)$(BUILD_TOOL) -compile -logger=human \ + -libraries "$(PWD)/libraries" \ + -core-api-version="10608" \ + -warnings=none \ + $(BUILDER_DEBUG_FLAG) \ + -build-path $(LOCAL_BUILD_DIR) \ + -tools $(ARDUINO_IDE_PATH)/tools-builder \ + -hardware $(HARDWARE_DIR)\ + -hardware $(ARDUINO_IDE_PATH)/hardware \ + -fqbn=$(FQBN) \ + $@ +endif +ifneq ("$(NO_UPLOAD)","1") + $(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \ + -cp $(UPLOAD_PORT) \ + -cb $(UPLOAD_BAUD) \ + -cd $(UPLOAD_BOARD) \ + -cf $(LOCAL_BUILD_DIR)/$(notdir $@).bin +endif +ifneq ("$(NO_RUN)","1") + @echo Running tests + $(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) -cp $(UPLOAD_PORT) -cd $(UPLOAD_BOARD) -cr + @source $(BS_DIR)/virtualenv/bin/activate && \ + python $(BS_DIR)/runner.py \ + $(RUNNER_DEBUG_FLAG) \ + -p $(UPLOAD_PORT) \ + -n $(basename $(notdir $@)) \ + -o $(LOCAL_BUILD_DIR)/test_result.xml +endif + +$(BUILD_DIR): + mkdir -p $(BUILD_DIR) + +$(HARDWARE_DIR): + mkdir -p $(HARDWARE_DIR)/esp8266com + cd $(HARDWARE_DIR)/esp8266com && ln -s $(realpath $(ESP8266_CORE_PATH)) esp8266 + +virtualenv: + make -C $(BS_DIR) virtualenv + +clean: + rm -rf $(BUILD_DIR) + rm -rf $(HARDWARE_DIR) + +$(TEST_CONFIG): + @echo "****** " + @echo "****** libraries/test_config/test_config.h does not exist" + @echo "****** Create one from libraries/test_config/test_config.h.template" + @echo "****** " + false + +.PHONY: tests all count venv $(BUILD_DIR) $(TEST_LIST) diff --git a/tests/device/libraries/BSTest/.gitignore b/tests/device/libraries/BSTest/.gitignore new file mode 100644 index 000000000..2f472a04d --- /dev/null +++ b/tests/device/libraries/BSTest/.gitignore @@ -0,0 +1,3 @@ +test/test +virtualenv + diff --git a/tests/device/libraries/BSTest/Makefile b/tests/device/libraries/BSTest/Makefile new file mode 100644 index 000000000..b638bb4fe --- /dev/null +++ b/tests/device/libraries/BSTest/Makefile @@ -0,0 +1,22 @@ +PYTHON_ENV_DIR=virtualenv +TEST_EXECUTABLE=test/test + +all: test + +install: $(PYTHON_ENV_DIR) + +clean: + rm -rf $(PYTHON_ENV_DIR) + rm -rf $(TEST_EXECUTABLE) + +$(PYTHON_ENV_DIR): + virtualenv --no-site-packages $(PYTHON_ENV_DIR) + source $(PYTHON_ENV_DIR)/bin/activate && pip install -r requirements.txt + +test: $(TEST_EXECUTABLE) $(PYTHON_ENV_DIR) + source $(PYTHON_ENV_DIR)/bin/activate && python runner.py -e $(TEST_EXECUTABLE) + +$(TEST_EXECUTABLE): test/test.cpp + g++ -std=c++11 -Isrc -o $@ test/test.cpp + +.PHONY: test clean install all diff --git a/tests/device/libraries/BSTest/library.properties b/tests/device/libraries/BSTest/library.properties new file mode 100644 index 000000000..cd6591349 --- /dev/null +++ b/tests/device/libraries/BSTest/library.properties @@ -0,0 +1,9 @@ +name=BSTest +version=0.1 +author=Ivan Grokhotkov +maintainer=Ivan Grokhotkov +sentence=BS Test library +paragraph= +category=Uncategorized +url= +architectures=esp8266 diff --git a/tests/device/libraries/BSTest/requirements.txt b/tests/device/libraries/BSTest/requirements.txt new file mode 100644 index 000000000..efec81018 --- /dev/null +++ b/tests/device/libraries/BSTest/requirements.txt @@ -0,0 +1,6 @@ +junit-xml==1.6 +pexpect==4.0.1 +ptyprocess==0.5.1 +pyserial==3.0.1 +six==1.10.0 +wheel==0.24.0 \ No newline at end of file diff --git a/tests/device/libraries/BSTest/runner.py b/tests/device/libraries/BSTest/runner.py new file mode 100644 index 000000000..849cdea6e --- /dev/null +++ b/tests/device/libraries/BSTest/runner.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python +from __future__ import print_function +import pexpect +from pexpect import EOF, TIMEOUT, fdpexpect +import sys +import os +import time +import argparse +import serial +import subprocess +from urlparse import urlparse +from junit_xml import TestSuite, TestCase +try: + from cStringIO import StringIO +except: + from StringIO import StringIO + +debug = False + +def debug_print(*args, **kwargs): + if not debug: + return + print(file=sys.stderr, *args, **kwargs) + +class BSTestRunner(object): + + SUCCESS = 0 + FAIL = 1 + TIMEOUT = 2 + CRASH = 3 + + def __init__(self, spawn_obj, name): + self.sp = spawn_obj + self.tests = [] + self.reset_timeout = 2 + self.name = name + + def get_test_list(self): + self.sp.sendline('-1') + timeout = 10 + while timeout > 0: + res = self.sp.expect(['>>>>>bs_test_menu_begin', EOF]) + if res == 0: + break + timeout-=1 + time.sleep(0.1) + debug_print('got begin') + self.tests = [] + while True: + res = self.sp.expect(['>>>>>bs_test_item id\=(\d+) name\="([^\"]*?)" desc="([^"]*?)"', + '>>>>>bs_test_menu_end', + EOF]) + if res == 0: + m = self.sp.match + t = {'id': m.group(1), 'name': m.group(2), 'desc': m.group(3)} + self.tests.append(t) + debug_print('added test', t) + elif res == 1: + break + elif res == 2: + time.sleep(0.1) + + debug_print('got {} tests'.format(len(self.tests))) + + def run_tests(self): + test_cases = [] + for test in self.tests: + desc = test['desc'] + name = test['name'] + index = test['id'] + test_case = TestCase(name, self.name) + if '[.]' in desc: + print('skipping test "{}"'.format(name)) + test_case.add_skipped_info(message="Skipped test marked with [.]") + else: + test_output = StringIO() + self.sp.logfile = test_output + t_start = time.time() + result = self.run_test(index) + t_stop = time.time() + self.sp.logfile = None + test_case.elapsed_sec = t_stop - t_start + debug_print('test output was:') + debug_print(test_output.getvalue()) + if result == BSTestRunner.SUCCESS: + test_case.stdout = test_output.getvalue() + print('test "{}" passed'.format(name)) + else: + print('test "{}" failed'.format(name)) + test_case.add_failure_info('Test failed', output=test_output.getvalue()) + test_output.close() + test_cases += [test_case]; + return TestSuite(self.name, test_cases) + + def run_test(self, index): + self.sp.sendline('{}'.format(index)) + timeout = 10 + while timeout > 0: + res = self.sp.expect(['>>>>>bs_test_start', EOF]) + if res == 0: + break + time.sleep(0.1) + timeout -= 0.1 + if timeout <= 0: + raise 'test begin timeout' + while timeout > 0: + res = self.sp.expect(['>>>>>bs_test_check_failure line=(\d+)', + '>>>>>bs_test_end line=(\d+) result=(\d+) checks=(\d+) failed_checks=(\d+)', + TIMEOUT, + EOF, + 'Exception', + 'ets Jan 8 2013', + 'wdt reset']) + if res == 0: + continue + elif res == 1: + test_result = self.sp.match.group(2) + if test_result == '1': + return BSTestRunner.SUCCESS + else: + if self.sp.match.group(1) != '0': + time.sleep(1.0) + self.sp.expect([TIMEOUT, + 'wdt reset', + 'Exception', + 'Panic', + 'Abort', + 'Soft WDT', + EOF], timeout=self.reset_timeout) + return BSTestRunner.FAIL + elif res == 2 or res == 3: + time.sleep(0.1) + timeout -= 0.1 + continue + elif res > 3: + return BSTestRunner.CRASH + if timeout <= 0: + return BSTestRunner.TIMEOUT + +ser = None + +def spawn_port(port_name, baudrate=115200): + global ser + ser = serial.serial_for_url(port_name, baudrate=baudrate) + return fdpexpect.fdspawn(ser, 'wb', timeout=0) + +def spawn_exec(name): + return pexpect.spawn(name, timeout=0) + +def run_tests(spawn, name): + tw = BSTestRunner(spawn, name) + tw.get_test_list() + return tw.run_tests() + +def parse_args(): + parser = argparse.ArgumentParser(description='BS test runner') + parser.add_argument('-d', '--debug', help='Send test output to stderr', action='store_true') + parser.add_argument('-p', '--port', help='Talk to the test over serial') + parser.add_argument('-e', '--executable', help='Talk to the test executable') + parser.add_argument('-n', '--name', help='Test run name') + parser.add_argument('-o', '--output', help='Output JUnit format test report') + return parser.parse_args() + +def main(): + args = parse_args() + spawn_func = None + spawn_arg = None + if args.port is not None: + spawn_func = spawn_port + spawn_arg = args.port + elif args.executable is not None: + spawn_func = spawn_exec + spawn_arg = args.executable + name = args.name or "" + global debug + if args.debug: + debug = True + if spawn_func is None: + debug_print("Please specify port or executable", file=sys.stderr) + return 1 + with spawn_func(spawn_arg) as sp: + ts = run_tests(sp, name) + if args.output: + with open(args.output, "w") as f: + TestSuite.to_file(f, [ts]) + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/device/libraries/BSTest/src/BSArduino.h b/tests/device/libraries/BSTest/src/BSArduino.h new file mode 100644 index 000000000..ff53ca19a --- /dev/null +++ b/tests/device/libraries/BSTest/src/BSArduino.h @@ -0,0 +1,57 @@ +#ifndef BS_ARDUINO_H +#define BS_ARDUINO_H + +#include +namespace bs +{ +class ArduinoIOHelper +{ +public: + ArduinoIOHelper(Stream& stream) : m_stream(stream) + { + } + + size_t printf(const char *format, ...) + { + va_list arg; + va_start(arg, format); + char temp[128]; + char* buffer = temp; + size_t len = vsnprintf(temp, sizeof(temp), format, arg); + va_end(arg); + if (len > sizeof(temp) - 1) { + buffer = new char[len + 1]; + if (!buffer) { + return 0; + } + va_start(arg, format); + ets_vsnprintf(buffer, len + 1, format, arg); + va_end(arg); + } + len = m_stream.write((const uint8_t*) buffer, len); + if (buffer != temp) { + delete[] buffer; + } + return len; + } + + bool read_int(int& result) + { + // TODO: fix this for 0 value + result = m_stream.parseInt(); + return result != 0; + } + +protected: + Stream& m_stream; +}; + +typedef ArduinoIOHelper IOHelper; + +inline void fatal() { + ESP.restart(); +} + +} // namespace bs + +#endif //BS_ARDUINO_H diff --git a/tests/device/libraries/BSTest/src/BSProtocol.h b/tests/device/libraries/BSTest/src/BSProtocol.h new file mode 100644 index 000000000..54d7bcc0e --- /dev/null +++ b/tests/device/libraries/BSTest/src/BSProtocol.h @@ -0,0 +1,55 @@ +#ifndef BS_PROTOCOL_H +#define BS_PROTOCOL_H + +#define BS_LINE_PREFIX ">>>>>bs_test_" + +namespace bs +{ +namespace protocol +{ +template +void output_test_start(IO& io, const char* file, size_t line, const char* name, const char* desc) +{ + io.printf(BS_LINE_PREFIX "start file=\"%s\" line=%d name=\"%s\" desc=\"%s\"\n", file, line, name, desc); +} + +template +void output_check_failure(IO& io, size_t line) +{ + io.printf(BS_LINE_PREFIX "check_failure line=%d\n", line); +} + +template +void output_test_end(IO& io, bool success, size_t checks, size_t failed_checks, size_t line=0) +{ + io.printf(BS_LINE_PREFIX "end line=%d result=%d checks=%d failed_checks=%d\n", line, success, checks, failed_checks); +} + +template +void output_menu_begin(IO& io) +{ + io.printf(BS_LINE_PREFIX "menu_begin\n"); +} + +template +void output_menu_item(IO& io, int index, const char* name, const char* desc) +{ + io.printf(BS_LINE_PREFIX "item id=%d name=\"%s\" desc=\"%s\"\n", index, name, desc); +} + +template +void output_menu_end(IO& io) +{ + io.printf(BS_LINE_PREFIX "menu_end\n"); +} + +template +bool input_menu_choice(IO& io, int& result) +{ + return io.read_int(result); +} + +} // ::protocol +} // ::bs + +#endif //BS_PROTOCOL_H diff --git a/tests/device/libraries/BSTest/src/BSStdio.h b/tests/device/libraries/BSTest/src/BSStdio.h new file mode 100644 index 000000000..ea22f2236 --- /dev/null +++ b/tests/device/libraries/BSTest/src/BSStdio.h @@ -0,0 +1,39 @@ +#ifndef BS_STDIO_H +#define BS_STDIO_H + +#include +#include + +namespace bs +{ +class StdIOHelper +{ +public: + StdIOHelper() + { + } + + size_t printf(const char *format, ...) + { + va_list arg; + va_start(arg, format); + size_t result = vprintf(format, arg); + va_end(arg); + return result; + } + + bool read_int(int& result) + { + return scanf("%d", &result) == 1; + } +}; + +typedef StdIOHelper IOHelper; + +inline void fatal() { + throw std::runtime_error("fatal error"); +} + +} // namespace bs + +#endif //BS_STDIO_H diff --git a/tests/device/libraries/BSTest/src/BSTest.h b/tests/device/libraries/BSTest/src/BSTest.h new file mode 100644 index 000000000..c15f9b64d --- /dev/null +++ b/tests/device/libraries/BSTest/src/BSTest.h @@ -0,0 +1,217 @@ +#ifndef BSTEST_H +#define BSTEST_H + +#include +#include +#include +#include +#include +#include "BSProtocol.h" + +#if defined(ARDUINO) +#include "BSArduino.h" +#else +#include "BSStdio.h" +#endif + +namespace bs +{ +typedef void(*test_case_func_t)(); + +class TestCase +{ +public: + TestCase(TestCase* prev, test_case_func_t func, const char* file, size_t line, const char* name, const char* desc) + : m_func(func), m_file(file), m_line(line), m_name(name), m_desc(desc) + { + if (prev) { + prev->m_next = this; + } + } + + void run() const + { + (*m_func)(); + } + + TestCase* next() const + { + return m_next; + } + + const char* file() const + { + return m_file; + } + + size_t line() const + { + return m_line; + } + + const char* name() const + { + return m_name; + } + + const char* desc() const + { + return (m_desc)?m_desc:""; + } + +protected: + TestCase* m_next = nullptr; + test_case_func_t m_func; + const char* m_file; + size_t m_line; + const char* m_name; + const char* m_desc; +}; + +struct Registry { + void add(test_case_func_t func, const char* file, size_t line, const char* name, const char* desc) + { + TestCase* tc = new TestCase(m_last, func, file, line, name, desc); + if (!m_first) { + m_first = tc; + } + m_last = tc; + } + TestCase* m_first = nullptr; + TestCase* m_last = nullptr; +}; + +struct Env { + std::function m_check_pass; + std::function m_check_fail; + std::function m_fail; + Registry m_registry; +}; + +extern Env g_env; + +template +class Runner +{ + typedef Runner Tself; +public: + Runner(IO& io) : m_io(io) + { + g_env.m_check_pass = std::bind(&Tself::check_pass, this); + g_env.m_check_fail = std::bind(&Tself::check_fail, this, std::placeholders::_1); + g_env.m_fail = std::bind(&Tself::fail, this, std::placeholders::_1); + } + + ~Runner() + { + g_env.m_check_pass = 0; + g_env.m_check_fail = 0; + g_env.m_fail = 0; + } + + void run() + { + do { + } while(do_menu()); + } + + void check_pass() + { + ++m_check_pass_count; + } + + void check_fail(size_t line) + { + ++m_check_fail_count; + protocol::output_check_failure(m_io, line); + } + + void fail(size_t line) + { + protocol::output_test_end(m_io, false, m_check_pass_count + m_check_fail_count, m_check_fail_count, line); + bs::fatal(); + } + +protected: + bool do_menu() + { + protocol::output_menu_begin(m_io); + int id = 1; + for (TestCase* tc = g_env.m_registry.m_first; tc; tc = tc->next(), ++id) { + protocol::output_menu_item(m_io, id, tc->name(), tc->desc()); + } + protocol::output_menu_end(m_io); + while(true) { + int id; + if (!protocol::input_menu_choice(m_io, id)) { + continue; + } + if (id < 0) { + return true; + } + TestCase* tc = g_env.m_registry.m_first; + for (int i = 0; i != id - 1 && tc; ++i, tc = tc->next()); + if (!tc) { + bs::fatal(); + } + m_check_pass_count = 0; + m_check_fail_count = 0; + protocol::output_test_start(m_io, tc->file(), tc->line(), tc->name(), tc->desc()); + tc->run(); + bool success = m_check_fail_count == 0; + protocol::output_test_end(m_io, success, m_check_pass_count + m_check_fail_count, m_check_fail_count); + } + } + +protected: + IO& m_io; + size_t m_check_pass_count; + size_t m_check_fail_count; +}; + +class AutoReg +{ +public: + AutoReg(test_case_func_t func, const char* file, size_t line, const char* name, const char* desc = nullptr) + { + g_env.m_registry.add(func, file, line, name, desc); + } +}; + +inline void check(bool condition, size_t line) +{ + if (!condition) { + g_env.m_check_fail(line); + } else { + g_env.m_check_pass(); + } +} + +inline void require(bool condition, size_t line) +{ + if (!condition) { + g_env.m_check_fail(line); + g_env.m_fail(line); + } else { + g_env.m_check_pass(); + } +} + +} // ::bs + +#define BS_NAME_LINE2( name, line ) name##line +#define BS_NAME_LINE( name, line ) BS_NAME_LINE2( name, line ) +#define BS_UNIQUE_NAME( name ) BS_NAME_LINE( name, __LINE__ ) + +#define TEST_CASE( ... ) \ + static void BS_UNIQUE_NAME( TEST_FUNC__ )(); \ + namespace{ bs::AutoReg BS_UNIQUE_NAME( test_autoreg__ )( &BS_UNIQUE_NAME( TEST_FUNC__ ), __FILE__, __LINE__, __VA_ARGS__ ); }\ + static void BS_UNIQUE_NAME( TEST_FUNC__ )() + +#define CHECK(condition) bs::check((condition), __LINE__) +#define REQUIRE(condition) bs::require((condition), __LINE__) +#define FAIL() bs::g_env.m_fail(__LINE__) + +#define BS_ENV_DECLARE() namespace bs { Env g_env; } +#define BS_RUN(...) do { bs::IOHelper helper = bs::IOHelper(__VA_ARGS__); bs::Runner runner(helper); runner.run(); } while(0); +#endif //BSTEST_H diff --git a/tests/device/libraries/BSTest/test/test.cpp b/tests/device/libraries/BSTest/test/test.cpp new file mode 100644 index 000000000..c6117c8b4 --- /dev/null +++ b/tests/device/libraries/BSTest/test/test.cpp @@ -0,0 +1,51 @@ +#include "BSTest.h" +#include + +BS_ENV_DECLARE(); + + +int main() +{ + while(true) { + try{ + BS_RUN(); + return 0; + }catch(...) { + printf("Exception\n\n"); + } + } + return 1; +} + + +TEST_CASE("this test runs successfully", "[bluesmoke]") +{ + CHECK(1 + 1 == 2); + REQUIRE(2 * 2 == 4); +} + +TEST_CASE("another test which fails", "[bluesmoke][fail]") +{ + CHECK(true); + CHECK(false); + CHECK(true); + CHECK(false); +} + +TEST_CASE("another test which fails and crashes", "[bluesmoke][fail]") +{ + CHECK(true); + REQUIRE(false); +} + + +TEST_CASE("third test which should be skipped", "[.]") +{ + FAIL(); +} + + +TEST_CASE("this test also runs successfully", "[bluesmoke]") +{ + +} diff --git a/tests/device/libraries/test_config/.gitignore b/tests/device/libraries/test_config/.gitignore new file mode 100644 index 000000000..4be1df52e --- /dev/null +++ b/tests/device/libraries/test_config/.gitignore @@ -0,0 +1 @@ +test_config.h diff --git a/tests/device/libraries/test_config/library.properties b/tests/device/libraries/test_config/library.properties new file mode 100644 index 000000000..5df5c966e --- /dev/null +++ b/tests/device/libraries/test_config/library.properties @@ -0,0 +1,9 @@ +name=TestConfig +version=0.0 +author= +maintainer= +sentence= +paragraph= +category=Uncategorized +url= +architectures=esp8266 diff --git a/tests/device/libraries/test_config/test_config.h.template b/tests/device/libraries/test_config/test_config.h.template new file mode 100644 index 000000000..f318f91f0 --- /dev/null +++ b/tests/device/libraries/test_config/test_config.h.template @@ -0,0 +1,8 @@ +#pragma once + +#define STA_SSID "test_wifi" +#define STA_PASS "test_wifi_pass" + +#define AP_SSID "test_wifi_ap" +#define AP_PASS "test_wifi_ap_pass" + diff --git a/tests/device/test_FS/test_FS.ino b/tests/device/test_FS/test_FS.ino new file mode 100644 index 000000000..cbd8b5821 --- /dev/null +++ b/tests/device/test_FS/test_FS.ino @@ -0,0 +1,136 @@ +#include +#include "FS.h" +#include + +BS_ENV_DECLARE(); + +void setup() +{ + Serial.begin(115200); + BS_RUN(Serial); +} + + + +TEST_CASE("read-write test","[fs]") +{ + REQUIRE(SPIFFS.begin()); + + String text = "write test"; + { + File out = SPIFFS.open("/tmp.txt", "w"); + REQUIRE(out); + out.print(text); + } + + { + File in = SPIFFS.open("/tmp.txt", "r"); + REQUIRE(in); + CHECK(in.size() == text.length()); + in.setTimeout(0); + String result = in.readString(); + CHECK(result == text); + } +} + +TEST_CASE("A bunch of files show up in openDir, and can be removed", "[fs]") +{ + REQUIRE(SPIFFS.begin()); + const int n = 10; + int found[n] = {0}; + + for (int i = 0; i < n; ++i) { + String name = "seq_"; + name += i; + name += ".txt"; + + File out = SPIFFS.open(name, "w"); + REQUIRE(out); + + out.println(i); + } + + { + Dir root = SPIFFS.openDir("/"); + while (root.next()) { + String fileName = root.fileName(); + CHECK(fileName.indexOf("seq_") == 0); + int i = fileName.substring(4).toInt(); + CHECK(i >= 0 && i < n); + found[i]++; + } + + for (auto f : found) { + CHECK(f == 1); + } + } + + { + Dir root = SPIFFS.openDir("/"); + while (root.next()) { + String fileName = root.fileName(); + CHECK(SPIFFS.remove(fileName)); + } + } +} + +TEST_CASE("files can be renamed", "[fs]") +{ + REQUIRE(SPIFFS.begin()); + { + File tmp = SPIFFS.open("/tmp1.txt", "w"); + tmp.println("rename test"); + } + + { + CHECK(SPIFFS.rename("/tmp1.txt", "/tmp2.txt")); + + File tmp2 = SPIFFS.open("/tmp2.txt", "r"); + CHECK(tmp2); + } +} + +TEST_CASE("FS::info works") +{ + REQUIRE(SPIFFS.begin()); + FSInfo info; + CHECK(SPIFFS.info(info)); + + Serial.printf("Total: %u\nUsed: %u\nBlock: %u\nPage: %u\nMax open files: %u\nMax path len: %u\n", + info.totalBytes, + info.usedBytes, + info.blockSize, + info.pageSize, + info.maxOpenFiles, + info.maxPathLength + ); +} + +TEST_CASE("FS is empty after format","[fs]") +{ + REQUIRE(SPIFFS.begin()); + REQUIRE(SPIFFS.format()); + + Dir root = SPIFFS.openDir("/"); + int count = 0; + while (root.next()) { + ++count; + } + CHECK(count == 0); +} + +TEST_CASE("Can reopen empty file","[fs]") +{ + REQUIRE(SPIFFS.begin()); + { + File tmp = SPIFFS.open("/tmp.txt", "w"); + } + { + File tmp = SPIFFS.open("/tmp.txt", "w"); + CHECK(tmp); + } +} + +void loop() +{ +} diff --git a/tests/device/test_Print_printf/test_Print_printf.ino b/tests/device/test_Print_printf/test_Print_printf.ino new file mode 100644 index 000000000..739657262 --- /dev/null +++ b/tests/device/test_Print_printf/test_Print_printf.ino @@ -0,0 +1,38 @@ +#include +#include + +BS_ENV_DECLARE(); + +void setup() +{ + Serial.begin(115200); + BS_RUN(Serial); +} + +TEST_CASE("Print::printf works for any reasonable output length", "[Print]") +{ + + auto test_printf = [](size_t size) { + StreamString str; + auto buf = new char[size + 1]; + for (int i = 0; i < size; ++i) { + buf[i] = 'a'; + } + buf[size] = 0; + str.printf("%s%8d", buf, 56789102); + delete[] buf; + CHECK(str.length() == size + 8); + CHECK(str.substring(size) == "56789102"); + }; + + auto before = ESP.getFreeHeap(); + test_printf(1); + test_printf(10); + test_printf(100); + test_printf(1000); + test_printf(10000); + auto after = ESP.getFreeHeap(); + CHECK(before == after); +} + +void loop() {} diff --git a/tests/device/test_http_client/test_http_client.ino b/tests/device/test_http_client/test_http_client.ino new file mode 100644 index 000000000..ef2fd6480 --- /dev/null +++ b/tests/device/test_http_client/test_http_client.ino @@ -0,0 +1,35 @@ +#include +#include +#include +#include +#include + +BS_ENV_DECLARE(); + +void setup() +{ + Serial.begin(115200); + WiFi.persistent(false); + WiFi.begin(STA_SSID, STA_PASS); + while (WiFi.status() != WL_CONNECTED) { + delay(500); + } + BS_RUN(Serial); +} + +TEST_CASE("HTTP client GET request", "[HTTPClient]") +{ + HTTPClient http; + http.begin("http://httpbin.org/get?a=1&b=asdadf"); + auto httpCode = http.GET(); + CHECK(httpCode == HTTP_CODE_OK); + String payload = http.getString(); + CHECK(payload.indexOf("\"a\": \"1\"") != -1); + CHECK(payload.indexOf("\"b\": \"asdad\"") != -1); + http.end(); +} + +void loop() +{ +} + diff --git a/tests/device/test_overrides/test_overrides.ino b/tests/device/test_overrides/test_overrides.ino new file mode 100644 index 000000000..655f12084 --- /dev/null +++ b/tests/device/test_overrides/test_overrides.ino @@ -0,0 +1,38 @@ +#include +BS_ENV_DECLARE(); + +ADC_MODE(ADC_VCC); +RF_MODE(RF_CAL); + +static int rf_pre_init_flag = 0; + +RF_PRE_INIT() +{ + rf_pre_init_flag = 42; +} + +static unsigned setup_micros; + +void setup() +{ + setup_micros = micros(); + Serial.begin(115200); + BS_RUN(Serial); +} + +TEST_CASE("ADC_MODE override works", "[core]") +{ + auto vcc = ESP.getVcc(); + Serial.printf("VCC: %d\r\n", vcc); + Serial.printf("A0: %d\r\n", analogRead(A0)); + CHECK(vcc > 3000 && vcc < 3600); +} + +TEST_CASE("RF_PRE_INIT override works", "[core]") +{ + CHECK(rf_pre_init_flag == 42); +} + +void loop() +{ +} diff --git a/tests/device/test_tests/test_tests.ino b/tests/device/test_tests/test_tests.ino new file mode 100644 index 000000000..9db064afa --- /dev/null +++ b/tests/device/test_tests/test_tests.ino @@ -0,0 +1,47 @@ +#include +#include + +BS_ENV_DECLARE(); + +void setup() +{ + Serial.begin(115200); + BS_RUN(Serial); +} + + +TEST_CASE("this test runs successfully", "[bs]") +{ + CHECK(1 + 1 == 2); + REQUIRE(2 * 2 == 4); +} + +TEST_CASE("another test which fails", "[bs][fail]") +{ + CHECK(true); + CHECK(false); + CHECK(true); + CHECK(false); +} + +TEST_CASE("another test which fails and crashes", "[bs][fail]") +{ + CHECK(true); + REQUIRE(false); +} + + +TEST_CASE("third test which should be skipped", "[.]") +{ + FAIL(); +} + + +TEST_CASE("this test also runs successfully", "[bs]") +{ + +} + +void loop() +{ +} diff --git a/tests/device/test_time/test_time.ino b/tests/device/test_time/test_time.ino new file mode 100644 index 000000000..52967fbba --- /dev/null +++ b/tests/device/test_time/test_time.ino @@ -0,0 +1,46 @@ +#include +#include +#include +#include +#include + +BS_ENV_DECLARE(); + +void setup() +{ + Serial.begin(115200); + WiFi.persistent(false); + WiFi.begin(STA_SSID, STA_PASS); + while (WiFi.status() != WL_CONNECTED) { + delay(500); + } + BS_RUN(Serial); +} + + +TEST_CASE("Can sync time", "[time]") +{ + int timezone = 3; + int dst = 0; + + configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov"); + Serial.println("\nWaiting for time"); + unsigned timeout = 5000; + unsigned start = millis(); + while (millis() - start < timeout) { + time_t now = time(nullptr); + if (now > (2016 - 1970) * 365 * 24 * 3600) { + return; + } + delay(100); + } + { + time_t now = time(nullptr); + Serial.println(ctime(&now)); + } + CHECK(false); +} + +void loop() +{ +} diff --git a/tests/test_umm_malloc/test_umm_malloc.ino b/tests/device/test_umm_malloc/test_umm_malloc.ino similarity index 51% rename from tests/test_umm_malloc/test_umm_malloc.ino rename to tests/device/test_umm_malloc/test_umm_malloc.ino index 16348bc70..b805f73f7 100644 --- a/tests/test_umm_malloc/test_umm_malloc.ino +++ b/tests/device/test_umm_malloc/test_umm_malloc.ino @@ -1,13 +1,21 @@ // test that we can include umm_malloc.h from sketch (#1652) #include +#include -void setup() { +BS_ENV_DECLARE(); + +void setup() +{ Serial.begin(115200); - delay(1000); + BS_RUN(Serial); +} + +TEST_CASE("umm_info can be called", "[umm_malloc]") +{ umm_info(NULL, 1); } -void loop() { - +void loop() +{ } diff --git a/tests/test_overrides/test_overrides.ino b/tests/test_overrides/test_overrides.ino deleted file mode 100644 index fe963492f..000000000 --- a/tests/test_overrides/test_overrides.ino +++ /dev/null @@ -1,11 +0,0 @@ -ADC_MODE(ADC_VCC); -RF_MODE(RF_DISABLED); -RF_PRE_INIT() -{ -} - -void setup() { -} - -void loop() { -}