Fix Python3 errors for device tests (#6670)

* Fix Python3 errors for device tests

The Python3 migration didn't include fixes for local scripts in the
device test tree.

Fatal build and run Python errors fixed.

The last update to xunitmerge is ~5 years ago, so it looks to be
unsupported now.

Use a local copy of the two components to allow patching to work with
Python3.

The serial test seems to send garbage chars (non-ASCII/etc.), so use a
codepage 437 which supports all 255 chars.

Fixes: #6660

* Run tests at 160MHz (req'd for some SSL connections)

* Fix debuglevel options for builder

* Fix Python3 interpreter path in xunitmerge

* Remove virtualenv on "make clean"

* Add appropriate attribution, license to xunitmerge

Add like to the original sources with the author's license to the
copied/fixed xunitmerge files.

tests/device/Makefile

@@ -11,9 +11,9 @@ UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB)
 UPLOAD_BAUD ?= 460800
 UPLOAD_BOARD ?= nodemcu
 BS_DIR ?= libraries/BSTest
-DEBUG_LEVEL ?= DebugLevel=None____
+DEBUG_LEVEL ?= lvl=None____
 #FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=dio,UploadSpeed=115200,FlashSize=4M1M,LwIPVariant=v2mss536,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL)
-FQBN ?= esp8266com:esp8266:generic:xtal=80,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL)
+FQBN ?= esp8266com:esp8266:generic:xtal=160,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL)
 BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder
 TEST_CONFIG := test_env.cfg
 TEST_REPORT_XML := test_report.xml
@@ -104,7 +104,7 @@ ifneq ("$(NO_RUN)","1")
 endif
 
 $(TEST_REPORT_XML): $(HARDWARE_DIR) virtualenv
-	$(SILENT)$(BS_DIR)/virtualenv/bin/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
+	$(SILENT)$(BS_DIR)/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
 
 $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv
 	$(SILENT)$(BS_DIR)/virtualenv/bin/junit2html $< $@
@@ -124,7 +124,8 @@ virtualenv:
 
 clean:
 	rm -rf $(BUILD_DIR)
-	rm -rf $(HARDWARE_DIR)
+	rm -rf $(HARDWARE_DIR)A
+	rm -rf $(BS_DIR)/virtualenv
 	rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML)
 
 distclean: clean

tests/device/libraries/BSTest/requirements.txt

@@ -3,6 +3,5 @@ junit-xml
 MarkupSafe
 pexpect
 pyserial
-xunitmerge
 junit2html
-poster
+poster3

tests/device/libraries/BSTest/runner.py

@@ -236,10 +236,10 @@ ser = None
 def spawn_port(port_name, baudrate=115200):
     global ser
     ser = serial.serial_for_url(port_name, baudrate=baudrate)
-    return fdpexpect.fdspawn(ser, 'wb', timeout=0)
+    return fdpexpect.fdspawn(ser, 'wb', timeout=0, encoding='cp437')
 
 def spawn_exec(name):
-    return pexpect.spawn(name, timeout=0)
+    return pexpect.spawn(name, timeout=0, encoding='cp437')
 
 def run_tests(spawn, name, mocks, env_vars):
     tw = BSTestRunner(spawn, name, mocks, env_vars)

tests/device/libraries/BSTest/xmerge.py

@@ -0,0 +1,154 @@
+# Cloned from https://github.com/miki725/xunitmerge
+# to fix a Python3 error.
+#
+# xunitmerge is MIT licensed by Miroslav Shubernetskiy https://github.com/miki725
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from contextlib import contextmanager
+from xml.etree import ElementTree as etree
+from xml.sax.saxutils import quoteattr
+
+import six
+
+
+CNAME_TAGS = ('system-out', 'skipped', 'error', 'failure')
+CNAME_PATTERN = '<![CDATA[{}]]>'
+TAG_PATTERN = '<{tag}{attrs}>{text}</{tag}>'
+
+
+@contextmanager
+def patch_etree_cname(etree):
+    """
+    Patch ElementTree's _serialize_xml function so that it will
+    write text as CDATA tag for tags tags defined in CNAME_TAGS.
+
+    >>> import re
+    >>> from xml.etree import ElementTree
+    >>> xml_string = '''
+    ... <testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
+    ...     <testcase classname="some.class.Foo" name="test_system_out" time="0.001">
+    ...         <system-out>Some output here</system-out>
+    ...     </testcase>
+    ...     <testcase classname="some.class.Foo" name="test_skipped" time="0.001">
+    ...         <skipped type="unittest.case.SkipTest" message="Skipped">Skipped</skipped>
+    ...     </testcase>
+    ...     <testcase classname="some.class.Foo" name="test_error" time="0.001">
+    ...         <error type="KeyError" message="Error here">Error here</error>
+    ...     </testcase>
+    ...     <testcase classname="some.class.Foo" name="test_failure" time="0.001">
+    ...         <failure type="AssertionError" message="Failure here">Failure here</failure>
+    ...     </testcase>
+    ... </testsuite>
+    ... '''
+    >>> tree = ElementTree.fromstring(xml_string)
+    >>> with patch_etree_cname(ElementTree):
+    ...    saved = str(ElementTree.tostring(tree))
+    >>> systemout = re.findall(r'(<system-out>.*?</system-out>)', saved)[0]
+    >>> print(systemout)
+    <system-out><![CDATA[Some output here]]></system-out>
+    >>> skipped = re.findall(r'(<skipped.*?</skipped>)', saved)[0]
+    >>> print(skipped)
+    <skipped message="Skipped" type="unittest.case.SkipTest"><![CDATA[Skipped]]></skipped>
+    >>> error = re.findall(r'(<error.*?</error>)', saved)[0]
+    >>> print(error)
+    <error message="Error here" type="KeyError"><![CDATA[Error here]]></error>
+    >>> failure = re.findall(r'(<failure.*?</failure>)', saved)[0]
+    >>> print(failure)
+    <failure message="Failure here" type="AssertionError"><![CDATA[Failure here]]></failure>
+    """
+    original_serialize = etree._serialize_xml
+
+    def _serialize_xml(write, elem, *args, **kwargs):
+        if elem.tag in CNAME_TAGS:
+            attrs = ' '.join(
+                ['{}={}'.format(k, quoteattr(v))
+                 for k, v in sorted(elem.attrib.items())]
+            )
+            attrs = ' ' + attrs if attrs else ''
+            text = CNAME_PATTERN.format(elem.text)
+            write(TAG_PATTERN.format(
+                tag=elem.tag,
+                attrs=attrs,
+                text=text
+            ))
+        else:
+            original_serialize(write, elem, *args, **kwargs)
+
+    etree._serialize_xml = etree._serialize['xml'] = _serialize_xml
+
+    yield
+
+    etree._serialize_xml = etree._serialize['xml'] = original_serialize
+
+
+def merge_trees(*trees):
+    """
+    Merge all given XUnit ElementTrees into a single ElementTree.
+    This combines all of the children test-cases and also merges
+    all of the metadata of how many tests were executed, etc.
+    """
+    first_tree = trees[0]
+    first_root = first_tree.getroot()
+
+    if len(trees) == 0:
+        return first_tree
+
+    for tree in trees[1:]:
+        root = tree.getroot()
+
+        # append children elements (testcases)
+        first_root.extend(root.getchildren())
+
+        # combine root attributes which stores the number
+        # of executed tests, skipped tests, etc
+        for key, value in first_root.attrib.items():
+            if not value.isdigit():
+                continue
+            combined = six.text_type(int(value) + int(root.attrib.get(key, '0')))
+            first_root.set(key, combined)
+
+    return first_tree
+
+
+def merge_xunit(files, output, callback=None):
+    """
+    Merge the given xunit xml files into a single output xml file.
+
+    If callback is not None, it will be called with the merged ElementTree
+    before the output file is written (useful for applying other fixes to
+    the merged file). This can either modify the element tree in place (and
+    return None) or return a completely new ElementTree to be written.
+    """
+    trees = []
+
+    for f in files:
+        trees.append(etree.parse(f))
+
+    merged = merge_trees(*trees)
+
+    if callback is not None:
+        result = callback(merged)
+        if result is not None:
+            merged = result
+
+    with patch_etree_cname(etree):
+        merged.write(output, encoding='utf-8', xml_declaration=True)

tests/device/libraries/BSTest/xunitmerge

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# Cloned from https://github.com/miki725/xunitmerge
+# to fix a Python3 error.
+#
+# xunitmerge is MIT licensed by Miroslav Shubernetskiy https://github.com/miki725
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import argparse
+from xmerge import merge_xunit
+
+
+parser = argparse.ArgumentParser(
+    description='Utility for merging multiple XUnit xml reports '
+                'into a single xml report.',
+)
+parser.add_argument(
+    'report',
+    nargs='+',
+    type=argparse.FileType('r'),
+    help='Path of XUnit xml report. Multiple can be provided.',
+)
+parser.add_argument(
+    'output',
+    help='Path where merged of XUnit will be saved.',
+)
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    merge_xunit(args.report, args.output)

tests/device/test_ClientContext/test_ClientContext.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from mock_decorators import setup, teardown
 from flask import Flask, request
 from threading import Thread
@@ -21,7 +23,7 @@ def setup_tcpsrv(e):
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         for port in range(8266, 8285 + 1):
             try:
-                print >>sys.stderr, 'trying port', port
+                print ('trying port %d' %port, file=sys.stderr)
                 server_address = ("0.0.0.0", port)
                 sock.bind(server_address)
                 sock.listen(1)
@@ -31,17 +33,17 @@ def setup_tcpsrv(e):
                 print >>sys.stderr, 'busy'
         if not running:
             return
-        print >>sys.stderr, 'starting up on %s port %s' % server_address
+        print ('starting up on %s port %s' % server_address, file=sys.stderr)
-        print >>sys.stderr, 'waiting for connections'
+        print ( 'waiting for connections', file=sys.stderr)
         while running:
-            print >>sys.stderr, 'loop'
+            print ('loop', file=sys.stderr)
             readable, writable, errored = select.select([sock], [], [], 1.0)
             if readable:
                 connection, client_address = sock.accept()
                 try:
-                    print >>sys.stderr, 'client connected:', client_address
+                    print('client connected: %s' % str(client_address), file=sys.stderr)
                 finally:
-                    print >>sys.stderr, 'close'
+                    print ('close', file=sys.stderr)
                     connection.shutdown(socket.SHUT_RDWR)
                     connection.close()
 
@@ -54,7 +56,7 @@ def teardown_tcpsrv(e):
     global thread
     global running
     
-    print >>sys.stderr, 'closing'
+    print ('closing', file=sys.stderr)
     running = False
     thread.join()
     return 0

tests/device/test_http_client/test_http_client.py

@@ -1,7 +1,7 @@
 from mock_decorators import setup, teardown
 from flask import Flask, request, redirect
 from threading import Thread
-import urllib2
+import urllib
 import os
 import ssl
 import time
@@ -20,7 +20,7 @@ def setup_http_get(e):
         return 'Server shutting down...'
     @app.route("/", methods = ['GET', 'POST'])
     def root():
-        print('Got data: ' + request.data);
+        print('Got data: ' + request.data.decode());
         return 'hello!!!'
     @app.route("/data")
     def get_data():
@@ -48,7 +48,7 @@ def setup_http_get(e):
 
 @teardown('HTTP GET & POST requests')
 def teardown_http_get(e):
-    response = urllib2.urlopen('http://localhost:8088/shutdown')
+    response = urllib.request.urlopen('http://localhost:8088/shutdown')
     html = response.read()
     time.sleep(1) # avoid address in use error on macOS
 
@@ -86,6 +86,6 @@ def teardown_http_get(e):
     ctx.check_hostname = False
     ctx.verify_mode = ssl.CERT_NONE
     p = os.path.dirname(os.path.abspath(__file__))
-    response = urllib2.urlopen('https://localhost:8088/shutdown', context=ctx)
+    response = urllib.request.urlopen('https://localhost:8088/shutdown', context=ctx)
     html = response.read()
 

tests/device/test_http_server/test_http_server.py

@@ -1,10 +1,9 @@
 from collections import OrderedDict
 from mock_decorators import setup, teardown
 from threading import Thread
-from poster.encode import MultipartParam
+from poster3.encode import MultipartParam
-from poster.encode import multipart_encode
+from poster3.encode import multipart_encode
-from poster.streaminghttp import register_openers
+from poster3.streaminghttp import register_openers
-import urllib2
 import urllib
 
 def http_test(res, url, get=None, post=None):
@@ -13,8 +12,8 @@ def http_test(res, url, get=None, post=None):
         if get:
             url += '?' + urllib.urlencode(get)
         if post:
-            post = urllib.urlencode(post)
+            post = urllib.parse.quote(post)
-        request = urllib2.urlopen(url, post, 2)
+        request = urllib.request.urlopen(url, post, 2)
         response = request.read()
     except:
         return 1
@@ -60,8 +59,8 @@ def setup_http_upload(e):
             register_openers()
             p = MultipartParam("file", "0123456789abcdef", "test.txt", "text/plain; charset=utf8")
             datagen, headers = multipart_encode( [("var4", "val with spaces"), p] )
-            request = urllib2.Request('http://etd.local/upload', datagen, headers)
+            request = urllib.request('http://etd.local/upload', datagen, headers)
-            response = urllib2.urlopen(request, None, 2).read()
+            response = urllib.request.urlopen(request, None, 2).read()
         except:
             return 1
         if response != 'test.txt:16\nvar4 = val with spaces':