Sketch emulation on host (#5342)

* WIP compile examples on host with 'make examples'

* WIP bufferize tcp input

* WIP Makefile

* WIP network to rework, tcp/udp to factorize, udp addresses broken

* minor changes to the core

* WIP basic udp working

* WIP mdns

* WIP mcast receiving, not sending

* WIP mdns OK

* beta version

* SSL + doc

* update travis host test command

* licenses

* typo

* doc: arduino builder is not around: declare functions before calling them

* fix with latest SSL PR, compile in 32 bits mode

* fix make clean

* make -m32 optional

* 32bits compiler ability tester

* WIP

* WIP (fix 1 vtable error, still another one to hunt with using spiffs)

* example astyle

* fix os_printf_plus

* load / save mock spiffs

* fix style

* fix using spiffs/mock

* don't mess ram

* update doc

* remove leftover

* optimization -Os except for CI, rename ARCH32 to FORCE32

* revert useless cast (not even compiled)

* remove unused function

* use proper type for pointer arithmetics

* makefile: sketch object and cpp file moved to bin/ directories
easier to clean, and IDE don't like them

* changes for review

* make use of %zd

* less verbose makefile by default (option)

* update readme

libraries/DNSServer/src/DNSServer.cpp

@@ -10,7 +10,7 @@
 
 DNSServer::DNSServer()
 {
-  _ttl = htonl(60);
+  _ttl = lwip_htonl(60);
   _errorReplyCode = DNSReplyCode::NonExistentDomain;
 }
 
@@ -35,7 +35,7 @@ void DNSServer::setErrorReplyCode(const DNSReplyCode &replyCode)
 
 void DNSServer::setTTL(const uint32_t &ttl)
 {
-  _ttl = htonl(ttl);
+  _ttl = lwip_htonl(ttl);
 }
 
 void DNSServer::stop()
@@ -81,7 +81,7 @@ void DNSServer::processNextRequest()
 
 bool DNSServer::requestIncludesOnlyOneQuestion(const DNSHeader* dnsHeader)
 {
-  return ntohs(dnsHeader->QDCount) == 1 &&
+  return lwip_ntohs(dnsHeader->QDCount) == 1 &&
          dnsHeader->ANCount == 0 &&
          dnsHeader->NSCount == 0 &&
          dnsHeader->ARCount == 0;

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -734,7 +734,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size)
         free(buff);
 
         if(size && (int) size != bytesWritten) {
-            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %d mismatch!.\n", bytesWritten, size);
+            DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %zd mismatch!.\n", bytesWritten, size);
             DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] ERROR SEND PAYLOAD FAILED!");
             return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED);
         } else {

libraries/ESP8266WebServer/src/ESP8266WebServer.cpp

@@ -438,12 +438,9 @@ void ESP8266WebServer::sendContent(const String& content) {
   const char * footer = "\r\n";
   size_t len = content.length();
   if(_chunked) {
-    char * chunkSize = (char *)malloc(11);
-    if(chunkSize){
-      sprintf(chunkSize, "%x%s", len, footer);
-      _currentClientWrite(chunkSize, strlen(chunkSize));
-      free(chunkSize);
-    }
+    char chunkSize[11];
+    sprintf(chunkSize, "%zx\r\n", len);
+    _currentClientWrite(chunkSize, strlen(chunkSize));
   }
   _currentClientWrite(content.c_str(), len);
   if(_chunked){
@@ -461,12 +458,9 @@ void ESP8266WebServer::sendContent_P(PGM_P content) {
 void ESP8266WebServer::sendContent_P(PGM_P content, size_t size) {
   const char * footer = "\r\n";
   if(_chunked) {
-    char * chunkSize = (char *)malloc(11);
-    if(chunkSize){
-      sprintf(chunkSize, "%x%s", size, footer);
-      _currentClientWrite(chunkSize, strlen(chunkSize));
-      free(chunkSize);
-    }
+    char chunkSize[11];
+    sprintf(chunkSize, "%zx\r\n", size);
+    _currentClientWrite(chunkSize, strlen(chunkSize));
   }
   _currentClientWrite_P(content, size);
   if(_chunked){

libraries/ESP8266WiFi/examples/udp/udp.ino

@@ -0,0 +1,80 @@
+/*
+  UDPSendReceive.pde:
+  This sketch receives UDP message strings, prints them to the serial port
+  and sends an "acknowledge" string back to the sender
+
+  A Processing sketch is included at the end of file that can be used to send
+  and received messages for testing with a computer.
+
+  created 21 Aug 2010
+  by Michael Margolis
+
+  This code is in the public domain.
+
+  adapted from Ethernet library examples
+*/
+
+
+#include <ESP8266WiFi.h>
+#include <WiFiUdp.h>
+
+#define SSID "ssid"
+#define PSK  "psk"
+
+unsigned int localPort = 8888;      // local port to listen on
+
+// buffers for receiving and sending data
+char packetBuffer[UDP_TX_PACKET_MAX_SIZE]; //buffer to hold incoming packet,
+char  ReplyBuffer[] = "acknowledged\r\n";       // a string to send back
+
+WiFiUDP Udp;
+
+void setup() {
+  Serial.begin(115200);
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(SSID, PSK);
+  while (WiFi.status() != WL_CONNECTED) {
+    Serial.print('.');
+    delay(500);
+  }
+  Serial.print("Connected! IP address: ");
+  Serial.println(WiFi.localIP());
+  Serial.printf("UDP server on port %d\n", localPort);
+  Udp.begin(localPort);
+}
+
+void loop() {
+  // if there's data available, read a packet
+  int packetSize = Udp.parsePacket();
+  if (packetSize) {
+    Serial.print("Received packet of size ");
+    Serial.println(packetSize);
+    Serial.print("From ");
+    IPAddress remote = Udp.remoteIP();
+    for (int i = 0; i < 4; i++) {
+      Serial.print(remote[i], DEC);
+      if (i < 3) {
+        Serial.print(".");
+      }
+    }
+    Serial.print(", port ");
+    Serial.println(Udp.remotePort());
+
+    // read the packet into packetBufffer
+    Udp.read(packetBuffer, UDP_TX_PACKET_MAX_SIZE);
+    Serial.println("Contents:");
+    Serial.println(packetBuffer);
+
+    // send a reply, to the IP address and port that sent us the packet we received
+    Udp.beginPacket(Udp.remoteIP(), Udp.remotePort());
+    Udp.write(ReplyBuffer);
+    Udp.endPacket();
+  }
+  delay(10);
+}
+
+/*
+  test (shell/netcat):
+  ---------------
+	  nc -u 192.168.esp.address 8888
+*/

libraries/ESP8266WiFi/src/BearSSLHelpers.cpp

@@ -826,6 +826,8 @@ bool X509List::append(const uint8_t *derCert, size_t derLen) {
   return true;
 }
 
+#if !CORE_MOCK
+
 // Second stack thunked helpers
 make_stack_thunk(br_ssl_engine_recvapp_ack);
 make_stack_thunk(br_ssl_engine_recvapp_buf);
@@ -836,4 +838,6 @@ make_stack_thunk(br_ssl_engine_sendapp_buf);
 make_stack_thunk(br_ssl_engine_sendrec_ack);
 make_stack_thunk(br_ssl_engine_sendrec_buf);
 
+#endif
+
 };

libraries/ESP8266WiFi/src/WiFiClient.cpp

@@ -38,7 +38,7 @@ extern "C"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
 #include "lwip/netif.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "c_types.h"
 
 uint16_t WiFiClient::_localPort = 0;

libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp

@@ -40,10 +40,12 @@ extern "C" {
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
 #include "lwip/netif.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "c_types.h"
 #include "coredecls.h"
 
+#if !CORE_MOCK
+
 // The BearSSL thunks in use for now
 #define br_ssl_engine_recvapp_ack thunk_br_ssl_engine_recvapp_ack
 #define br_ssl_engine_recvapp_buf thunk_br_ssl_engine_recvapp_buf
@@ -54,6 +56,8 @@ extern "C" {
 #define br_ssl_engine_sendrec_ack thunk_br_ssl_engine_sendrec_ack
 #define br_ssl_engine_sendrec_buf thunk_br_ssl_engine_sendrec_buf
 
+#endif
+
 namespace BearSSL {
 
 void WiFiClientSecure::_clear() {
@@ -1377,6 +1381,21 @@ bool WiFiClientSecure::loadPrivateKey(Stream& stream, size_t size) {
 // SSL debugging which should focus on the WiFiClientBearSSL objects.
 
 extern "C" {
+
+#if CORE_MOCK
+
+  void br_esp8266_stack_proxy_init(uint8_t *space, uint16_t size) {
+    (void)space;
+    (void)size;
+  }
+  void _BearSSLCheckStack(const char *fcn, const char *file, int line) {
+    (void)fcn;
+    (void)file;
+    (void)line;
+  }
+
+#else // !CORE_MOCK
+
   extern size_t br_esp8266_stack_proxy_usage();
 
   void _BearSSLCheckStack(const char *fcn, const char *file, int line) {
@@ -1386,7 +1405,7 @@ extern "C" {
     int freeheap = ESP.getFreeHeap();
     static int laststack, lastheap, laststack2;
     if ((laststack != freestack) || (lastheap != freeheap) || (laststack2 != (int)br_esp8266_stack_proxy_usage())) {
-      Serial.printf("%s:%s(%d): FREESTACK=%d, STACK2USAGE=%d, FREEHEAP=%d\n", file, fcn, line, freestack, br_esp8266_stack_proxy_usage(), freeheap);
+      Serial.printf("%s:%s(%d): FREESTACK=%d, STACK2USAGE=%zd, FREEHEAP=%d\n", file, fcn, line, freestack, br_esp8266_stack_proxy_usage(), freeheap);
       if (freestack < 256) {
         Serial.printf("!!! Out of main stack space\n");
       }
@@ -1405,6 +1424,8 @@ extern "C" {
     }
   }
 
+#endif // !CORE_MOCK
+
   void _BearSSLSerialPrint(const char *str) {
     static int cnt = 0;
     Serial.printf("%s", str);

libraries/ESP8266WiFi/src/WiFiServer.cpp

@@ -35,7 +35,7 @@ extern "C" {
 #include "lwip/opt.h"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 
 WiFiServer::WiFiServer(IPAddress addr, uint16_t port)
 : _port(port)

libraries/ESP8266WiFi/src/WiFiServerSecureBearSSL.cpp

@@ -34,7 +34,7 @@ extern "C" {
 #include "lwip/opt.h"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "WiFiServerSecureBearSSL.h"
 
 namespace BearSSL {

libraries/ESP8266WiFi/src/WiFiUdp.cpp

@@ -38,7 +38,7 @@ extern "C"
 #include "lwip/inet.h"
 #include "lwip/igmp.h"
 #include "lwip/mem.h"
-#include "include/UdpContext.h"
+#include <include/UdpContext.h>
 
 
 template<>
@@ -106,7 +106,9 @@ uint8_t WiFiUDP::beginMulticast(IPAddress interfaceAddr, IPAddress multicast, ui
 
     _ctx = new UdpContext;
     _ctx->ref();
-    if (!_ctx->listen(*IP_ADDR_ANY, port)) {
+    ip_addr_t addr;
+    addr.addr = INADDR_ANY;
+    if (!_ctx->listen(addr, port)) {
         return 0;
     }
 
@@ -284,7 +286,7 @@ uint16_t WiFiUDP::localPort()
 void WiFiUDP::stopAll()
 {
     for (WiFiUDP* it = _s_first; it; it = it->_next) {
-        DEBUGV("%s %08x %08x\n", __func__, (uint32_t) it, (uint32_t) _s_first);
+        DEBUGV("%s %p %p\n", __func__, it, _s_first);
         it->stop();
     }
 }
@@ -292,7 +294,7 @@ void WiFiUDP::stopAll()
 void WiFiUDP::stopAllExcept(WiFiUDP * exC) {
     for (WiFiUDP* it = _s_first; it; it = it->_next) {
         if (it->_ctx != exC->_ctx) {
-            DEBUGV("%s %08x %08x\n", __func__, (uint32_t) it, (uint32_t) _s_first);
+            DEBUGV("%s %p %p\n", __func__, it, _s_first);
             it->stop();
         }
     }

libraries/ESP8266WiFi/src/include/SSLContext.h

@@ -37,7 +37,7 @@ extern "C"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
 #include "lwip/netif.h"
-#include "include/ClientContext.h"
+#include <include/ClientContext.h>
 #include "c_types.h"
 
 namespace axTLS {

libraries/ESP8266WiFi/src/include/UdpContext.h

@@ -174,7 +174,7 @@ public:
             return 0;
 
         udp_hdr* udphdr = GET_UDP_HDR(_rx_buf);
-        return ntohs(udphdr->src);
+        return lwip_ntohs(udphdr->src);
     }
 
     uint32_t getDestAddress()
@@ -252,6 +252,7 @@ public:
 
     void flush()
     {
+        //XXX this does not follow Arduino's flush definition
         if (!_rx_buf)
             return;
 

libraries/ESP8266mDNS/ESP8266mDNS.cpp

@@ -431,11 +431,11 @@ MDNSTxt * MDNSResponder::_getServiceTxt(char *name, char *proto){
   for (servicePtr = _services; servicePtr; servicePtr = servicePtr->_next) {
     if(servicePtr->_port > 0 && strcmp(servicePtr->_name, name) == 0 && strcmp(servicePtr->_proto, proto) == 0){
       if (servicePtr->_txts == 0) 
-        return false;
+        return nullptr;
       return servicePtr->_txts;
     }
   }
-  return 0;
+  return nullptr;
 }
 
 uint16_t MDNSResponder::_getServiceTxtLen(char *name, char *proto){