lwip2 updates: no more git sub-sub-module deps, faster checksum, backlog limitation and other fixes (#6887)

* upstream lwIP is now downloaded by a makefile, not subsubmoduled

* lwip2: upstream lwIP not sub-sub-modules anymore
lwip2: Allow IPv4 and IPv6 DNS and SNTP server configured via DHCP to co-exist (patch against upstream)

* lwip2: enable tcp-listen-with-backlog feature

* lwip2 submodule update:
- enable more efficient chksum algorithm thanks to Richard Allen
- enable tcp listener with backlog

* more comments, fix backlog management, fix API
* move default value definition in .cpp
because one must not believe it can be redefined before including WiFiServer.h

* improved backlog handling, it is no more a breaking change

libraries/ESP8266WiFi/src/WiFiServer.cpp

@@ -35,12 +35,17 @@ extern "C" {
 #include "lwip/opt.h"
 #include "lwip/tcp.h"
 #include "lwip/inet.h"
+#include "lwip/init.h" // LWIP_VERSION_
 #include <include/ClientContext.h>
 
+#ifndef MAX_PENDING_CLIENTS_PER_PORT
+#define MAX_PENDING_CLIENTS_PER_PORT 5
+#endif
+
 WiFiServer::WiFiServer(const IPAddress& addr, uint16_t port)
 : _port(port)
 , _addr(addr)
-, _pcb(nullptr)
+, _listen_pcb(nullptr)
 , _unclaimed(nullptr)
 , _discarded(nullptr)
 {
@@ -49,7 +54,7 @@ WiFiServer::WiFiServer(const IPAddress& addr, uint16_t port)
 WiFiServer::WiFiServer(uint16_t port)
 : _port(port)
 , _addr(IP_ANY_TYPE)
-, _pcb(nullptr)
+, _listen_pcb(nullptr)
 , _unclaimed(nullptr)
 , _discarded(nullptr)
 {
@@ -60,9 +65,14 @@ void WiFiServer::begin() {
 }
 
 void WiFiServer::begin(uint16_t port) {
+    return begin(port, MAX_PENDING_CLIENTS_PER_PORT);
+}
+
+void WiFiServer::begin(uint16_t port, uint8_t backlog) {
     close();
+    if (!backlog)
+        return;
     _port = port;
-    err_t err;
     tcp_pcb* pcb = tcp_new();
     if (!pcb)
         return;
@@ -70,20 +80,23 @@ void WiFiServer::begin(uint16_t port) {
     pcb->so_options |= SOF_REUSEADDR;
 
     // (IPAddress _addr) operator-converted to (const ip_addr_t*)
-    err = tcp_bind(pcb, _addr, _port);
+    if (tcp_bind(pcb, _addr, _port) != ERR_OK) {
-
-    if (err != ERR_OK) {
         tcp_close(pcb);
         return;
     }
 
+#if LWIP_VERSION_MAJOR == 1
     tcp_pcb* listen_pcb = tcp_listen(pcb);
+#else
+    tcp_pcb* listen_pcb = tcp_listen_with_backlog(pcb, backlog);
+#endif
+
     if (!listen_pcb) {
         tcp_close(pcb);
         return;
     }
-    _pcb = listen_pcb;
+    _listen_pcb = listen_pcb;
-    _port = _pcb->local_port;
+    _port = _listen_pcb->local_port;
     tcp_accept(listen_pcb, &WiFiServer::_s_accept);
     tcp_arg(listen_pcb, (void*) this);
 }
@@ -111,6 +124,10 @@ WiFiClient WiFiServer::available(byte* status) {
     (void) status;
     if (_unclaimed) {
         WiFiClient result(_unclaimed);
+#if LWIP_VERSION_MAJOR != 1
+        _unclaimed->acceptPCB();
+        tcp_backlog_accepted(_unclaimed->getPCB());
+#endif
         _unclaimed = _unclaimed->next();
         result.setNoDelay(getNoDelay());
         DEBUGV("WS:av\r\n");
@@ -122,9 +139,9 @@ WiFiClient WiFiServer::available(byte* status) {
 }
 
 uint8_t WiFiServer::status()  {
-    if (!_pcb)
+    if (!_listen_pcb)
         return CLOSED;
-    return _pcb->state;
+    return _listen_pcb->state;
 }
 
 uint16_t WiFiServer::port() const {
@@ -132,11 +149,11 @@ uint16_t WiFiServer::port() const {
 }
 
 void WiFiServer::close() {
-    if (!_pcb) {
+    if (!_listen_pcb) {
       return;
     }
-    tcp_close(_pcb);
+    tcp_close(_listen_pcb);
-    _pcb = nullptr;
+    _listen_pcb = nullptr;
 }
 
 void WiFiServer::stop() {
@@ -169,9 +186,36 @@ T* slist_append_tail(T* head, T* item) {
 long WiFiServer::_accept(tcp_pcb* apcb, long err) {
     (void) err;
     DEBUGV("WS:ac\r\n");
+
+#if LWIP_VERSION_MAJOR == 1
+
     ClientContext* client = new ClientContext(apcb, &WiFiServer::_s_discard, this);
+    tcp_accepted(_listen_pcb);
+
+#else
+
+    // backlog doc:
+    // http://lwip.100.n7.nabble.com/Problem-re-opening-listening-pbc-tt32484.html#a32494
+    // https://www.nongnu.org/lwip/2_1_x/group__tcp__raw.html#gaeff14f321d1eecd0431611f382fcd338
+
+    // lwip-v2: Tell ClientContext to not accept yet the connection (final 'false' below)
+    ClientContext* client = new ClientContext(apcb, &WiFiServer::_s_discard, this, false);
+    // increase lwIP's backlog
+    tcp_backlog_delayed(apcb);
+
+    // Optimization Path:
+    // when lwip-v1.4 is not allowed anymore,
+    // - _accept() should not create ClientContext anymore
+    // - apcb should be stored into some sort of linked list (->_unclaimed)
+    //   (the linked list would store tcp_pcb* instead of ClientContext*)
+    //   (TCP_PCB_EXTARGS might be used for that (lwIP's struct tcp_pcb))
+    // - on available(), get the pcb back and create the ClientContext
+    // (this is not done today for better source readability with lwip-1.4 around)
+
+#endif
+
     _unclaimed = slist_append_tail(_unclaimed, client);
-    tcp_accepted(_pcb);
+
     return ERR_OK;
 }
 

libraries/ESP8266WiFi/src/WiFiServer.h

@@ -31,6 +31,37 @@ extern "C" {
 #include "Server.h"
 #include "IPAddress.h"
 
+// lwIP-v2 backlog facility allows to keep memory safe by limiting the
+// maximum number of incoming *pending clients*.  Default number of possibly
+// simultaneously pending clients is defined in WiFiServer.cpp
+// (MAX_PENDING_CLIENTS_PER_PORT=5).  User can overide it at runtime from
+// sketch:
+//      WiFiServer::begin(port, max-simultaneous-pending-clients);
+//
+// An "incoming pending" client is a new incoming TCP connection trying to
+// reach the TCP server.  It is "pending" until lwIP acknowledges it and
+// "accepted / no more pending" when user calls WiFiServer::available().
+//
+// Before the backlog feature or with lwIP-v1.4, there was no pending
+// connections: They were immediately accepted and filling RAM.
+//
+// Several pending clients can appear during the time when one client is
+// served by a long not-async service like ESP8266WebServer.  During that
+// time WiFiServer::available() cannot be called.
+//
+// Note: This *does not limit* the number of *simultaneously accepted
+//       clients*.  Such limit management is left to the user.
+//
+// Thus, when the maximum number of pending connections is reached, new
+// connections are delayed.
+// By "delayed", it is meant that WiFiServer(lwIP) will not answer to the
+// SYN packet until there is room for a new one: The TCP server on that port
+// will be mute.  The TCP client will regularly try to connect until success
+// or a timeout occurs (72s on windows).
+//
+// When user calls WiFiServer::available(), the tcp server stops muting and
+// answers to newcomers (until the "backlog" pending list is full again).
+
 class ClientContext;
 class WiFiClient;
 
@@ -39,7 +70,7 @@ class WiFiServer : public Server {
 protected:
   uint16_t _port;
   IPAddress _addr;
-  tcp_pcb* _pcb;
+  tcp_pcb* _listen_pcb;
 
   ClientContext* _unclaimed;
   ClientContext* _discarded;
@@ -53,6 +84,7 @@ public:
   bool hasClient();
   void begin();
   void begin(uint16_t port);
+  void begin(uint16_t port, uint8_t backlog);
   void setNoDelay(bool nodelay);
   bool getNoDelay();
   virtual size_t write(uint8_t);

libraries/ESP8266WiFi/src/include/ClientContext.h

@@ -36,16 +36,27 @@ bool getDefaultPrivateGlobalSyncValue ();
 class ClientContext
 {
 public:
-    ClientContext(tcp_pcb* pcb, discard_cb_t discard_cb, void* discard_cb_arg) :
+    ClientContext(tcp_pcb* pcb, discard_cb_t discard_cb, void* discard_cb_arg, bool acceptNow = true) :
         _pcb(pcb), _rx_buf(0), _rx_buf_offset(0), _discard_cb(discard_cb), _discard_cb_arg(discard_cb_arg), _refcnt(0), _next(0),
         _sync(::getDefaultPrivateGlobalSyncValue())
     {
-        tcp_setprio(pcb, TCP_PRIO_MIN);
+        if (acceptNow)
-        tcp_arg(pcb, this);
+            acceptPCB();
-        tcp_recv(pcb, &_s_recv);
+    }
-        tcp_sent(pcb, &_s_acked);
+
-        tcp_err(pcb, &_s_error);
+    tcp_pcb* getPCB ()
-        tcp_poll(pcb, &_s_poll, 1);
+    {
+        return _pcb;
+    }
+
+    void acceptPCB()
+    {
+        tcp_setprio(_pcb, TCP_PRIO_MIN);
+        tcp_arg(_pcb, this);
+        tcp_recv(_pcb, &_s_recv);
+        tcp_sent(_pcb, &_s_acked);
+        tcp_err(_pcb, &_s_error);
+        tcp_poll(_pcb, &_s_poll, 1);
 
         // keep-alive not enabled by default
         //keepAlive();

tests/host/common/MockWiFiServer.cpp

@@ -59,7 +59,7 @@ WiFiClient WiFiServer::available (uint8_t* status)
 {
 	(void)status;
 	if (hasClient())
-		return WiFiClient(new ClientContext(serverAccept(pcb2int(_pcb))));
+		return WiFiClient(new ClientContext(serverAccept(pcb2int(_listen_pcb))));
 	return WiFiClient();
 }
 

tests/host/common/MockWiFiServerSocket.cpp

@@ -60,6 +60,13 @@ int serverAccept (int srvsock)
 
 void WiFiServer::begin (uint16_t port)
 {
+    return begin(port, !0);
+}
+
+void WiFiServer::begin (uint16_t port, uint8_t backlog)
+{
+    if (!backlog)
+        return;
 	_port = port;
 	return begin();
 }
@@ -109,13 +116,13 @@ void WiFiServer::begin ()
 
 
 	// store int into pointer
-	_pcb = int2pcb(sock);
+	_listen_pcb = int2pcb(sock);
 }
 
 bool WiFiServer::hasClient ()
 {
 	struct pollfd p;
-	p.fd = pcb2int(_pcb);
+	p.fd = pcb2int(_listen_pcb);
 	p.events = POLLIN;
 	return poll(&p, 1, 0) && p.revents == POLLIN;
 }
@@ -134,7 +141,7 @@ size_t WiFiServer::write (const uint8_t *buf, size_t size)
 
 void WiFiServer::close ()
 {
-	if (pcb2int(_pcb) >= 0)
+	if (pcb2int(_listen_pcb) >= 0)
-		::close(pcb2int(_pcb));
+		::close(pcb2int(_listen_pcb));
-	_pcb = int2pcb(-1);
+	_listen_pcb = int2pcb(-1);
 }

tools/sdk/lwip2/builder

@@ -1 +1 @@
-Subproject commit 354887a25f83064dc0c795e11704190845812713
+Subproject commit b543b1f1f4aff4787693da71b6ea91ca23e9b01f

tools/sdk/lwip2/include/lwip-git-hash.h

@@ -1,5 +1,5 @@
 // generated by makefiles/make-lwip2-hash
 #ifndef LWIP_HASH_H
 #define LWIP_HASH_H
-#define LWIP_HASH_STR "STABLE-2_1_2_RELEASE/glue:1.2-16-ge23a07e"
+#define LWIP_HASH_STR "STABLE-2_1_2_RELEASE/glue:1.2-29-gb543b1f"
 #endif // LWIP_HASH_H

tools/sdk/lwip2/include/lwipopts.h

@@ -1402,7 +1402,7 @@
  * TCP_LISTEN_BACKLOG: Enable the backlog option for tcp listen pcb.
  */
 #if !defined TCP_LISTEN_BACKLOG || defined __DOXYGEN__
-#define TCP_LISTEN_BACKLOG              0
+#define TCP_LISTEN_BACKLOG              LWIP_FEATURES // 0
 #endif
 
 /**
@@ -2278,6 +2278,12 @@
  * @ingroup lwip_opts_infrastructure
  * @{
  */
+/**
+ * LWIP_CHKSUM_ALGORITHM==3: Checksum algorithm fastest for ESP8266
+ */
+#if !defined LWIP_CHKSUM_ALGORITHM || defined __DOXYGEN__
+#define LWIP_CHKSUM_ALGORITHM           3 // 2
+#endif
 /**
  * LWIP_CHECKSUM_CTRL_PER_NETIF==1: Checksum generation/check can be enabled/disabled
  * per netif.
@@ -3573,7 +3579,7 @@ extern "C" {
 #define SNTP_SUPPRESS_DELAY_CHECK 1
 #define SNTP_UPDATE_DELAY_DEFAULT 3600000   // update delay defined by a default weak function
 #define SNTP_UPDATE_DELAY sntp_update_delay_MS_rfc_not_less_than_15000()
-extern uint32_t SNTP_UPDATE_DELAY;
+uint32_t SNTP_UPDATE_DELAY;
 
 #if LWIP_FEATURES
 // esp8266/arduino/lwip-1.4 had 3 possible SNTP servers (constant was harcoded)
@@ -3591,7 +3597,7 @@ extern uint32_t SNTP_UPDATE_DELAY;
 #define SNTP_STARTUP_DELAY 1                // enable startup delay
 #define SNTP_STARTUP_DELAY_FUNC_DEFAULT 0   // to 0 by default via a default weak function
 #define SNTP_STARTUP_DELAY_FUNC sntp_startup_delay_MS_rfc_not_less_than_60000()
-extern uint32_t SNTP_STARTUP_DELAY_FUNC;
+uint32_t SNTP_STARTUP_DELAY_FUNC;
 
 /*
    --------------------------------------------------
@@ -3611,7 +3617,7 @@ struct netif;
 #error LWIP_ERR_T definition should come from lwip1.4 from espressif
 #endif
 //#define LWIP_ERR_T s8
-LWIP_ERR_T lwip_unhandled_packet (struct pbuf* pbuf, struct netif* netif) __attribute__((weak));
+LWIP_ERR_T lwip_unhandled_packet (struct pbuf* pbuf, struct netif* netif);
 
 /*
    --------------------------------------------------