Implement esp_yield() as a replacement for delay(0)

esp_yield() now also calls esp_schedule(), original esp_yield() function renamed to esp_suspend().

Don't use delay(0) in the Core internals, libraries and examples. Use yield() when the code is
supposed to be called from CONT, use esp_yield() when the code can be called from either CONT or SYS.
Clean-up esp_yield() and esp_schedule() declarations across the code and use coredecls.h instead.

Implement helper functions for libraries that were previously using esp_yield(), esp_schedule() and
esp_delay() directly to wait for certain SYS context tasks to complete. Correctly use esp_delay()
for timeouts, make sure scheduled functions have a chance to run (e.g. LwIP_Ethernet uses recurrent)

Related issues:
- #6107 - discussion about the esp_yield() and esp_delay() usage in ClientContext
- #6212 - discussion about replacing delay() with a blocking loop
- #6680 - pull request introducing LwIP-based Ethernet
- #7146 - discussion that originated UART code changes
- #7969 - proposal to remove delay(0) from the example code
- #8291 - discussion related to the run_scheduled_recurrent_functions() usage in LwIP Ethernet
- #8317 - yieldUntil() implementation, similar to the esp_delay() overload with a timeout and a 0 interval

cores/esp8266/Esp.cpp

@@ -115,20 +115,18 @@ void EspClass::wdtFeed(void)
     system_soft_wdt_feed();
 }
 
-extern "C" void esp_yield();
-
 void EspClass::deepSleep(uint64_t time_us, WakeMode mode)
 {
     system_deep_sleep_set_option(static_cast<int>(mode));
     system_deep_sleep(time_us);
-    esp_yield();
+    esp_suspend();
 }
 
 void EspClass::deepSleepInstant(uint64_t time_us, WakeMode mode)
 {
     system_deep_sleep_set_option(static_cast<int>(mode));
     system_deep_sleep_instant(time_us);
-    esp_yield();
+    esp_suspend();
 }
 
 //this calculation was taken verbatim from the SDK api reference for SDK 2.1.0.
@@ -200,7 +198,7 @@ void EspClass::reset(void)
 void EspClass::restart(void)
 {
     system_restart();
-    esp_yield();
+    esp_suspend();
 }
 
 [[noreturn]] void EspClass::rebootIntoUartDownloadMode()

cores/esp8266/HardwareSerial.cpp

@@ -143,7 +143,7 @@ unsigned long HardwareSerial::detectBaudrate(time_t timeoutMillis)
         if ((detectedBaudrate = testBaudrate())) {
           break;
         }
-        delay(100);
+        esp_delay(100);
     }
     return detectedBaudrate;
 }

cores/esp8266/PolledTimeout.h

@@ -27,6 +27,7 @@
 #include <limits>                  // std::numeric_limits
 #include <type_traits>             // std::is_unsigned
 #include <core_esp8266_features.h>
+#include <coredecls.h>
 
 namespace esp8266
 {
@@ -45,7 +46,7 @@ struct DoNothing
 
 struct YieldOrSkip
 {
-  static void execute() {delay(0);}
+  static void execute() {esp_yield();}
 };
 
 template <unsigned long delayMs>

cores/esp8266/Schedule.cpp

@@ -135,8 +135,6 @@ bool schedule_recurrent_function_us(const std::function<bool(void)>& fn,
 
 void run_scheduled_functions()
 {
-    esp8266::polledTimeout::periodicFastMs yieldNow(100); // yield every 100ms
-
     // prevent scheduling of new functions during this run
     auto stop = sLast;
     bool done = false;
@@ -161,13 +159,10 @@ void run_scheduled_functions()
             recycle_fn_unsafe(to_recycle);
         }
 
-        if (yieldNow)
-        {
-            // because scheduled functions might last too long for watchdog etc,
-            // this is yield() in cont stack:
-            esp_schedule();
-            cont_yield(g_pcont);
-        }
+        // scheduled functions might last too long for watchdog etc.
+        // yield() is allowed in scheduled functions, therefore
+        // recursion into run_scheduled_recurrent_functions() is permitted
+        optimistic_yield(100000);
     }
 }
 
@@ -241,9 +236,10 @@ void run_scheduled_recurrent_functions()
         if (yieldNow)
         {
             // because scheduled functions might last too long for watchdog etc,
-            // this is yield() in cont stack:
+            // this is yield() in cont stack, but need to call cont_suspend directly
+            // to prevent recursion into run_scheduled_recurrent_functions()
             esp_schedule();
-            cont_yield(g_pcont);
+            cont_suspend(g_pcont);
         }
     } while (current && !done);
 

cores/esp8266/cont.S

@@ -21,9 +21,9 @@
     .section .irom0.text
     .align    4
     .literal_position
-    .global   cont_yield
-    .type     cont_yield, @function
-cont_yield:
+    .global   cont_suspend
+    .type     cont_suspend, @function
+cont_suspend:
     /* a1: sp */
     /* a2: void* cont_ctx */
     /* adjust stack and save registers */
@@ -35,10 +35,10 @@ cont_yield:
     s32i    a0,  a1, 16
     s32i    a2,  a1, 20
 
-    /* &cont_continue -> cont_ctx.pc_yield */
+    /* &cont_continue -> cont_ctx.pc_suspend */
     movi    a3, cont_continue
     s32i    a3, a2, 8
-    /* sp -> cont_ctx.sp_yield */
+    /* sp -> cont_ctx.sp_suspend */
     s32i    a1, a2, 12
 
     /* a0 <- cont_ctx.pc_ret */
@@ -56,7 +56,7 @@ cont_continue:
     l32i    a2,  a1, 20
     addi    a1,  a1, 24
     ret
-    .size    cont_yield, . - cont_yield
+    .size    cont_suspend, . - cont_suspend
 
 ////////////////////////////////////////////////////
 
@@ -108,7 +108,7 @@ cont_run:
     /* sp -> cont_ctx.sp_ret */
     s32i    a1, a2, 4
 
-    /* if cont_ctx.pc_yield != 0, goto cont_resume */
+    /* if cont_ctx.pc_suspend != 0, goto cont_resume */
     l32i    a4, a2, 8
     bnez    a4, cont_resume
     /* else */
@@ -119,12 +119,12 @@ cont_run:
     jx      a2
 
 cont_resume:
-    /* a1 <- cont_ctx.sp_yield */
+    /* a1 <- cont_ctx.sp_suspend */
     l32i    a1, a2, 12
-    /* reset yield flag, 0 -> cont_ctx.pc_yield */
+    /* reset yield flag, 0 -> cont_ctx.pc_suspend */
     movi    a3, 0
     s32i    a3, a2, 8
-    /* jump to saved cont_ctx.pc_yield */
+    /* jump to saved cont_ctx.pc_suspend */
     movi    a0, cont_ret
     jx       a4
 

cores/esp8266/cont.h

@@ -35,8 +35,8 @@ typedef struct cont_ {
         void (*pc_ret)(void);
         unsigned* sp_ret;
 
-        void (*pc_yield)(void);
-        unsigned* sp_yield;
+        void (*pc_suspend)(void);
+        unsigned* sp_suspend;
 
         unsigned* stack_end;
         unsigned unused1;
@@ -55,12 +55,12 @@ extern cont_t* g_pcont;
 void cont_init(cont_t*);
 
 // Run function pfn in a separate stack, or continue execution
-// at the point where cont_yield was called
+// at the point where cont_suspend was called
 void cont_run(cont_t*, void (*pfn)(void));
 
 // Return to the point where cont_run was called, saving the
 // execution state (registers and stack)
-void cont_yield(cont_t*);
+void cont_suspend(cont_t*);
 
 // Check guard bytes around the stack. Return 0 in case everything is ok,
 // return 1 if guard bytes were overwritten.
@@ -70,9 +70,9 @@ int cont_check(cont_t* cont);
 // and thus weren't used by the user code. i.e. that stack space is free. (high water mark)
 int cont_get_free_stack(cont_t* cont);
 
-// Check if yield() may be called. Returns true if we are running inside
+// Check if cont_suspend() may be called. Returns true if we are running inside
 // continuation stack
-bool cont_can_yield(cont_t* cont);
+bool cont_can_suspend(cont_t* cont);
 
 // Repaint the stack from the current SP to the end, to allow individual
 // routines' stack usages to be calculated by re-painting, checking current

cores/esp8266/cont_util.cpp

@@ -62,9 +62,9 @@ int cont_get_free_stack(cont_t* cont) {
     return freeWords * 4;
 }
 
-bool IRAM_ATTR cont_can_yield(cont_t* cont) {
+bool IRAM_ATTR cont_can_suspend(cont_t* cont) {
     return !ETS_INTR_WITHINISR() &&
-           cont->pc_ret != 0 && cont->pc_yield == 0;
+           cont->pc_ret != 0 && cont->pc_suspend == 0;
 }
 
 // No need for this to be in IRAM, not expected to be IRQ called

cores/esp8266/core_esp8266_main.cpp

@@ -62,7 +62,7 @@ cont_t* g_pcont __attribute__((section(".noinit")));
 static os_event_t s_loop_queue[LOOP_QUEUE_SIZE];
 
 /* Used to implement optimistic_yield */
-static uint32_t s_cycles_at_yield_start;
+static uint32_t s_cycles_at_resume;
 
 /* For ets_intr_lock_nest / ets_intr_unlock_nest
  * Max nesting seen by SDK so far is 2.
@@ -80,6 +80,10 @@ const char* core_release =
 #else
     NULL;
 #endif
+
+static os_timer_t delay_timer;
+#define ONCE 0
+#define REPEAT 1
 } // extern "C"
 
 void initVariant() __attribute__((weak));
@@ -106,32 +110,71 @@ extern "C" void __preloop_update_frequency() {
 extern "C" void preloop_update_frequency() __attribute__((weak, alias("__preloop_update_frequency")));
 
 extern "C" bool can_yield() {
-  return cont_can_yield(g_pcont);
+  return cont_can_suspend(g_pcont);
 }
 
-static inline void esp_yield_within_cont() __attribute__((always_inline));
-static void esp_yield_within_cont() {
-        cont_yield(g_pcont);
-        s_cycles_at_yield_start = ESP.getCycleCount();
+static inline void esp_suspend_within_cont() __attribute__((always_inline));
+static void esp_suspend_within_cont() {
+        cont_suspend(g_pcont);
+        s_cycles_at_resume = ESP.getCycleCount();
         run_scheduled_recurrent_functions();
 }
 
-extern "C" void __esp_yield() {
-    if (can_yield()) {
-        esp_yield_within_cont();
+extern "C" void __esp_suspend() {
+    if (cont_can_suspend(g_pcont)) {
+        esp_suspend_within_cont();
     }
 }
 
-extern "C" void esp_yield() __attribute__ ((weak, alias("__esp_yield")));
+extern "C" void esp_suspend() __attribute__ ((weak, alias("__esp_suspend")));
 
 extern "C" IRAM_ATTR void esp_schedule() {
     ets_post(LOOP_TASK_PRIORITY, 0, 0);
 }
 
-extern "C" void __yield() {
-    if (can_yield()) {
+// Replacement for delay(0). In CONT, same as yield(). Whereas yield() panics
+// in SYS, esp_yield() is safe to call and only schedules CONT. Use yield()
+// whereever only called from CONT, use esp_yield() if code is called from SYS
+// or both CONT and SYS.
+extern "C" void esp_yield() {
+    esp_schedule();
+    esp_suspend();
+}
+
+void delay_end(void* arg) {
+    (void)arg;
+    esp_schedule();
+}
+
+extern "C" void __esp_delay(unsigned long ms) {
+    if (ms) {
+        os_timer_setfn(&delay_timer, (os_timer_func_t*)&delay_end, 0);
+        os_timer_arm(&delay_timer, ms, ONCE);
+    }
+    else {
         esp_schedule();
-        esp_yield_within_cont();
+    }
+    esp_suspend();
+    if (ms) {
+        os_timer_disarm(&delay_timer);
+    }
+}
+
+extern "C" void esp_delay(unsigned long ms) __attribute__((weak, alias("__esp_delay")));
+
+bool esp_try_delay(const uint32_t start_ms, const uint32_t timeout_ms, const uint32_t intvl_ms) {
+    uint32_t expired = millis() - start_ms;
+    if (expired >= timeout_ms) {
+        return true;
+    }
+    esp_delay(std::min((timeout_ms - expired), intvl_ms));
+    return false;
+}
+
+extern "C" void __yield() {
+    if (cont_can_suspend(g_pcont)) {
+        esp_schedule();
+        esp_suspend_within_cont();
     }
     else {
         panic();
@@ -140,6 +183,9 @@ extern "C" void __yield() {
 
 extern "C" void yield(void) __attribute__ ((weak, alias("__yield")));
 
+// In CONT, actually performs yield() only once the given time interval
+// has elapsed since the last time yield() occured. Whereas yield() panics
+// in SYS, optimistic_yield() additionally is safe to call and does nothing.
 extern "C" void optimistic_yield(uint32_t interval_us) {
     const uint32_t intvl_cycles = interval_us *
 #if defined(F_CPU)
@@ -147,7 +193,7 @@ extern "C" void optimistic_yield(uint32_t interval_us) {
 #else
         ESP.getCpuFreqMHz();
 #endif
-    if ((ESP.getCycleCount() - s_cycles_at_yield_start) > intvl_cycles &&
+    if ((ESP.getCycleCount() - s_cycles_at_resume) > intvl_cycles &&
         can_yield())
     {
         yield();
@@ -207,7 +253,7 @@ static void loop_wrapper() {
 
 static void loop_task(os_event_t *events) {
     (void) events;
-    s_cycles_at_yield_start = ESP.getCycleCount();
+    s_cycles_at_resume = ESP.getCycleCount();
     ESP.resetHeap();
     cont_run(g_pcont, &loop_wrapper);
     ESP.setDramHeap();
@@ -215,8 +261,8 @@ static void loop_task(os_event_t *events) {
         panic();
     }
 }
-extern "C" {
 
+extern "C" {
 struct object { long placeholder[ 10 ]; };
 void __register_frame_info (const void *begin, struct object *ob);
 extern char __eh_frame[];
@@ -253,7 +299,6 @@ static void  __unhandled_exception_cpp()
     }
 #endif
 }
-
 }
 
 void init_done() {

cores/esp8266/core_esp8266_postmortem.cpp

@@ -245,12 +245,12 @@ static void print_stack(uint32_t start, uint32_t end) {
     }
 }
 
-static void uart_write_char_d(char c) {
+static void IRAM_ATTR uart_write_char_d(char c) {
     uart0_write_char_d(c);
     uart1_write_char_d(c);
 }
 
-static void uart0_write_char_d(char c) {
+static void IRAM_ATTR uart0_write_char_d(char c) {
     while (((USS(0) >> USTXC) & 0xff)) { }
 
     if (c == '\n') {
@@ -259,7 +259,7 @@ static void uart0_write_char_d(char c) {
     USF(0) = c;
 }
 
-static void uart1_write_char_d(char c) {
+static void IRAM_ATTR uart1_write_char_d(char c) {
     while (((USS(1) >> USTXC) & 0xff) >= 0x7e) { }
 
     if (c == '\n') {

cores/esp8266/core_esp8266_waveform_phase.cpp

@@ -211,7 +211,7 @@ int startWaveformClockCycles_weak(uint8_t pin, uint32_t highCcys, uint32_t lowCc
   }
   std::atomic_thread_fence(std::memory_order_acq_rel);
   while (waveform.toSetBits) {
-    delay(0); // Wait for waveform to update
+    esp_yield(); // Wait for waveform to update
     std::atomic_thread_fence(std::memory_order_acquire);
   }
   return true;

cores/esp8266/core_esp8266_waveform_pwm.cpp

@@ -40,6 +40,7 @@
 
 
 #include <Arduino.h>
+#include <coredecls.h>
 #include "ets_sys.h"
 #include "core_esp8266_waveform.h"
 #include "user_interface.h"
@@ -162,7 +163,7 @@ static IRAM_ATTR void _notifyPWM(PWMState *p, bool idle) {
   forceTimerInterrupt();
   while (pwmState.pwmUpdate) {
     if (idle) {
-      delay(0);
+      esp_yield();
     }
     MEMBARRIER();
   }
@@ -372,8 +373,8 @@ int startWaveformClockCycles_weak(uint8_t pin, uint32_t timeHighCycles, uint32_t
   if (wvfState.waveformEnabled & mask) {
     // Make sure no waveform changes are waiting to be applied
     while (wvfState.waveformToChange) {
-      delay(0); // Wait for waveform to update
-      // No mem barrier here, the call to a global function implies global state updated
+      esp_yield(); // Wait for waveform to update
+      MEMBARRIER();
     }
     wvfState.waveformNewHigh = timeHighCycles;
     wvfState.waveformNewLow = timeLowCycles;
@@ -392,8 +393,8 @@ int startWaveformClockCycles_weak(uint8_t pin, uint32_t timeHighCycles, uint32_t
     initTimer();
     forceTimerInterrupt();
     while (wvfState.waveformToEnable) {
-      delay(0); // Wait for waveform to update
-      // No mem barrier here, the call to a global function implies global state updated
+      esp_yield(); // Wait for waveform to update
+      MEMBARRIER();
     }
   }
 

cores/esp8266/core_esp8266_wiring.cpp

@@ -23,37 +23,18 @@
 #include "ets_sys.h"
 #include "osapi.h"
 #include "user_interface.h"
-#include "cont.h"
+#include "coredecls.h"
 
 extern "C" {
 
-extern void ets_delay_us(uint32_t us);
-extern void esp_schedule();
-extern void esp_yield();
-
-static os_timer_t delay_timer;
 static os_timer_t micros_overflow_timer;
 static uint32_t micros_at_last_overflow_tick = 0;
 static uint32_t micros_overflow_count = 0;
 #define ONCE 0
 #define REPEAT 1
 
-void delay_end(void* arg) {
-    (void) arg;
-    esp_schedule();
-}
-
 void __delay(unsigned long ms) {
-    if(ms) {
-        os_timer_setfn(&delay_timer, (os_timer_func_t*) &delay_end, 0);
-        os_timer_arm(&delay_timer, ms, ONCE);
-    } else {
-        esp_schedule();
-    }
-    esp_yield();
-    if(ms) {
-        os_timer_disarm(&delay_timer);
-    }
+    esp_delay(ms);
 }
 
 void delay(unsigned long ms) __attribute__ ((weak, alias("__delay"))); 

cores/esp8266/coredecls.h

@@ -2,6 +2,10 @@
 #ifndef __COREDECLS_H
 #define __COREDECLS_H
 
+#define HAVE_ESP_SUSPEND 1
+
+#include "core_esp8266_features.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -13,8 +17,10 @@ extern "C" {
 #include <cont.h> // g_pcont declaration
 
 bool can_yield();
-void esp_yield();
+void esp_suspend();
+void esp_delay(unsigned long ms);
 void esp_schedule();
+void esp_yield();
 void tune_timeshift64 (uint64_t now_us);
 void disable_extra4k_at_link_time (void) __attribute__((noinline));
 bool sntp_set_timezone_in_seconds(int32_t timezone);
@@ -32,9 +38,45 @@ uint32_t crc32 (const void* data, size_t length, uint32_t crc = 0xffffffff);
 using BoolCB = std::function<void(bool)>;
 using TrivialCB = std::function<void()>;
 
+void settimeofday_cb (BoolCB&& cb);
 void settimeofday_cb (const BoolCB& cb);
 void settimeofday_cb (const TrivialCB& cb);
 
+// This overload of esp_suspend() performs the blocked callback whenever it is resumed,
+// and if that returns true, it immediately suspends again.
+template <typename T>
+inline void esp_suspend(T&& blocked) {
+    do {
+        esp_suspend();
+    } while (blocked());
+}
+
+// Try to delay until timeout_ms has expired since start_ms.
+// Returns true if timeout_ms has completely expired on entry.
+// Otherwise returns false after delaying for the relative
+// remainder of timeout_ms, or an absolute intvl_ms, whichever is shorter.
+// The delay may be asynchronously cancelled, before that timeout is reached.
+bool esp_try_delay(const uint32_t start_ms, const uint32_t timeout_ms, const uint32_t intvl_ms);
+
+// This overload of esp_delay() delays for a duration of at most timeout_ms milliseconds.
+// Whenever it is resumed, as well as every intvl_ms millisconds, it performs
+// the blocked callback, and if that returns true, it keeps delaying for the remainder
+// of the original timeout_ms period.
+template <typename T>
+inline void esp_delay(const uint32_t timeout_ms, T&& blocked, const uint32_t intvl_ms) {
+    const auto start_ms = millis();
+    while (!esp_try_delay(start_ms, timeout_ms, intvl_ms) && blocked()) {
+    }
+}
+
+// This overload of esp_delay() delays for a duration of at most timeout_ms milliseconds.
+// Whenever it is resumed, it performs the blocked callback, and if that returns true,
+// it keeps delaying for the remainder of the original timeout_ms period.
+template <typename T>
+inline void esp_delay(const uint32_t timeout_ms, T&& blocked) {
+    esp_delay(timeout_ms, std::forward<T>(blocked), timeout_ms);
+}
+
 #endif // __cplusplus
 
 #endif // __COREDECLS_H

cores/esp8266/hwdt_app_entry.cpp

@@ -952,13 +952,13 @@ STATIC void IRAM_MAYBE handle_hwdt(void) {
                 /* Print separate ctx: cont stack */
 
                 /* Check if cont stack is yielding to SYS */
-                if (0 == hwdt_info.cont_integrity && 0 != g_pcont->pc_yield) {
-                    ctx_cont_ptr = (const uint32_t *)((uintptr_t)g_pcont->sp_yield - 8u);
+                if (0 == hwdt_info.cont_integrity && 0 != g_pcont->pc_suspend) {
+                    ctx_cont_ptr = (const uint32_t *)((uintptr_t)g_pcont->sp_suspend - 8u);
                 }
                 print_stack((uintptr_t)ctx_cont_ptr, (uintptr_t)g_pcont->stack_end, PRINT_STACK::CONT);
             } else {
-                if (0 == hwdt_info.cont_integrity && 0 != g_pcont->pc_yield) {
-                    ETS_PRINTF("\nCont stack is yielding. Active stack starts at 0x%08X.\n", (uint32_t)g_pcont->sp_yield - 8u);
+                if (0 == hwdt_info.cont_integrity && 0 != g_pcont->pc_suspend) {
+                    ETS_PRINTF("\nCont stack is yielding. Active stack starts at 0x%08X.\n", (uint32_t)g_pcont->sp_suspend - 8u);
                 }
             }
 

cores/esp8266/time.cpp

@@ -214,6 +214,11 @@ void settimeofday_cb (const TrivialCB& cb)
     _settimeofday_cb = [cb](bool sntp) { (void)sntp; cb(); };
 }
 
+void settimeofday_cb (BoolCB&& cb)
+{
+    _settimeofday_cb = std::move(cb);
+}
+
 void settimeofday_cb (const BoolCB& cb)
 {
     _settimeofday_cb = cb;

cores/esp8266/uart.cpp

@@ -41,6 +41,7 @@
  *
  */
 #include "Arduino.h"
+#include "coredecls.h"
 #include <pgmspace.h>
 #include "gdb_hooks.h"
 #include "uart.h"
@@ -493,13 +494,13 @@ uart_stop_isr(uart_t* uart)
   -tools/sdk/uart_register.h
   -cores/esp8266/esp8266_peri.h
   */
-inline size_t
+inline __attribute__((always_inline)) size_t
 uart_tx_fifo_available(const int uart_nr)
 {
     return (USS(uart_nr) >> USTXC) & 0xff;
 }
 
-inline bool
+inline __attribute__((always_inline)) bool
 uart_tx_fifo_full(const int uart_nr)
 {
     return uart_tx_fifo_available(uart_nr) >= 0x7f;
@@ -566,7 +567,7 @@ uart_wait_tx_empty(uart_t* uart)
         return;
 
     while(uart_tx_fifo_available(uart->uart_nr) > 0)
-        delay(0);
+        esp_yield();
 
 }
 
@@ -943,23 +944,23 @@ uart_ignore_char(char c)
     (void) c;
 }
 
-inline void
+inline __attribute__((always_inline)) void
 uart_write_char_delay(const int uart_nr, char c)
 {
     while(uart_tx_fifo_full(uart_nr))
-        delay(0);
+        esp_yield();
 
     USF(uart_nr) = c;
 
 }
 
-static void
+static void IRAM_ATTR
 uart0_write_char(char c)
 {
     uart_write_char_delay(0, c);
 }
 
-static void
+static void IRAM_ATTR
 uart1_write_char(char c)
 {
     uart_write_char_delay(1, c);