Add time to filesystem API (#6544)

* Add time to filesystem API

Support the ESP32 File::getLastWrite() call and setting the time on
all filesystems automatically (assuming the system clock has
been set properly and time(NULL) returns the proper time!).

Adds Dir::fileTime() to get the time of a file being listed, similar to
Dir::fileName() and Dir::fileSize().

Adds ::setTimeCallback(time_t (*cb)()) to File, Dir, and FS, allowing
users to override the default timestamp on a per-file, directory, or
filesystem basis. By default, a simple callback returning time(nullptr)
is implemented.

LittleFS uses the 't' attribute and should be backwards compatible.

SD/SDFS work and include wrappers for obsolete SdFat timestamp callbacks
using the MSDOS time.

This PR does not update SPIFFS, due to compatability concerns and a
possible massive rewrite which would make it possible to determine if an
old-style ot metadata enabled FS is present at mount time.

Includes an updated SD/listfiles and LittleFS_time example.

Replaces #6315

* Add links to new mklittlefs w/timestamp support

Include the update mklittlefs which generated 't' metadata on imported
files.
	../tools/sdk/lwip2/include/netif/lowpan6_opts.h

* Add explicit note about timestamp being local time

* Address review concerns

Clean up some awkward object instantiations.

Remove the _enableTime flag/setter from SPIFFS.

Clean up the FSConfig constructors using C++ style init lists.

libraries/LittleFS/examples/LittleFS_Timestamp/LittleFS_Timestamp.ino

@@ -0,0 +1,181 @@
+/* Example showing timestamp support in LittleFS */
+/* Released into the public domain. */
+/* Earle F. Philhower, III <earlephilhower@yahoo.com> */
+
+#include <FS.h>
+#include <LittleFS.h>
+#include <time.h>
+#include <ESP8266WiFi.h>
+
+#ifndef STASSID
+#define STASSID "your-ssid"
+#define STAPSK  "your-password"
+#endif
+
+const char *ssid = STASSID;
+const char *pass = STAPSK;
+
+long timezone = 2;
+byte daysavetime = 1;
+
+
+bool getLocalTime(struct tm * info, uint32_t ms) {
+  uint32_t count = ms / 10;
+  time_t now;
+
+  time(&now);
+  localtime_r(&now, info);
+
+  if (info->tm_year > (2016 - 1900)) {
+    return true;
+  }
+
+  while (count--) {
+    delay(10);
+    time(&now);
+    localtime_r(&now, info);
+    if (info->tm_year > (2016 - 1900)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+
+void listDir(const char * dirname) {
+  Serial.printf("Listing directory: %s\n", dirname);
+
+  Dir root = LittleFS.openDir(dirname);
+
+  while (root.next()) {
+    File file = root.openFile("r");
+    Serial.print("  FILE: ");
+    Serial.print(root.fileName());
+    Serial.print("  SIZE: ");
+    Serial.print(file.size());
+    time_t t = file.getLastWrite();
+    struct tm * tmstruct = localtime(&t);
+    file.close();
+    Serial.printf("  LAST WRITE: %d-%02d-%02d %02d:%02d:%02d\n", (tmstruct->tm_year) + 1900, (tmstruct->tm_mon) + 1, tmstruct->tm_mday, tmstruct->tm_hour, tmstruct->tm_min, tmstruct->tm_sec);
+  }
+}
+
+
+void readFile(const char * path) {
+  Serial.printf("Reading file: %s\n", path);
+
+  File file = LittleFS.open(path, "r");
+  if (!file) {
+    Serial.println("Failed to open file for reading");
+    return;
+  }
+
+  Serial.print("Read from file: ");
+  while (file.available()) {
+    Serial.write(file.read());
+  }
+  file.close();
+}
+
+void writeFile(const char * path, const char * message) {
+  Serial.printf("Writing file: %s\n", path);
+
+  File file = LittleFS.open(path, "w");
+  if (!file) {
+    Serial.println("Failed to open file for writing");
+    return;
+  }
+  if (file.print(message)) {
+    Serial.println("File written");
+  } else {
+    Serial.println("Write failed");
+  }
+  file.close();
+}
+
+void appendFile(const char * path, const char * message) {
+  Serial.printf("Appending to file: %s\n", path);
+
+  File file = LittleFS.open(path, "a");
+  if (!file) {
+    Serial.println("Failed to open file for appending");
+    return;
+  }
+  if (file.print(message)) {
+    Serial.println("Message appended");
+  } else {
+    Serial.println("Append failed");
+  }
+  file.close();
+}
+
+void renameFile(const char * path1, const char * path2) {
+  Serial.printf("Renaming file %s to %s\n", path1, path2);
+  if (LittleFS.rename(path1, path2)) {
+    Serial.println("File renamed");
+  } else {
+    Serial.println("Rename failed");
+  }
+}
+
+void deleteFile(const char * path) {
+  Serial.printf("Deleting file: %s\n", path);
+  if (LittleFS.remove(path)) {
+    Serial.println("File deleted");
+  } else {
+    Serial.println("Delete failed");
+  }
+}
+
+void setup() {
+  Serial.begin(115200);
+  // We start by connecting to a WiFi network
+  Serial.println();
+  Serial.println();
+  Serial.print("Connecting to ");
+  Serial.println(ssid);
+
+  WiFi.begin(ssid, pass);
+
+  while (WiFi.status() != WL_CONNECTED) {
+    delay(500);
+    Serial.print(".");
+  }
+  Serial.println("WiFi connected");
+  Serial.println("IP address: ");
+  Serial.println(WiFi.localIP());
+  Serial.println("Contacting Time Server");
+  configTime(3600 * timezone, daysavetime * 3600, "time.nist.gov", "0.pool.ntp.org", "1.pool.ntp.org");
+  struct tm tmstruct ;
+  delay(2000);
+  tmstruct.tm_year = 0;
+  getLocalTime(&tmstruct, 5000);
+  Serial.printf("\nNow is : %d-%02d-%02d %02d:%02d:%02d\n", (tmstruct.tm_year) + 1900, (tmstruct.tm_mon) + 1, tmstruct.tm_mday, tmstruct.tm_hour, tmstruct.tm_min, tmstruct.tm_sec);
+  Serial.println("");
+
+  Serial.printf("Formatting LittleFS filesystem\n");
+  LittleFS.format();
+  listDir("/");
+  deleteFile("/hello.txt");
+  writeFile("/hello.txt", "Hello ");
+  appendFile("/hello.txt", "World!\n");
+  listDir("/");
+
+  Serial.printf("The timestamp should be valid above\n");
+
+  Serial.printf("Now unmount and remount and perform the same operation.\n");
+  Serial.printf("Timestamp should be valid, data should be good.\n");
+  LittleFS.end();
+  Serial.printf("Now mount it\n");
+  if (!LittleFS.begin()) {
+    Serial.println("LittleFS mount failed");
+    return;
+  }
+  readFile("/hello.txt");
+  listDir("/");
+
+
+}
+
+void loop() { }
+

libraries/LittleFS/src/LittleFS.h

@@ -47,11 +47,8 @@ class LittleFSDirImpl;
 class LittleFSConfig : public FSConfig
 {
 public:
-    LittleFSConfig(bool autoFormat = true) {
-        _type = LittleFSConfig::fsid::FSId;
-        _autoFormat = autoFormat;
-    }
-    enum fsid { FSId = 0x4c495454 };
+    static constexpr uint32_t FSId = 0x4c495454;
+    LittleFSConfig(bool autoFormat = true) : FSConfig(FSId, autoFormat) { }
 };
 
 class LittleFSImpl : public FSImpl
@@ -176,7 +173,7 @@ public:
     }
 
     bool setConfig(const FSConfig &cfg) override {
-        if ((cfg._type != LittleFSConfig::fsid::FSId) || _mounted) {
+        if ((cfg._type != LittleFSConfig::FSId) || _mounted) {
             return false;
         }
         _cfg = *static_cast<const LittleFSConfig *>(&cfg);
@@ -422,9 +419,27 @@ public:
             lfs_file_close(_fs->getFS(), _getFD());
             _opened = false;
             DEBUGV("lfs_file_close: fd=%p\n", _getFD());
+	    if (timeCallback) {
+                // Add metadata with last write time
+                time_t now = timeCallback();
+                int rc = lfs_setattr(_fs->getFS(), _name.get(), 't', (const void *)&now, sizeof(now));
+                if (rc < 0) {
+                    DEBUGV("Unable to set time on '%s' to %d\n", _name.get(), now);
+               }
+            }
         }
     }
 
+    time_t getLastWrite() override {
+        time_t ftime = 0;
+        if (_opened && _fd) {
+            int rc = lfs_getattr(_fs->getFS(), _name.get(), 't', (void *)&ftime, sizeof(ftime));
+            if (rc != sizeof(ftime))
+                ftime = 0; // Error, so clear read value
+        }
+        return ftime;
+    }
+
     const char* name() const override {
         if (!_opened) {
             return nullptr;
@@ -520,6 +535,27 @@ public:
         return _dirent.size;
     }
 
+    time_t fileTime() override {
+        if (!_valid) {
+            return 0;
+        }
+        int nameLen = 3; // Slashes, terminator
+        nameLen += _dirPath.get() ? strlen(_dirPath.get()) : 0;
+        nameLen += strlen(_dirent.name);
+        char *tmpName = (char*)malloc(nameLen);
+        if (!tmpName) {
+            return 0;
+        }
+        snprintf(tmpName, nameLen, "%s%s%s", _dirPath.get() ? _dirPath.get() : "", _dirPath.get()&&_dirPath.get()[0]?"/":"", _dirent.name);
+        time_t ftime = 0;
+        int rc = lfs_getattr(_fs->getFS(), tmpName, 't', (void *)&ftime, sizeof(ftime));
+        if (rc != sizeof(ftime))
+            ftime = 0; // Error, so clear read value
+        free(tmpName);
+        return ftime;
+    }
+
+
     bool isFile() const override {
         return _valid && (_dirent.type == LFS_TYPE_REG);
     }

libraries/SD/examples/listfiles/listfiles.ino

@@ -28,14 +28,14 @@ File root;
 
 void setup() {
   // Open serial communications and wait for port to open:
-  Serial.begin(9600);
+  Serial.begin(115200);
   while (!Serial) {
     ; // wait for serial port to connect. Needed for Leonardo only
   }
 
   Serial.print("Initializing SD card...");
 
-  if (!SD.begin(4)) {
+  if (!SD.begin(SS)) {
     Serial.println("initialization failed!");
     return;
   }
@@ -70,11 +70,13 @@ void printDirectory(File dir, int numTabs) {
     } else {
       // files have sizes, directories do not
       Serial.print("\t\t");
-      Serial.println(entry.size(), DEC);
+      Serial.print(entry.size(), DEC);
+      Serial.print("\t\t");
+      time_t ft = entry.getLastWrite();
+      struct tm *tm = localtime(&ft);
+      // US format.  Feel free to convert to your own locale...
+      Serial.printf("%02d-%02d-%02d %02d:%02d:%02d\n", tm->tm_mon + 1, tm->tm_mday, tm->tm_year % 100, tm->tm_hour, tm->tm_min, tm->tm_sec);
     }
     entry.close();
   }
 }
-
-
-

libraries/SD/src/SD.cpp

@@ -3,3 +3,5 @@
 #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SD)
 SDClass SD;
 #endif
+
+void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*) = nullptr;

libraries/SD/src/SD.h

@@ -29,6 +29,7 @@
 #undef FILE_WRITE
 #define FILE_WRITE (sdfat::O_READ | sdfat::O_WRITE | sdfat::O_CREAT | sdfat::O_APPEND)
 
+
 class SDClass {
 public:
     boolean begin(uint8_t csPin, SPISettings cfg = SPI_HALF_SPEED) {
@@ -137,6 +138,17 @@ public:
         return ((uint64_t)clusterSize() * (uint64_t)totalClusters());
     }
 
+    void setTimeCallback(time_t (*cb)(void)) {
+        SDFS.setTimeCallback(cb);
+    }
+
+    // Wrapper to allow obsolete datetimecallback use, silently convert to time_t in wrappertimecb
+    void dateTimeCallback(void (*cb)(uint16_t*, uint16_t*)) {
+        extern void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*);
+        __SD__userDateTimeCB = cb;
+	SDFS.setTimeCallback(wrapperTimeCB);
+    }
+
 private:
     const char *getMode(uint8_t mode) {
         bool read = (mode & sdfat::O_READ) ? true : false;
@@ -150,8 +162,46 @@ private:
         else                                 { return "r";  }
     }
 
+    static time_t wrapperTimeCB(void) {
+        extern void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*);
+        if (__SD__userDateTimeCB) {
+            uint16_t d, t;
+            __SD__userDateTimeCB(&d, &t);
+            return sdfs::SDFSImpl::FatToTimeT(d, t);
+        }
+        return time(nullptr);
+    }
+
 };
 
+
+// Expose FatStructs.h helpers for MSDOS date/time for use with dateTimeCallback
+static inline uint16_t FAT_DATE(uint16_t year, uint8_t month, uint8_t day) {
+  return (year - 1980) << 9 | month << 5 | day;
+}
+static inline uint16_t FAT_YEAR(uint16_t fatDate) {
+  return 1980 + (fatDate >> 9);
+}
+static inline uint8_t FAT_MONTH(uint16_t fatDate) {
+  return (fatDate >> 5) & 0XF;
+}
+static inline uint8_t FAT_DAY(uint16_t fatDate) {
+  return fatDate & 0X1F;
+}
+static inline uint16_t FAT_TIME(uint8_t hour, uint8_t minute, uint8_t second) {
+  return hour << 11 | minute << 5 | second >> 1;
+}
+static inline uint8_t FAT_HOUR(uint16_t fatTime) {
+  return fatTime >> 11;
+}
+static inline uint8_t FAT_MINUTE(uint16_t fatTime) {
+  return (fatTime >> 5) & 0X3F;
+}
+static inline uint8_t FAT_SECOND(uint16_t fatTime) {
+  return 2*(fatTime & 0X1F);
+}
+
+
 #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SD)
 extern SDClass SD;
 #endif

libraries/SDFS/src/SDFS.h

@@ -45,22 +45,9 @@ class SDFSDirImpl;
 class SDFSConfig : public FSConfig
 {
 public:
-    SDFSConfig() {
-        _type = SDFSConfig::fsid::FSId;
-        _autoFormat = false;
-        _csPin = 4;
-        _spiSettings = SD_SCK_MHZ(10);
-	_part = 0;
-    }
-    SDFSConfig(uint8_t csPin, SPISettings spi) {
-        _type = SDFSConfig::fsid::FSId;
-        _autoFormat = false;
-        _csPin = csPin;
-        _spiSettings = spi;
-	_part = 0;
-    }
+    static constexpr uint32_t FSId = 0x53444653;
 
-    enum fsid { FSId = 0x53444653 };
+    SDFSConfig(uint8_t csPin = 4, SPISettings spi = SD_SCK_MHZ(10)) : FSConfig(FSId, false), _csPin(csPin), _part(0), _spiSettings(spi)  { }
 
     SDFSConfig setAutoFormat(bool val = true) {
         _autoFormat = val;
@@ -152,7 +139,7 @@ public:
 
     bool setConfig(const FSConfig &cfg) override
     {
-        if ((cfg._type != SDFSConfig::fsid::FSId) || _mounted) {
+        if ((cfg._type != SDFSConfig::FSId) || _mounted) {
             DEBUGV("SDFS::setConfig: invalid config or already mounted\n");
             return false;
         }
@@ -203,6 +190,20 @@ public:
         return (clusterSize() * totalClusters());
     }
 
+    // Helper function, takes FAT and makes standard time_t
+    static time_t FatToTimeT(uint16_t d, uint16_t t) {
+        struct tm tiempo;
+        memset(&tiempo, 0, sizeof(tiempo));
+        tiempo.tm_sec  = (((int)t) <<  1) & 0x3e;
+        tiempo.tm_min  = (((int)t) >>  5) & 0x3f;
+        tiempo.tm_hour = (((int)t) >> 11) & 0x1f;
+        tiempo.tm_mday = (int)(d & 0x1f);
+        tiempo.tm_mon  = ((int)(d >> 5) & 0x0f) - 1;
+        tiempo.tm_year = ((int)(d >> 9) & 0x7f) + 80;
+        tiempo.tm_isdst = -1;
+        return mktime(&tiempo);
+    }
+
 protected:
     friend class SDFileImpl;
     friend class SDFSDirImpl;
@@ -212,6 +213,7 @@ protected:
         return &_fs;
     }
 
+
     static uint8_t _getFlags(OpenMode openMode, AccessMode accessMode) {
         uint8_t mode = 0;
         if (openMode & OM_CREATE) {
@@ -350,6 +352,17 @@ public:
         return _opened ? _fd->isDirectory() : false;
     }
 
+    time_t getLastWrite() override {
+        time_t ftime = 0;
+        if (_opened && _fd) {
+            sdfat::dir_t tmp;
+            if (_fd.get()->dirEntry(&tmp)) {
+                ftime = SDFSImpl::FatToTimeT(tmp.lastWriteDate, tmp.lastWriteTime);
+            }
+        }
+        return ftime;
+    }
+
 
 protected:
     SDFSImpl*                     _fs;
@@ -425,6 +438,12 @@ public:
                 _size = file.fileSize();
                 _isFile = file.isFile();
                 _isDirectory = file.isDirectory();
+                sdfat::dir_t tmp;
+                if (file.dirEntry(&tmp)) {
+                    _time = SDFSImpl::FatToTimeT(tmp.lastWriteDate, tmp.lastWriteTime);
+		} else {
+                    _time = 0;
+               }
                 file.getName(_lfn, sizeof(_lfn));
                 file.close();
             } else {
@@ -447,6 +466,7 @@ protected:
     std::shared_ptr<sdfat::File> _dir;
     bool                         _valid;
     char                         _lfn[64];
+    time_t                       _time;
     std::shared_ptr<char>        _dirPath;
     uint32_t                     _size;
     bool                         _isFile;