WLED/wled00/file.cpp

412 lines
12 KiB
C++
Raw Normal View History

#include "wled.h"
/*
* Utility for SPIFFS filesystem
*/
2020-11-08 23:44:10 +01:00
#ifdef ARDUINO_ARCH_ESP32 //FS info bare IDF function until FS wrapper is available for ESP32
#if WLED_FS != LITTLEFS && ESP_IDF_VERSION_MAJOR < 4
2020-11-08 23:44:10 +01:00
#include "esp_spiffs.h"
#endif
2020-10-07 17:48:22 +02:00
#endif
2020-10-13 01:39:34 +02:00
#define FS_BUFSIZE 256
/*
* Structural requirements for files managed by writeObjectToFile() and readObjectFromFile() utilities:
* 1. File must be a string representation of a valid JSON object
* 2. File must have '{' as first character
* 3. There must not be any additional characters between a root-level key and its value object (e.g. space, tab, newline)
* 4. There must not be any characters between an root object-separating ',' and the next object key string
* 5. There may be any number of spaces, tabs, and/or newlines before such object-separating ','
* 6. There must not be more than 5 consecutive spaces at any point except for those permitted in condition 5
* 7. If it is desired to delete the first usable object (e.g. preset file), a dummy object '"0":{}' is inserted at the beginning.
* It shall be disregarded by receiving software.
* The reason for it is that deleting the first preset would require special code to handle commas between it and the 2nd preset
*/
2020-10-13 01:39:34 +02:00
// There are no consecutive spaces longer than this in the file, so if more space is required, findSpace() can return false immediately
// Actual space may be lower
uint16_t knownLargestSpace = UINT16_MAX;
File f;
//wrapper to find out how long closing takes
void closeFile() {
2021-02-27 01:16:06 +01:00
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINT(F("Close -> "));
uint32_t s = millis();
#endif
2020-10-13 01:39:34 +02:00
f.close();
DEBUGFS_PRINTF("took %d ms\n", millis() - s);
doCloseFile = false;
}
2020-05-08 22:53:59 +02:00
//find() that reads and buffers data from file stream in 256-byte blocks.
//Significantly faster, f.find(key) can take SECONDS for multi-kB files
2020-10-13 01:39:34 +02:00
bool bufferedFind(const char *target, bool fromStart = true) {
2020-09-13 22:00:47 +02:00
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINT("Find ");
DEBUGFS_PRINTLN(target);
uint32_t s = millis();
#endif
2020-09-07 20:39:12 +02:00
if (!f || !f.size()) return false;
2020-05-05 09:01:09 +02:00
size_t targetLen = strlen(target);
2020-09-13 22:00:47 +02:00
2020-05-05 09:01:09 +02:00
size_t index = 0;
byte buf[FS_BUFSIZE];
2020-10-13 01:39:34 +02:00
if (fromStart) f.seek(0);
2020-05-05 09:01:09 +02:00
2020-05-08 22:53:59 +02:00
while (f.position() < f.size() -1) {
uint16_t bufsize = f.read(buf, FS_BUFSIZE);
uint16_t count = 0;
2020-05-08 22:53:59 +02:00
while (count < bufsize) {
if(buf[count] != target[index])
2020-05-05 09:01:09 +02:00
index = 0; // reset index if any char does not match
2020-05-08 22:53:59 +02:00
if(buf[count] == target[index]) {
if(++index >= targetLen) { // return true if all chars in the target match
f.seek((f.position() - bufsize) + count +1);
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
2020-05-08 22:53:59 +02:00
return true;
}
2020-05-05 09:01:09 +02:00
}
2020-05-08 22:53:59 +02:00
count++;
2020-05-05 09:01:09 +02:00
}
}
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
2020-05-05 09:01:09 +02:00
return false;
}
2020-05-08 22:53:59 +02:00
//find empty spots in file stream in 256-byte blocks.
2020-10-13 01:39:34 +02:00
bool bufferedFindSpace(uint16_t targetLen, bool fromStart = true) {
2020-09-13 22:00:47 +02:00
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTF("Find %d spaces\n", targetLen);
uint32_t s = millis();
#endif
2020-10-13 01:39:34 +02:00
if (knownLargestSpace < targetLen) {
DEBUGFS_PRINT(F("No match, KLS "));
DEBUGFS_PRINTLN(knownLargestSpace);
return false;
}
2020-09-07 20:39:12 +02:00
if (!f || !f.size()) return false;
2020-09-13 22:00:47 +02:00
2020-09-07 20:39:12 +02:00
uint16_t index = 0;
byte buf[FS_BUFSIZE];
2020-10-13 01:39:34 +02:00
if (fromStart) f.seek(0);
2020-05-08 22:53:59 +02:00
while (f.position() < f.size() -1) {
2021-05-11 23:21:57 +02:00
uint16_t bufsize = f.read(buf, FS_BUFSIZE);
uint16_t count = 0;
2020-09-13 22:00:47 +02:00
2020-05-08 22:53:59 +02:00
while (count < bufsize) {
if(buf[count] == ' ') {
if(++index >= targetLen) { // return true if space long enough
2020-10-13 01:39:34 +02:00
if (fromStart) {
f.seek((f.position() - bufsize) + count +1 - targetLen);
knownLargestSpace = UINT16_MAX; //there may be larger spaces after, so we don't know
}
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("Found at pos %d, took %d ms", f.position(), millis() - s);
2020-05-08 22:53:59 +02:00
return true;
}
2020-10-08 00:52:15 +02:00
} else {
2020-10-13 01:39:34 +02:00
if (!fromStart) return false;
if (index) {
if (knownLargestSpace < index || knownLargestSpace == UINT16_MAX) knownLargestSpace = index;
index = 0; // reset index if not space
}
2020-05-08 22:53:59 +02:00
}
2020-10-08 00:52:15 +02:00
2020-05-08 22:53:59 +02:00
count++;
}
}
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
2020-05-08 22:53:59 +02:00
return false;
}
//find the closing bracket corresponding to the opening bracket at the file pos when calling this function
2020-10-13 01:39:34 +02:00
bool bufferedFindObjectEnd() {
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTLN(F("Find obj end"));
uint32_t s = millis();
#endif
if (!f || !f.size()) return false;
uint16_t objDepth = 0; //num of '{' minus num of '}'. return once 0
//size_t start = f.position();
2020-10-08 00:52:15 +02:00
byte buf[FS_BUFSIZE];
while (f.position() < f.size() -1) {
2021-05-11 23:21:57 +02:00
uint16_t bufsize = f.read(buf, FS_BUFSIZE);
uint16_t count = 0;
while (count < bufsize) {
if (buf[count] == '{') objDepth++;
if (buf[count] == '}') objDepth--;
if (objDepth == 0) {
f.seek((f.position() - bufsize) + count +1);
DEBUGFS_PRINTF("} at pos %d, took %d ms", f.position(), millis() - s);
return true;
}
count++;
}
}
DEBUGFS_PRINTF("No match, took %d ms\n", millis() - s);
return false;
}
//fills n bytes from current file pos with ' ' characters
2020-10-13 01:39:34 +02:00
void writeSpace(uint16_t l)
{
byte buf[FS_BUFSIZE];
memset(buf, ' ', FS_BUFSIZE);
while (l > 0) {
uint16_t block = (l>FS_BUFSIZE) ? FS_BUFSIZE : l;
f.write(buf, block);
l -= block;
}
2020-10-13 01:39:34 +02:00
if (knownLargestSpace < l) knownLargestSpace = l;
}
2020-10-13 01:39:34 +02:00
bool appendObjectToFile(const char* key, JsonDocument* content, uint32_t s, uint32_t contentLen = 0)
2020-09-13 22:00:47 +02:00
{
#ifdef WLED_DEBUG_FS
DEBUGFS_PRINTLN(F("Append"));
2020-09-13 22:00:47 +02:00
uint32_t s1 = millis();
#endif
uint32_t pos = 0;
if (!f) return false;
if (f.size() < 3) {
char init[10];
strcpy_P(init, PSTR("{\"0\":{}}"));
f.print(init);
}
if (content->isNull()) {
2020-10-13 01:39:34 +02:00
doCloseFile = true;
return true; //nothing to append
}
2020-09-13 22:00:47 +02:00
//if there is enough empty space in file, insert there instead of appending
2020-10-13 01:39:34 +02:00
if (!contentLen) contentLen = measureJson(*content);
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("CLen %d\n", contentLen);
2020-10-13 01:39:34 +02:00
if (bufferedFindSpace(contentLen + strlen(key) + 1)) {
2020-09-13 22:00:47 +02:00
if (f.position() > 2) f.write(','); //add comma if not first object
f.print(key);
serializeJson(*content, f);
2020-09-14 00:31:38 +02:00
DEBUGFS_PRINTF("Inserted, took %d ms (total %d)", millis() - s1, millis() - s);
2020-10-13 01:39:34 +02:00
doCloseFile = true;
2020-09-13 22:00:47 +02:00
return true;
}
2020-10-07 17:48:22 +02:00
//not enough space, append at end
2020-10-16 23:59:34 +02:00
//permitted space for presets exceeded
updateFSInfo();
if (f.size() + 9000 > (fsBytesTotal - fsBytesUsed)) { //make sure there is enough space to at least copy the file once
2020-10-07 17:48:22 +02:00
errorFlag = ERR_FS_QUOTA;
2020-10-13 01:39:34 +02:00
doCloseFile = true;
2020-10-07 17:48:22 +02:00
return false;
}
2020-09-13 22:00:47 +02:00
//check if last character in file is '}' (typical)
2020-11-15 20:58:04 +01:00
uint32_t eof = f.size() -1;
f.seek(eof, SeekSet);
if (f.read() == '}') pos = eof;
2020-09-13 22:00:47 +02:00
if (pos == 0) //not found
{
DEBUGFS_PRINTLN("not }");
2020-11-09 00:50:13 +01:00
f.seek(0);
while (bufferedFind("}",false)) //find last closing bracket in JSON if not last char
2020-09-13 22:00:47 +02:00
{
pos = f.position();
}
if (pos > 0) pos--;
2020-09-13 22:00:47 +02:00
}
DEBUGFS_PRINT("pos "); DEBUGFS_PRINTLN(pos);
if (pos > 2)
{
f.seek(pos, SeekSet);
2020-09-13 22:00:47 +02:00
f.write(',');
} else { //file content is not valid JSON object
f.seek(0, SeekSet);
2020-11-09 00:50:13 +01:00
f.print('{'); //start JSON
2020-09-13 22:00:47 +02:00
}
f.print(key);
//Append object
serializeJson(*content, f);
f.write('}');
2020-10-13 01:39:34 +02:00
doCloseFile = true;
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("Appended, took %d ms (total %d)", millis() - s1, millis() - s);
2020-10-07 17:48:22 +02:00
return true;
2020-09-13 22:00:47 +02:00
}
2020-05-08 22:53:59 +02:00
bool writeObjectToFileUsingId(const char* file, uint16_t id, JsonDocument* content)
2020-05-05 09:01:09 +02:00
{
char objKey[10];
sprintf(objKey, "\"%d\":", id);
2020-10-07 17:48:22 +02:00
return writeObjectToFile(file, objKey, content);
2020-05-05 09:01:09 +02:00
}
2020-05-08 22:53:59 +02:00
bool writeObjectToFile(const char* file, const char* key, JsonDocument* content)
2020-05-05 09:01:09 +02:00
{
2020-09-13 22:00:47 +02:00
uint32_t s = 0; //timing
#ifdef WLED_DEBUG_FS
2020-11-09 00:50:13 +01:00
DEBUGFS_PRINTF("Write to %s with key %s >>>\n", file, (key==nullptr)?"nullptr":key);
2020-09-13 22:00:47 +02:00
serializeJson(*content, Serial); DEBUGFS_PRINTLN();
s = millis();
#endif
2020-05-05 09:01:09 +02:00
uint32_t pos = 0;
2020-10-13 01:39:34 +02:00
f = WLED_FS.open(file, "r+");
2020-09-13 22:00:47 +02:00
if (!f && !WLED_FS.exists(file)) f = WLED_FS.open(file, "w+");
if (!f) {
DEBUGFS_PRINTLN(F("Failed to open!"));
2020-09-13 22:00:47 +02:00
return false;
}
2020-05-05 09:01:09 +02:00
2020-10-13 01:39:34 +02:00
if (!bufferedFind(key)) //key does not exist in file
2020-05-05 09:01:09 +02:00
{
2020-10-13 01:39:34 +02:00
return appendObjectToFile(key, content, s);
2020-05-05 09:01:09 +02:00
}
2020-10-13 01:39:34 +02:00
//an object with this key already exists, replace or delete it
2020-05-05 09:01:09 +02:00
pos = f.position();
//measure out end of old object
2020-10-13 01:39:34 +02:00
bufferedFindObjectEnd();
2020-05-05 09:01:09 +02:00
uint32_t pos2 = f.position();
2020-09-09 09:51:04 +02:00
2020-05-05 09:01:09 +02:00
uint32_t oldLen = pos2 - pos;
DEBUGFS_PRINTF("Old obj len %d\n", oldLen);
2020-10-13 01:39:34 +02:00
//Three cases:
//1. The new content is null, overwrite old obj with spaces
//2. The new content is smaller than the old, overwrite and fill diff with spaces
//3. The new content is larger than the old, but smaller than old + trailing spaces, overwrite with new
//4. The new content is larger than old + trailing spaces, delete old and append
2020-05-05 09:01:09 +02:00
2020-10-13 01:39:34 +02:00
uint32_t contentLen = 0;
if (!content->isNull()) contentLen = measureJson(*content);
if (contentLen && contentLen <= oldLen) { //replace and fill diff with spaces
DEBUGFS_PRINTLN(F("replace"));
2020-09-07 20:39:12 +02:00
f.seek(pos);
2020-05-08 22:53:59 +02:00
serializeJson(*content, f);
2020-10-13 01:39:34 +02:00
writeSpace(pos2 - f.position());
} else if (contentLen && bufferedFindSpace(contentLen - oldLen, false)) { //enough leading spaces to replace
DEBUGFS_PRINTLN(F("replace (trailing)"));
f.seek(pos);
serializeJson(*content, f);
} else {
DEBUGFS_PRINTLN(F("delete"));
2020-09-07 20:39:12 +02:00
pos -= strlen(key);
2020-09-09 09:51:04 +02:00
if (pos > 3) pos--; //also delete leading comma if not first object
2020-09-07 20:39:12 +02:00
f.seek(pos);
2020-10-13 01:39:34 +02:00
writeSpace(pos2 - pos);
if (contentLen) return appendObjectToFile(key, content, s, contentLen);
2020-05-05 09:01:09 +02:00
}
2020-10-13 01:39:34 +02:00
doCloseFile = true;
DEBUGFS_PRINTF("Replaced/deleted, took %d ms\n", millis() - s);
2020-09-13 22:00:47 +02:00
return true;
2020-05-05 09:01:09 +02:00
}
bool readObjectFromFileUsingId(const char* file, uint16_t id, JsonDocument* dest)
{
char objKey[10];
sprintf(objKey, "\"%d\":", id);
2020-10-07 17:48:22 +02:00
return readObjectFromFile(file, objKey, dest);
2020-05-05 09:01:09 +02:00
}
2020-11-04 17:17:54 +01:00
//if the key is a nullptr, deserialize entire object
2020-05-05 09:01:09 +02:00
bool readObjectFromFile(const char* file, const char* key, JsonDocument* dest)
{
2020-10-13 01:39:34 +02:00
if (doCloseFile) closeFile();
2020-09-13 22:00:47 +02:00
#ifdef WLED_DEBUG_FS
2020-11-09 00:50:13 +01:00
DEBUGFS_PRINTF("Read from %s with key %s >>>\n", file, (key==nullptr)?"nullptr":key);
2020-09-13 22:00:47 +02:00
uint32_t s = millis();
#endif
2020-10-13 01:39:34 +02:00
f = WLED_FS.open(file, "r");
2020-05-05 09:01:09 +02:00
if (!f) return false;
2020-09-09 09:51:04 +02:00
2020-11-04 17:17:54 +01:00
if (key != nullptr && !bufferedFind(key)) //key does not exist in file
2020-05-05 09:01:09 +02:00
{
f.close();
2020-10-13 01:39:34 +02:00
dest->clear();
2020-10-08 00:52:15 +02:00
DEBUGFS_PRINTLN(F("Obj not found."));
2020-05-05 09:01:09 +02:00
return false;
}
deserializeJson(*dest, f);
f.close();
2020-09-13 22:00:47 +02:00
DEBUGFS_PRINTF("Read, took %d ms\n", millis() - s);
2020-05-05 09:01:09 +02:00
return true;
}
2020-10-07 17:48:22 +02:00
void updateFSInfo() {
#ifdef ARDUINO_ARCH_ESP32
#if WLED_FS == LITTLEFS || ESP_IDF_VERSION_MAJOR >= 4
fsBytesTotal = WLED_FS.totalBytes();
fsBytesUsed = WLED_FS.usedBytes();
2020-11-08 23:44:10 +01:00
#else
2020-10-07 17:48:22 +02:00
esp_spiffs_info(nullptr, &fsBytesTotal, &fsBytesUsed);
2020-11-08 23:44:10 +01:00
#endif
2020-10-07 17:48:22 +02:00
#else
FSInfo fsi;
WLED_FS.info(fsi);
fsBytesUsed = fsi.usedBytes;
fsBytesTotal = fsi.totalBytes;
#endif
}
2019-03-16 02:09:37 +01:00
//Un-comment any file types you need
String getContentType(AsyncWebServerRequest* request, String filename){
if(request->hasArg("download")) return "application/octet-stream";
else if(filename.endsWith(".htm")) return "text/html";
else if(filename.endsWith(".html")) return "text/html";
2021-03-28 17:15:26 +02:00
else if(filename.endsWith(".css")) return "text/css";
else if(filename.endsWith(".js")) return "application/javascript";
2019-03-16 02:09:37 +01:00
else if(filename.endsWith(".json")) return "application/json";
else if(filename.endsWith(".png")) return "image/png";
else if(filename.endsWith(".gif")) return "image/gif";
else if(filename.endsWith(".jpg")) return "image/jpeg";
else if(filename.endsWith(".ico")) return "image/x-icon";
2019-03-16 02:09:37 +01:00
// else if(filename.endsWith(".xml")) return "text/xml";
// else if(filename.endsWith(".pdf")) return "application/x-pdf";
// else if(filename.endsWith(".zip")) return "application/x-zip";
// else if(filename.endsWith(".gz")) return "application/x-gzip";
return "text/plain";
}
2019-03-16 02:09:37 +01:00
bool handleFileRead(AsyncWebServerRequest* request, String path){
DEBUG_PRINTLN("FileRead: " + path);
if(path.endsWith("/")) path += "index.htm";
2020-10-07 17:48:22 +02:00
if(path.indexOf("sec") > -1) return false;
2019-03-16 02:09:37 +01:00
String contentType = getContentType(request, path);
2020-05-05 09:01:09 +02:00
/*String pathWithGz = path + ".gz";
2020-09-09 09:51:04 +02:00
if(WLED_FS.exists(pathWithGz)){
request->send(WLED_FS, pathWithGz, contentType);
return true;
2020-05-05 09:01:09 +02:00
}*/
2020-09-09 09:51:04 +02:00
if(WLED_FS.exists(path)) {
request->send(WLED_FS, path, contentType);
2019-03-16 02:09:37 +01:00
return true;
}
2019-03-16 02:09:37 +01:00
return false;
}