Binary Websockets for Peek (#2516)

* Binary Websockets for Peek

* No IRAM_ATTR here

* Use builtin LittleFS for all ESP32 builds

* Attempt LittleFS compilation fix

* Use tasmota zip for all ESP32 builds

* Revert to Arduino Core 1 for the time being

platformio.ini

@@ -15,7 +15,7 @@
 default_envs = nodemcuv2, esp8266_2m, esp01_1m_full, esp32dev, esp32_eth, esp32s2_saola, esp32c3
 
 # Build everything
-; default_envs = esp32dev, esp8285_4CH_MagicHome, esp8285_4CH_H801, codm-controller-0.6-rev2, codm-controller-0.6, esp32s2_saola, d1_mini_5CH_Shojo_PCB, d1_mini, sp501e, travis_esp8266, travis_esp32, nodemcuv2, esp32_eth, anavi_miracle_controller, esp07, esp01_1m_full, m5atom, h803wf, d1_mini_ota, heltec_wifi_kit_8, esp8285_5CH_H801, d1_mini_debug, wemos_shield_esp32, elekstube_ips
+; default_envs = esp32dev, esp8285_4CH_MagicHome, codm-controller-0.6-rev2, codm-controller-0.6, esp32s2_saola, d1_mini_5CH_Shojo_PCB, d1_mini, sp501e, travis_esp8266, travis_esp32, nodemcuv2, esp32_eth, anavi_miracle_controller, esp07, esp01_1m_full, m5atom, h803wf, d1_mini_ota, heltec_wifi_kit_8, esp8285_H801, d1_mini_debug, wemos_shield_esp32, elekstube_ips
 
 # Single binaries (uncomment your board)
 ; default_envs = elekstube_ips
@@ -30,12 +30,12 @@ default_envs = nodemcuv2, esp8266_2m, esp01_1m_full, esp32dev, esp32_eth, esp32s
 ; default_envs = d1_mini_ota
 ; default_envs = esp32dev
 ; default_envs = esp8285_4CH_MagicHome
-; default_envs = esp8285_4CH_H801
+; default_envs = esp8285_H801
-; default_envs = esp8285_5CH_H801
 ; default_envs = d1_mini_5CH_Shojo_PCB
 ; default_envs = wemos_shield_esp32
 ; default_envs = m5atom
 ; default_envs = esp32_eth
+; default_envs = esp32s2_saola
 
 src_dir  = ./wled00
 data_dir = ./wled00/data
@@ -78,10 +78,8 @@ debug_flags = -D DEBUG=1 -D WLED_DEBUG -DDEBUG_ESP_WIFI -DDEBUG_ESP_HTTP_CLIENT
 
 # ------------------------------------------------------------------------------
 # FLAGS: ldscript (available ldscripts at https://github.com/esp8266/Arduino/tree/master/tools/sdk/ld)
-#    ldscript_1m0m (1024 KB) =  999 KB sketch, 4 KB eeprom,      no spiffs, 16 KB reserved
 #    ldscript_2m1m (2048 KB) = 1019 KB sketch, 4 KB eeprom, 1004 KB spiffs, 16 KB reserved
 #    ldscript_4m1m (4096 KB) = 1019 KB sketch, 4 KB eeprom, 1002 KB spiffs, 16 KB reserved, 2048 KB empty/ota?
-#    ldscript_4m3m (4096 KB) = 1019 KB sketch, 4 KB eeprom, 3040 KB spiffs, 16 KB reserved
 #
 # Available lwIP variants (macros):
 #    -DPIO_FRAMEWORK_ARDUINO_LWIP_HIGHER_BANDWIDTH  = v1.4 Higher Bandwidth (default)
@@ -178,7 +176,7 @@ lib_deps =
     ; adafruit/Adafruit CCS811 Library @ 1.0.4
     ; adafruit/Adafruit Si7021 Library @ 1.4.0
 
-extra_scripts             = ${scripts_defaults.extra_scripts}
+extra_scripts = ${scripts_defaults.extra_scripts}
 
 [esp8266]
 build_flags =
@@ -200,16 +198,21 @@ build_flags =
 
 lib_deps = 
   ${env.lib_deps}
-  https://github.com/lorol/LITTLEFS.git
+  #https://github.com/lorol/LITTLEFS.git
   # ESPAsyncTCP @ 1.2.0
   ESPAsyncUDP
   makuna/NeoPixelBus @ 2.6.7 # 2.6.5/2.6.6 and newer do not compile on ESP core < 3.0.0
 
 [esp32]
+#platform = https://github.com/tasmota/platform-espressif32/releases/download/v2.0.2/platform-tasmota-espressif32-2.0.2.zip
+platform = espressif32@3.5.0
+
 build_flags = -g
   -DARDUINO_ARCH_ESP32
-  -DCONFIG_LITTLEFS_FOR_IDF_3_2
+  #-DCONFIG_LITTLEFS_FOR_IDF_3_2
   -D CONFIG_ASYNC_TCP_USE_WDT=0
+#use LITTLEFS library by lorol in ESP32 core 1.x.x instead of built-in in 2.x.x
+  -D LOROL_LITTLEFS
 
 default_partitions = tools/WLED_ESP32_4MB_1MB_FS.csv
 
@@ -314,15 +317,16 @@ lib_deps = ${esp8266.lib_deps}
 
 [env:esp32dev]
 board = esp32dev
-platform = espressif32@2.0
+platform = ${esp32.platform}
 build_unflags = ${common.build_unflags}
 build_flags = ${common.build_flags_esp32} -D WLED_RELEASE_NAME=ESP32 #-D WLED_DISABLE_BROWNOUT_DET
 lib_deps = ${esp32.lib_deps}
+monitor_filters = esp32_exception_decoder
 board_build.partitions = ${esp32.default_partitions}
 
 [env:esp32_eth]
 board = esp32-poe
-platform = espressif32@2.0
+platform = ${esp32.platform}
 upload_speed = 921600
 build_unflags = ${common.build_unflags}
 build_flags = ${common.build_flags_esp32} -D WLED_RELEASE_NAME=ESP32_Ethernet -D RLYPIN=-1 -D WLED_USE_ETHERNET -D BTNPIN=-1
@@ -331,7 +335,7 @@ board_build.partitions = ${esp32.default_partitions}
 
 [env:esp32s2_saola]
 board = esp32-s2-saola-1
-platform = https://github.com/tasmota/platform-espressif32/releases/download/v2.0.2.1/platform-tasmota-espressif32-2.0.2.1.zip
+platform = https://github.com/tasmota/platform-espressif32/releases/download/v2.0.2.2/platform-tasmota-espressif32-2.0.2.zip
 platform_packages =
 framework = arduino
 board_build.partitions = tools/WLED_ESP32_4MB_1MB_FS.csv
@@ -342,7 +346,7 @@ lib_deps = ${esp32s2.lib_deps}
 
 [env:esp32c3]
 board = esp32-c3-devkitm-1
-platform = https://github.com/tasmota/platform-espressif32/releases/download/v2.0.2.1/platform-tasmota-espressif32-2.0.2.1.zip
+platform = https://github.com/tasmota/platform-espressif32/releases/download/v2.0.2.2/platform-tasmota-espressif32-2.0.2.zip
 platform_packages =
 framework = arduino
 board_build.partitions = tools/WLED_ESP32_4MB_1MB_FS.csv
@@ -359,16 +363,7 @@ build_unflags = ${common.build_unflags}
 build_flags = ${common.build_flags_esp8266} -D WLED_DISABLE_OTA
 lib_deps = ${esp8266.lib_deps}
 
-[env:esp8285_4CH_H801]
+[env:esp8285_H801]
-board = esp8285
-platform = ${common.platform_wled_default}
-platform_packages = ${common.platform_packages}
-board_build.ldscript = ${common.ldscript_1m128k}
-build_unflags = ${common.build_unflags}
-build_flags = ${common.build_flags_esp8266} -D WLED_DISABLE_OTA
-lib_deps = ${esp8266.lib_deps}
-
-[env:esp8285_5CH_H801]
 board = esp8285
 platform = ${common.platform_wled_default}
 platform_packages = ${common.platform_packages}
@@ -473,6 +468,13 @@ board_build.ldscript = ${common.ldscript_2m512k}
 build_flags = ${common.build_flags_esp8266} -D WLED_MAX_CCT_BLEND=0 -D BTNPIN=-1 -D IRPIN=-1 -D WLED_DISABLE_INFRARED
 lib_deps = ${esp8266.lib_deps}
 
+[env:athom15w]
+board = esp_wroom_02
+platform = ${common.platform_wled_default}
+board_build.ldscript = ${common.ldscript_2m512k}
+build_flags = ${common.build_flags_esp8266} -D WLED_USE_IC_CCT -D BTNPIN=-1 -D IRPIN=-1 -D WLED_DISABLE_INFRARED
+lib_deps = ${esp8266.lib_deps}
+
 # ------------------------------------------------------------------------------
 # travis test board configurations
 # ------------------------------------------------------------------------------

wled00/bus_manager.h

@@ -113,7 +113,7 @@ struct ColorOrderMap {
     return &(_mappings[n]);
   }
 
-  inline uint8_t getPixelColorOrder(uint16_t pix, uint8_t defaultColorOrder) const {
+  inline uint8_t IRAM_ATTR getPixelColorOrder(uint16_t pix, uint8_t defaultColorOrder) const {
     if (_count == 0) return defaultColorOrder;
 
     for (uint8_t i = 0; i < _count; i++) {
@@ -640,7 +640,7 @@ class BusManager {
 		}
 	}
 
-  void setPixelColor(uint16_t pix, uint32_t c, int16_t cct=-1) {
+  void IRAM_ATTR setPixelColor(uint16_t pix, uint32_t c, int16_t cct=-1) {
     for (uint8_t i = 0; i < numBusses; i++) {
       Bus* b = busses[i];
       uint16_t bstart = b->getStart();

wled00/data/index.js

@@ -956,9 +956,10 @@ function cmpP(a, b) {
 function makeWS() {
 	if (ws) return;
 	ws = new WebSocket('ws://'+(loc?locip:window.location.hostname)+'/ws');
+  ws.binaryType = "arraybuffer";
 	ws.onmessage = function(event) {
+    if (event.data instanceof ArrayBuffer) return; //liveview packet
 		var json = JSON.parse(event.data);
-		if (json.leds) return; //liveview packet
 		clearTimeout(jsonTimeout);
 		jsonTimeout = null;
 		clearErrorToast();

wled00/data/liveviewws.htm

@@ -24,40 +24,40 @@
     function updatePreview(leds) {
       var str = "linear-gradient(90deg,";
       var len = leds.length;        
-      for (i = 0; i < len; i++) {
+      for (i = 2; i < len; i+=3) {
-        var leddata = leds[i];
+        str += `rgb(${leds[i]},${leds[i+1]},${leds[i+2]})`;
-        if (leddata.length > 6) leddata = leddata.substring(2);
+        if (i < len -3) str += ","
-        str += "#" + leddata;
-        if (i < len -1) str += ","
       }
       str += ")";
       document.getElementById("canv").style.background = str;
     }
 
-    function getLiveJson(event) {
+    function getLiveJson(e) {
       try {
-        var json = JSON.parse(event.data);
+        if (toString.call(e.data) === '[object ArrayBuffer]') {
-        if (json && json.leds) {
+          let leds = new Uint8Array(event.data);
-          requestAnimationFrame(function () {updatePreview(json.leds);});
+          if (leds[0] != 76) return; //'L'
-        } 
+          updatePreview(leds);
+        }
       }
       catch (err) {
-        console.error("Live-Preview ws error:",err);
+        console.error("Peek WS error:",err);
       } 
     }
 
     var ws = top.window.ws;
     if (ws && ws.readyState === WebSocket.OPEN) {
-      console.info("Use top WS for peek");
+      console.info("Peek uses top WS");
       ws.send("{'lv':true}");
     } else {
-      console.info("Peek ws opening");
+      console.info("Peek WS opening");
       ws = new WebSocket("ws://"+document.location.host+"/ws");
       ws.onopen = function () {
-        console.info("Peek WS opened");
+        console.info("Peek WS open");
         ws.send("{'lv':true}");
       }
     }
+    ws.binaryType = "arraybuffer";
     ws.addEventListener('message',getLiveJson);
   </script>
 </body>

wled00/fcn_declare.h

@@ -137,7 +137,9 @@ void serializeSegment(JsonObject& root, WS2812FX::Segment& seg, byte id, bool fo
 void serializeState(JsonObject root, bool forPreset = false, bool includeBri = true, bool segmentBounds = true);
 void serializeInfo(JsonObject root);
 void serveJson(AsyncWebServerRequest* request);
+#ifdef WLED_ENABLE_JSONLIVE
 bool serveLiveLeds(AsyncWebServerRequest* request, uint32_t wsClient = 0);
+#endif
 
 //led.cpp
 void setValuesFromMainSeg();

wled00/html_other.h

@@ -85,7 +85,7 @@ charset="utf-8"><meta name="theme-color" content="#222222"><title>
 WLED Live Preview</title><style>
 body{margin:0}#canv{background:#000;filter:brightness(175%);width:100%;height:100%;position:absolute}
 </style></head><body><div id="canv"><script>
-function updatePreview(e){var n="linear-gradient(90deg,",o=e.length;for(i=0;i<o;i++){var t=e[i];t.length>6&&(t=t.substring(2)),n+="#"+t,i<o-1&&(n+=",")}n+=")",document.getElementById("canv").style.background=n}function getLiveJson(e){try{var n=JSON.parse(e.data);n&&n.leds&&requestAnimationFrame((function(){updatePreview(n.leds)}))}catch(e){console.error("Live-Preview ws error:",e)}}var ws=top.window.ws;ws&&ws.readyState===WebSocket.OPEN?(console.info("Use top WS for peek"),ws.send("{'lv':true}")):(console.info("Peek ws opening"),(ws=new WebSocket("ws://"+document.location.host+"/ws")).onopen=function(){console.info("Peek WS opened"),ws.send("{'lv':true}")}),ws.addEventListener("message",getLiveJson)
+function updatePreview(e){var n="linear-gradient(90deg,",t=e.length;for(i=2;i<t;i+=3)n+=`rgb(${e[i]},${e[i+1]},${e[i+2]})`,i<t-3&&(n+=",");n+=")",document.getElementById("canv").style.background=n}function getLiveJson(e){try{if(e.data instanceof ArrayBuffer||"[object ArrayBuffer]"===toString.call(e.data)){let e=new Uint8Array(event.data);if(76!=e[0])return;updatePreview(e)}}catch(e){console.error("Peek WS error:",e)}}var ws=top.window.ws;ws&&ws.readyState===WebSocket.OPEN?(console.info("Peek uses top WS"),ws.send("{'lv':true}")):(console.info("Peek WS opening"),(ws=new WebSocket("ws://"+document.location.host+"/ws")).onopen=function(){console.info("Peek WS open"),ws.send("{'lv':true}")}),ws.binaryType="arraybuffer",ws.addEventListener("message",getLiveJson)
 </script></body></html>)=====";
 
 

wled00/json.cpp

@@ -829,10 +829,12 @@ void serveJson(AsyncWebServerRequest* request)
   else if (url.indexOf("si")    > 0) subJson = 3;
   else if (url.indexOf("nodes") > 0) subJson = 4;
   else if (url.indexOf("palx")  > 0) subJson = 5;
+  #ifdef WLED_ENABLE_JSONLIVE
   else if (url.indexOf("live")  > 0) {
     serveLiveLeds(request);
     return;
   }
+  #endif
   else if (url.indexOf(F("eff")) > 0) {
     request->send_P(200, "application/json", JSON_mode_names);
     return;
@@ -888,6 +890,7 @@ void serveJson(AsyncWebServerRequest* request)
   releaseJSONBufferLock();
 }
 
+#ifdef WLED_ENABLE_JSONLIVE
 #define MAX_LIVE_LEDS 180
 
 bool serveLiveLeds(AsyncWebServerRequest* request, uint32_t wsClient)
@@ -929,3 +932,4 @@ bool serveLiveLeds(AsyncWebServerRequest* request, uint32_t wsClient)
   #endif
   return true;
 }
+#endif

wled00/my_config_sample.h

@@ -13,8 +13,8 @@
  * DO NOT make changes to the "my_config_sample.h" file directly! Your changes will not be applied.
  */
 
-// force the compiler to show a warning to confirm that this file is included
+// uncomment to force the compiler to show a warning to confirm that this file is included
-#warning **** my_config.h: Settings from this file are honored ****
+//#warning **** my_config.h: Settings from this file are honored ****
 
 /* Uncomment to use your WIFI settings as defaults
   //WARNING: this will hardcode these as the default even after a factory reset

wled00/wled.h

@@ -8,7 +8,7 @@
  */
 
 // version code in format yymmddb (b = daily build)
-#define VERSION 2201260
+#define VERSION 2201280
 
 //uncomment this if you have a "my_config.h" file you'd like to use
 //#define WLED_USE_MY_CONFIG
@@ -34,6 +34,7 @@
 #endif
 #define WLED_ENABLE_ADALIGHT     // saves 500b only (uses GPIO3 (RX) for serial)
 //#define WLED_ENABLE_DMX          // uses 3.5kb (use LEDPIN other than 2)
+//#define WLED_ENABLE_JSONLIVE     // peek LED output via /json/live (WS binary peek is always enabled)
 #ifndef WLED_DISABLE_LOXONE
   #define WLED_ENABLE_LOXONE       // uses 1.2kb
 #endif
@@ -70,8 +71,7 @@
   #include "esp_wifi.h"
   #include <ESPmDNS.h>
   #include <AsyncTCP.h>
-  //#include "SPIFFS.h"
+  #if LOROL_LITTLEFS
-  #if ESP_IDF_VERSION_MAJOR < 4
     #ifndef CONFIG_LITTLEFS_FOR_IDF_3_2
       #define CONFIG_LITTLEFS_FOR_IDF_3_2
     #endif
@@ -177,7 +177,7 @@ using PSRAMDynamicJsonDocument = BasicJsonDocument<PSRAM_Allocator>;
 #ifdef ESP8266
   #define WLED_FS LittleFS
 #else
-  #if ESP_IDF_VERSION_MAJOR < 4
+  #if LOROL_LITTLEFS
     #define WLED_FS LITTLEFS
   #else
     #define WLED_FS LittleFS

wled00/ws.cpp

@@ -123,6 +123,34 @@ void sendDataWs(AsyncWebSocketClient * client)
   }
 }
 
+#define MAX_LIVE_LEDS_WS 256
+
+bool sendLiveLedsWs(uint32_t wsClient)
+{
+  AsyncWebSocketClient * wsc = ws.client(wsClient);
+  if (!wsc || wsc->queueLength() > 0) return false; //only send if queue free
+
+  uint16_t used = strip.getLengthTotal();
+  uint16_t n = (used -1) /MAX_LIVE_LEDS_WS +1; //only serve every n'th LED if count over MAX_LIVE_LEDS
+  AsyncWebSocketMessageBuffer * wsBuf = ws.makeBuffer(2 + MIN(used, MAX_LIVE_LEDS_WS)*3);
+  if (!wsBuf) return false; //out of memory
+  uint8_t* buffer = wsBuf->get();
+  buffer[0] = 'L';
+  buffer[1] = 1; //version
+
+  uint16_t pos = 2;
+  for (uint16_t i= 0; i < used; i += n)
+  {
+    uint32_t c = strip.getPixelColor(i);
+    buffer[pos++] = qadd8(W(c), R(c)); //R, add white channel to RGB channels as a simple RGBW -> RGB map
+    buffer[pos++] = qadd8(W(c), G(c)); //G
+    buffer[pos++] = qadd8(W(c), B(c)); //B
+  }
+
+  wsc->binary(wsBuf);
+  return true;
+}
+
 void handleWs()
 {
   if (millis() - wsLastLiveTime > WS_LIVE_INTERVAL)
@@ -130,7 +158,7 @@ void handleWs()
     ws.cleanupClients();
     bool success = true;
     if (wsLiveClientId)
-      success = serveLiveLeds(nullptr, wsLiveClientId);
+      success = sendLiveLedsWs(wsLiveClientId);
     wsLastLiveTime = millis();
     if (!success) wsLastLiveTime -= 20; //try again in 20ms if failed due to non-empty WS queue
   }