Added sd support and fixed proto malloc

app/src/lib/filesystem/chunkedTransfer.ts

@@ -15,7 +15,7 @@ import type {
 } from '$lib/platform_shared/filesystem'
 import type { Result, DataResult, ListResult, ProgressCallback } from '$lib/types/models'
 
-const MAX_CHUNK_SIZE = 2 ** 14
+const MAX_CHUNK_SIZE = 1024 * 64 // 64KB - must match ESP32 FS_MAX_CHUNK_SIZE
 
 type TimeoutId = ReturnType<typeof setTimeout>
 type CleanupFn = (() => void) | null
@@ -51,7 +51,7 @@ export class FileSystemClient {
 	private downloadListenerCleanup: CleanupFn = null
 	private completeListenerCleanup: CleanupFn = null
 	private uploadCompleteListenerCleanup: CleanupFn = null
-	private transferTimeout = 60000
+	private transferTimeout = 300000
 
 	constructor() {
 		this.setupListeners()

app/src/lib/stores/socket.ts

@@ -153,6 +153,8 @@ function createWebSocket() {
             }
 
             const { tag, msg } = decodeMessage(frame.data)
+            const key: keyof Message = (MESSAGE_TAG_TO_KEY.get(tag) ?? "") as keyof Message;
+            console.log(key + ": ", msg[key])
             if (msg.correlationResponse) {
                 const pending = pending_requests.get(msg.correlationResponse.correlationId)
                 if (pending) {

app/vite.config.ts

@@ -21,7 +21,7 @@ export default defineConfig({
     server: {
         proxy: {
             '/api': {
-                target: 'http://spot-micro.local/',
+                target: 'http://192.168.50.141/',
                 changeOrigin: true,
                 ws: true
             }

esp32/include/communication/comm_base.hpp

@@ -54,7 +54,8 @@ class CommAdapterBase {
 
         pb_ostream_t stream = pb_ostream_from_buffer(buffer, out_size);
         if (!pb_encode(&stream, socket_message_Message_fields, &msg_)) {
-            ESP_LOGE("ProtoComm", "Failed to encode message (tag %d), buffer too small?", (int)tag);
+            ESP_LOGE("ProtoComm", "Failed to encode message (tag %d): %s (calc=%u, written=%u)",
+                     (int)tag, PB_GET_ERROR(&stream), out_size, stream.bytes_written);
             return;
         }
 

esp32/include/communication/proto_helpers.h

@@ -3,6 +3,7 @@
 #include <pb_encode.h>
 #include <pb_decode.h>
 #include <platform_shared/message.pb.h>
+#include <esp_log.h>
 #include <functional>
 #include <map>
 
@@ -71,32 +72,48 @@ class ProtoDecoder {
     bool decode(const uint8_t* data, size_t len, int clientId) {
         pb_istream_t stream = pb_istream_from_buffer(data, len);
 
-        if (!pb_decode(&stream, socket_message_Message_fields, &msg_)) {
+        // Reset message before decoding (nanopb will malloc FT_POINTER fields)
+        msg_ = socket_message_Message_init_zero;
+
+        bool success = pb_decode(&stream, socket_message_Message_fields, &msg_);
+
+        if (!success) {
+            ESP_LOGE("ProtoHelpers", "Decode failed: %s (len=%u)", PB_GET_ERROR(&stream), len);
+            pb_release(socket_message_Message_fields, &msg_);
             return false;
         }
 
+        bool handled = false;
         switch (msg_.which_message) {
             case socket_message_Message_sub_notif_tag:
                 if (subscribeHandler_) subscribeHandler_(msg_.message.sub_notif.tag, clientId);
-                return true;
+                handled = true;
+                break;
 
             case socket_message_Message_unsub_notif_tag:
                 if (unsubscribeHandler_) unsubscribeHandler_(msg_.message.unsub_notif.tag, clientId);
-                return true;
+                handled = true;
+                break;
 
             case socket_message_Message_pingmsg_tag:
                 if (pingHandler_) pingHandler_(clientId);
-                return true;
+                handled = true;
+                break;
 
             default: {
                 auto it = handlers_.find(msg_.which_message);
                 if (it != handlers_.end()) {
                     it->second(clientId);
-                    return true;
+                    handled = true;
                 }
-                return false;
+                break;
             }
         }
+
+        // Free any malloc'd FT_POINTER fields
+        pb_release(socket_message_Message_fields, &msg_);
+
+        return handled;
     }
 
   private:

esp32/include/filesystem.h

@@ -9,16 +9,18 @@
 #include <cstdio>
 #include <platform_shared/api.pb.h>
 
-#define MOUNT_POINT "/littlefs"
+#define MOUNT_POINT "/"
+#define LITTLEFS_MOUNT_POINT "/littlefs"
+#define SD_MOUNT_POINT "/sdcard"
 
-#define FS_CONFIG_DIRECTORY MOUNT_POINT "/config"
+#define FS_CONFIG_DIRECTORY LITTLEFS_MOUNT_POINT "/config"
-#define DEVICE_CONFIG_FILE MOUNT_POINT "/config/peripheral.pb"
+#define DEVICE_CONFIG_FILE LITTLEFS_MOUNT_POINT "/config/peripheral.pb"
-#define CAMERA_SETTINGS_FILE MOUNT_POINT "/config/cameraSettings.pb"
+#define CAMERA_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/cameraSettings.pb"
-#define AP_SETTINGS_FILE MOUNT_POINT "/config/apSettings.pb"
+#define AP_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/apSettings.pb"
-#define MDNS_SETTINGS_FILE MOUNT_POINT "/config/mdnsSettings.pb"
+#define MDNS_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/mdnsSettings.pb"
-#define WIFI_SETTINGS_FILE MOUNT_POINT "/config/wifiSettings.pb"
+#define WIFI_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/wifiSettings.pb"
-#define PERIPHERAL_SETTINGS_FILE MOUNT_POINT "/config/peripheralSettings.pb"
+#define PERIPHERAL_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/peripheralSettings.pb"
-#define SERVO_SETTINGS_FILE MOUNT_POINT "/config/servoSettings.pb"
+#define SERVO_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/servoSettings.pb"
 
 namespace FileSystem {
 

esp32/include/filesystem_ws.h

@@ -7,7 +7,7 @@
 #include <functional>
 #include <cstdio>
 
-#define FS_MAX_CHUNK_SIZE 16384
+#define FS_MAX_CHUNK_SIZE (1024*64)
 #define FS_TRANSFER_TIMEOUT_MS 30000
 
 namespace FileSystemWS {

esp32/src/filesystem.cpp

@@ -6,6 +6,9 @@
 #include <esp_log.h>
 #include <pb_encode.h>
 #include <pb_decode.h>
+#include "esp_vfs_fat.h"
+#include <sdmmc_cmd.h>
+#include <driver/sdmmc_host.h>
 
 static const char *TAG = "FileSystem";
 
@@ -79,7 +82,7 @@ void listFilesProto(const std::string &directory, api_FileEntry *entry) {
     if (path.empty() || path[0] != '/') {
         path = "/" + directory;
     }
-    std::string fullPath = std::string(MOUNT_POINT) + path;
+    std::string fullPath = path;
     listFilesProtoRecursive(fullPath, entry);
 }
 
@@ -108,7 +111,7 @@ esp_err_t getFilesProto(httpd_req_t *request) {
 
 bool init() {
     esp_vfs_littlefs_conf_t conf = {
-        .base_path = MOUNT_POINT,
+        .base_path = LITTLEFS_MOUNT_POINT,
         .partition_label = "spiffs",
         .format_if_mount_failed = true,
         .dont_mount = false,
@@ -134,6 +137,51 @@ bool init() {
 
     mkdirRecursive(FS_CONFIG_DIRECTORY);
 
+
+    // Optional SD card mounting via SDMMC (1-bit mode for ESP32-S3-CAM)
+    // Pin definitions - override in build flags if needed
+#ifndef SD_CMD_PIN
+#define SD_CMD_PIN GPIO_NUM_38
+#endif
+#ifndef SD_CLK_PIN
+#define SD_CLK_PIN GPIO_NUM_39
+#endif
+#ifndef SD_DATA_PIN
+#define SD_DATA_PIN GPIO_NUM_40
+#endif
+
+    esp_vfs_fat_sdmmc_mount_config_t sd_mount_config = {
+        .format_if_mount_failed = false,
+        .max_files = 4,
+        .allocation_unit_size = 16 * 1024,
+    };
+
+    sdmmc_host_t host = SDMMC_HOST_DEFAULT();
+    host.flags = SDMMC_HOST_FLAG_1BIT;  // Use 1-bit mode
+    host.max_freq_khz = SDMMC_FREQ_DEFAULT;
+
+    sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT();
+    slot_config.width = 1;  // 1-bit mode
+    slot_config.clk = SD_CLK_PIN;
+    slot_config.cmd = SD_CMD_PIN;
+    slot_config.d0 = SD_DATA_PIN;
+    slot_config.flags |= SDMMC_SLOT_FLAG_INTERNAL_PULLUP;
+
+    sdmmc_card_t *card = nullptr;
+    esp_err_t err = esp_vfs_fat_sdmmc_mount(SD_MOUNT_POINT, &host, &slot_config, &sd_mount_config, &card);
+    if (err != ESP_OK) {
+        if (err == ESP_FAIL) {
+            ESP_LOGW(TAG, "Failed to mount SD card filesystem");
+        } else {
+            ESP_LOGW(TAG, "SD card not present or failed to initialize (%s)", esp_err_to_name(err));
+        }
+        // Don't fail - SD card is optional
+    } else {
+        ESP_LOGI(TAG, "SD card mounted at %s", SD_MOUNT_POINT);
+        ESP_LOGI(TAG, "SD card: %s, %lluMB", card->cid.name,
+                 ((uint64_t)card->csd.capacity) * card->csd.sector_size / (1024 * 1024));
+    }
+
     return true;
 }
 
@@ -231,7 +279,7 @@ esp_err_t getFiles(httpd_req_t *request) {
 
 esp_err_t getConfigFile(httpd_req_t *request) {
     const char *uri = request->uri;
-    std::string path = std::string(MOUNT_POINT) + "/config" + std::string(uri).substr(11);
+    std::string path = std::string(LITTLEFS_MOUNT_POINT) + "/config" + std::string(uri).substr(11);
 
     if (!fileExists(path.c_str())) {
         return WebServer::sendError(request, 404, "File not found");
@@ -253,7 +301,7 @@ esp_err_t getConfigFile(httpd_req_t *request) {
 }
 
 esp_err_t handleDelete(httpd_req_t *request, const api_FileDeleteRequest &req) {
-    std::string fullPath = std::string(MOUNT_POINT) + req.path;
+    std::string fullPath = req.path;
     ESP_LOGI(TAG, "Deleting file: %s", fullPath.c_str());
 
     api_Response res = api_Response_init_zero;
@@ -267,7 +315,7 @@ esp_err_t handleDelete(httpd_req_t *request, const api_FileDeleteRequest &req) {
 }
 
 esp_err_t handleEdit(httpd_req_t *request, const api_FileEditRequest &req) {
-    std::string fullPath = std::string(MOUNT_POINT) + req.path;
+    std::string fullPath = req.path;
     ESP_LOGI(TAG, "Editing file: %s", fullPath.c_str());
 
     api_Response res = api_Response_init_zero;
@@ -326,7 +374,7 @@ bool editFile(const char *filename, const uint8_t *content, size_t size) { retur
 bool editFile(const char *filename, const char *content) { return writeFile(filename, content); }
 
 esp_err_t mkdir(httpd_req_t *request, const api_FileMkdirRequest &req) {
-    std::string fullPath = std::string(MOUNT_POINT) + req.path;
+    std::string fullPath = req.path;
     ESP_LOGI(TAG, "Creating directory: %s", fullPath.c_str());
 
     api_Response res = api_Response_init_zero;

esp32/src/filesystem_ws.cpp

@@ -7,6 +7,7 @@
 #include <dirent.h>
 #include <sys/stat.h>
 #include <cstring>
+#include <cerrno>
 
 static const char* TAG = "FileSystemWS";
 
@@ -80,7 +81,7 @@ void FileSystemHandler::cleanupExpiredTransfers() {
 socket_message_FSDeleteResponse FileSystemHandler::handleDelete(const socket_message_FSDeleteRequest& req) {
     socket_message_FSDeleteResponse response = socket_message_FSDeleteResponse_init_zero;
 
-    std::string path = std::string(MOUNT_POINT) + req.path;
+    std::string path = req.path;
     ESP_LOGI(TAG, "Delete request: %s", path.c_str());
 
     struct stat st;
@@ -129,7 +130,7 @@ bool FileSystemHandler::deleteRecursive(const std::string& path) {
 socket_message_FSMkdirResponse FileSystemHandler::handleMkdir(const socket_message_FSMkdirRequest& req) {
     socket_message_FSMkdirResponse response = socket_message_FSMkdirResponse_init_zero;
 
-    std::string path = std::string(MOUNT_POINT) + req.path;
+    std::string path = req.path;
     ESP_LOGI(TAG, "Mkdir request: %s", path.c_str());
 
     struct stat st;
@@ -150,6 +151,15 @@ socket_message_FSMkdirResponse FileSystemHandler::handleMkdir(const socket_messa
 }
 
 void FileSystemHandler::listDirectory(const std::string& path, socket_message_FSListResponse& response) {
+
+    // Root "/" is virtual - list mount points instead
+    if (strcmp(path.c_str(), "/") == 0) {
+        strncpy(response.directories[0].name, LITTLEFS_MOUNT_POINT + 1, sizeof(response.directories[0].name) - 1);
+        strncpy(response.directories[1].name, SD_MOUNT_POINT + 1, sizeof(response.directories[1].name) - 1);
+        response.directories_count = 2;
+        return;
+    }
+
     DIR* dir = opendir(path.c_str());
     if (!dir) {
         return;
@@ -191,15 +201,13 @@ void FileSystemHandler::listDirectory(const std::string& path, socket_message_FS
 socket_message_FSListResponse FileSystemHandler::handleList(const socket_message_FSListRequest& req) {
     socket_message_FSListResponse response = socket_message_FSListResponse_init_zero;
 
-    std::string path = std::string(MOUNT_POINT);
+    std::string path = req.path;
-    if (strlen(req.path) > 0 && req.path[0] != '\0') {
-        path += req.path;
-    }
 
     ESP_LOGI(TAG, "List request: %s", path.c_str());
 
     struct stat st;
-    if (stat(path.c_str(), &st) != 0) {
+    // Make sure that path exists, or that it is a root listing
+    if (strcmp(path.c_str(), "/") != 0 && stat(path.c_str(), &st) != 0) {
         response.success = false;
         strncpy(response.error, "Path not found", sizeof(response.error) - 1);
         return response;
@@ -212,7 +220,7 @@ socket_message_FSListResponse FileSystemHandler::handleList(const socket_message
 }
 
 void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadRequest& req, int clientId) {
-    std::string path = std::string(MOUNT_POINT) + req.path;
+    std::string path = req.path;
     ESP_LOGI(TAG, "Download request: %s", path.c_str());
 
     struct stat st;
@@ -268,7 +276,7 @@ void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadReq
     ESP_LOGI(TAG, "Download started: %s, size=%u, chunks=%u, id=%u", path.c_str(), fileSize, totalChunks, transferId);
 
     while (sendNextDownloadChunk(transferId)) {
-        taskYIELD();
+        vTaskDelay(pdMS_TO_TICKS(5));  // Give network time to send (5 ms)
     }
 }
 
@@ -308,8 +316,28 @@ bool FileSystemHandler::sendNextDownloadChunk(uint32_t transferId) {
         bytesToRead = state.fileSize - position;
     }
 
-    size_t bytesRead = fread(data->data.bytes, 1, bytesToRead, state.file);
+    // Allocate buffer for FT_POINTER data field
+    data->data = (pb_bytes_array_t*)malloc(PB_BYTES_ARRAY_T_ALLOCSIZE(bytesToRead));
+    if (!data->data) {
+        delete data;
+        if (sendCompleteCallback_) {
+            socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
+            complete.transfer_id = transferId;
+            complete.success = false;
+            strncpy(complete.error, "Memory allocation failed", sizeof(complete.error) - 1);
+            complete.total_chunks = state.chunksSent;
+            complete.file_size = state.fileSize;
+            sendCompleteCallback_(complete, state.clientId);
+        }
+        fclose(state.file);
+        downloads_.erase(it);
+        ESP_LOGE(TAG, "Download failed - memory allocation: %u", transferId);
+        return false;
+    }
+
+    size_t bytesRead = fread(data->data->bytes, 1, bytesToRead, state.file);
     if (bytesRead == 0 && bytesToRead > 0) {
+        free(data->data);
         delete data;
         if (sendCompleteCallback_) {
             socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
@@ -326,12 +354,13 @@ bool FileSystemHandler::sendNextDownloadChunk(uint32_t transferId) {
         ESP_LOGE(TAG, "Download failed - read error: %u", transferId);
         return false;
     }
-    data->data.size = bytesRead;
+    data->data->size = bytesRead;
 
     if (sendDataCallback_) {
         sendDataCallback_(*data, state.clientId);
     }
 
+    free(data->data);
     delete data;
     state.chunksSent++;
     ESP_LOGD(TAG, "Download chunk %u/%u sent: %u bytes", state.chunksSent, state.totalChunks, bytesRead);
@@ -343,17 +372,22 @@ socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const
                                                                           int clientId) {
     socket_message_FSUploadStartResponse response = socket_message_FSUploadStartResponse_init_zero;
 
-    std::string path = std::string(MOUNT_POINT) + req.path;
+    std::string path = req.path;
     ESP_LOGI(TAG, "Upload start request: %s, size=%u, chunks=%u", path.c_str(), req.file_size, req.total_chunks);
 
-    size_t fs_total = 0, fs_used = 0;
+    // Check available space on the target filesystem
-    esp_littlefs_info("spiffs", &fs_total, &fs_used);
+    if (path.find(SD_MOUNT_POINT) != 0) {
-    size_t freeSpace = fs_total - fs_used;
+        // LittleFS path
-    if (freeSpace < req.file_size + 4096) {
+        size_t fs_total = 0, fs_used = 0;
-        response.success = false;
+        esp_littlefs_info("spiffs", &fs_total, &fs_used);
-        strncpy(response.error, "Insufficient storage space", sizeof(response.error) - 1);
+        size_t freeSpace = fs_total - fs_used;
-        return response;
+        if (freeSpace < req.file_size + 4096) {
+            response.success = false;
+            strncpy(response.error, "Insufficient storage space", sizeof(response.error) - 1);
+            return response;
+        }
     }
+    // TODO: SD card space check skipped - FAT doesn't have a simple API for this
 
     size_t lastSlash = path.find_last_of('/');
     if (lastSlash != std::string::npos && lastSlash > 0) {
@@ -368,8 +402,9 @@ socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const
 
     FILE* file = fopen(path.c_str(), "wb");
     if (!file) {
+        ESP_LOGE(TAG, "fopen failed for '%s': %s (errno=%d)", path.c_str(), strerror(errno), errno);
         response.success = false;
-        strncpy(response.error, "Cannot open file for writing", sizeof(response.error) - 1);
+        snprintf(response.error, sizeof(response.error) - 1, "Cannot open file: %s", strerror(errno));
         return response;
     }
 
@@ -416,8 +451,15 @@ void FileSystemHandler::handleUploadData(const socket_message_FSUploadData& req)
         ESP_LOGW(TAG, "Upload chunk out of order: expected %u, got %u", state.chunksReceived, req.chunk_index);
     }
 
-    size_t bytesWritten = fwrite(req.data.bytes, 1, req.data.size, state.file);
+    if (!req.data || req.data->size == 0) {
-    if (bytesWritten != req.data.size) {
+        state.hasError = true;
+        state.errorMessage = "Empty or invalid data chunk";
+        finalizeUpload(transferId, false, state.errorMessage);
+        return;
+    }
+
+    size_t bytesWritten = fwrite(req.data->bytes, 1, req.data->size, state.file);
+    if (bytesWritten != req.data->size) {
         state.hasError = true;
         state.errorMessage = "Failed to write chunk";
         finalizeUpload(transferId, false, state.errorMessage);

esp32/src/main.cpp

@@ -1,6 +1,7 @@
 #include <freertos/FreeRTOS.h>
 #include <freertos/task.h>
 #include <esp_log.h>
+#include <esp_heap_caps.h>
 #include <nvs_flash.h>
 #include <wifi/wifi_idf.h>
 #include <mdns.h>
@@ -42,6 +43,8 @@ WiFiService wifiService;
 APService apService;
 
 void setupServer() {
+    ESP_LOGI("Main", "Free heap before server: %lu, largest block: %lu",
+             esp_get_free_heap_size(), heap_caps_get_largest_free_block(MALLOC_CAP_8BIT));
     server.config(50 + WWW_ASSETS_COUNT, 16384);
     server.listen(80);
 

platform_shared/filesystem.options

@@ -16,11 +16,11 @@ socket_message.FSListResponse.directories max_count:20
 # Streaming download messages
 socket_message.FSDownloadRequest.path max_size:256
 socket_message.FSDownloadMetadata.error max_size:128
-socket_message.FSDownloadData.data max_size:16384
+socket_message.FSDownloadData.data type:FT_POINTER
 socket_message.FSDownloadComplete.error max_size:128
 
 # Streaming upload messages
 socket_message.FSUploadStart.path max_size:256
 socket_message.FSUploadStartResponse.error max_size:128
-socket_message.FSUploadData.data max_size:16384
+socket_message.FSUploadData.data type:FT_POINTER
 socket_message.FSUploadComplete.error max_size:128

platformio.ini

@@ -82,8 +82,8 @@ platform = espressif32 @ 6.8.1
 framework = espidf
 monitor_speed = 115200
 monitor_filters =
-    direct
+	direct
-    esp32_exception_decoder
+	esp32_exception_decoder
 build_flags =
 	${factory_settings.build_flags}
     ${features.build_flags}
@@ -96,6 +96,8 @@ build_flags =
     -fdata-sections
     -Wl,--gc-sections
     -I submodules/nanopb
+    -D PB_FIELD_32BIT=1
+    -D PB_ENABLE_MALLOC=1
     -Wno-missing-braces
     -Wno-format
     -D CONFIG_HTTPD_WS_SUPPORT=1