From 8d4ce16460142a738e3d82fa3b7c428f3729dc6b Mon Sep 17 00:00:00 2001 From: Niklas Jensen Date: Sat, 31 Jan 2026 20:03:52 +0100 Subject: [PATCH] Added sd support and fixed proto malloc --- app/src/lib/filesystem/chunkedTransfer.ts | 4 +- app/src/lib/stores/socket.ts | 2 + app/vite.config.ts | 2 +- esp32/include/communication/comm_base.hpp | 3 +- esp32/include/communication/proto_helpers.h | 29 +++++-- esp32/include/filesystem.h | 20 ++--- esp32/include/filesystem_ws.h | 2 +- esp32/src/filesystem.cpp | 60 ++++++++++++-- esp32/src/filesystem_ws.cpp | 86 +++++++++++++++------ esp32/src/main.cpp | 3 + platform_shared/filesystem.options | 4 +- platformio.ini | 6 +- 12 files changed, 169 insertions(+), 52 deletions(-) diff --git a/app/src/lib/filesystem/chunkedTransfer.ts b/app/src/lib/filesystem/chunkedTransfer.ts index 1d76333..7fa763b 100644 --- a/app/src/lib/filesystem/chunkedTransfer.ts +++ b/app/src/lib/filesystem/chunkedTransfer.ts @@ -15,7 +15,7 @@ import type { } from '$lib/platform_shared/filesystem' import type { Result, DataResult, ListResult, ProgressCallback } from '$lib/types/models' -const MAX_CHUNK_SIZE = 2 ** 14 +const MAX_CHUNK_SIZE = 1024 * 64 // 64KB - must match ESP32 FS_MAX_CHUNK_SIZE type TimeoutId = ReturnType type CleanupFn = (() => void) | null @@ -51,7 +51,7 @@ export class FileSystemClient { private downloadListenerCleanup: CleanupFn = null private completeListenerCleanup: CleanupFn = null private uploadCompleteListenerCleanup: CleanupFn = null - private transferTimeout = 60000 + private transferTimeout = 300000 constructor() { this.setupListeners() diff --git a/app/src/lib/stores/socket.ts b/app/src/lib/stores/socket.ts index c488683..dd4d2d9 100644 --- a/app/src/lib/stores/socket.ts +++ b/app/src/lib/stores/socket.ts @@ -153,6 +153,8 @@ function createWebSocket() { } const { tag, msg } = decodeMessage(frame.data) + const key: keyof Message = (MESSAGE_TAG_TO_KEY.get(tag) ?? "") as keyof Message; + console.log(key + ": ", msg[key]) if (msg.correlationResponse) { const pending = pending_requests.get(msg.correlationResponse.correlationId) if (pending) { diff --git a/app/vite.config.ts b/app/vite.config.ts index 33284b8..7a07ec3 100644 --- a/app/vite.config.ts +++ b/app/vite.config.ts @@ -21,7 +21,7 @@ export default defineConfig({ server: { proxy: { '/api': { - target: 'http://spot-micro.local/', + target: 'http://192.168.50.141/', changeOrigin: true, ws: true } diff --git a/esp32/include/communication/comm_base.hpp b/esp32/include/communication/comm_base.hpp index b9464b4..53cff81 100644 --- a/esp32/include/communication/comm_base.hpp +++ b/esp32/include/communication/comm_base.hpp @@ -54,7 +54,8 @@ class CommAdapterBase { pb_ostream_t stream = pb_ostream_from_buffer(buffer, out_size); if (!pb_encode(&stream, socket_message_Message_fields, &msg_)) { - ESP_LOGE("ProtoComm", "Failed to encode message (tag %d), buffer too small?", (int)tag); + ESP_LOGE("ProtoComm", "Failed to encode message (tag %d): %s (calc=%u, written=%u)", + (int)tag, PB_GET_ERROR(&stream), out_size, stream.bytes_written); return; } diff --git a/esp32/include/communication/proto_helpers.h b/esp32/include/communication/proto_helpers.h index 02313ab..764aa3b 100644 --- a/esp32/include/communication/proto_helpers.h +++ b/esp32/include/communication/proto_helpers.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -71,32 +72,48 @@ class ProtoDecoder { bool decode(const uint8_t* data, size_t len, int clientId) { pb_istream_t stream = pb_istream_from_buffer(data, len); - if (!pb_decode(&stream, socket_message_Message_fields, &msg_)) { + // Reset message before decoding (nanopb will malloc FT_POINTER fields) + msg_ = socket_message_Message_init_zero; + + bool success = pb_decode(&stream, socket_message_Message_fields, &msg_); + + if (!success) { + ESP_LOGE("ProtoHelpers", "Decode failed: %s (len=%u)", PB_GET_ERROR(&stream), len); + pb_release(socket_message_Message_fields, &msg_); return false; } + bool handled = false; switch (msg_.which_message) { case socket_message_Message_sub_notif_tag: if (subscribeHandler_) subscribeHandler_(msg_.message.sub_notif.tag, clientId); - return true; + handled = true; + break; case socket_message_Message_unsub_notif_tag: if (unsubscribeHandler_) unsubscribeHandler_(msg_.message.unsub_notif.tag, clientId); - return true; + handled = true; + break; case socket_message_Message_pingmsg_tag: if (pingHandler_) pingHandler_(clientId); - return true; + handled = true; + break; default: { auto it = handlers_.find(msg_.which_message); if (it != handlers_.end()) { it->second(clientId); - return true; + handled = true; } - return false; + break; } } + + // Free any malloc'd FT_POINTER fields + pb_release(socket_message_Message_fields, &msg_); + + return handled; } private: diff --git a/esp32/include/filesystem.h b/esp32/include/filesystem.h index 384e98b..8da54f7 100644 --- a/esp32/include/filesystem.h +++ b/esp32/include/filesystem.h @@ -9,16 +9,18 @@ #include #include -#define MOUNT_POINT "/littlefs" +#define MOUNT_POINT "/" +#define LITTLEFS_MOUNT_POINT "/littlefs" +#define SD_MOUNT_POINT "/sdcard" -#define FS_CONFIG_DIRECTORY MOUNT_POINT "/config" -#define DEVICE_CONFIG_FILE MOUNT_POINT "/config/peripheral.pb" -#define CAMERA_SETTINGS_FILE MOUNT_POINT "/config/cameraSettings.pb" -#define AP_SETTINGS_FILE MOUNT_POINT "/config/apSettings.pb" -#define MDNS_SETTINGS_FILE MOUNT_POINT "/config/mdnsSettings.pb" -#define WIFI_SETTINGS_FILE MOUNT_POINT "/config/wifiSettings.pb" -#define PERIPHERAL_SETTINGS_FILE MOUNT_POINT "/config/peripheralSettings.pb" -#define SERVO_SETTINGS_FILE MOUNT_POINT "/config/servoSettings.pb" +#define FS_CONFIG_DIRECTORY LITTLEFS_MOUNT_POINT "/config" +#define DEVICE_CONFIG_FILE LITTLEFS_MOUNT_POINT "/config/peripheral.pb" +#define CAMERA_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/cameraSettings.pb" +#define AP_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/apSettings.pb" +#define MDNS_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/mdnsSettings.pb" +#define WIFI_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/wifiSettings.pb" +#define PERIPHERAL_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/peripheralSettings.pb" +#define SERVO_SETTINGS_FILE LITTLEFS_MOUNT_POINT "/config/servoSettings.pb" namespace FileSystem { diff --git a/esp32/include/filesystem_ws.h b/esp32/include/filesystem_ws.h index 69648fd..2cb4a15 100644 --- a/esp32/include/filesystem_ws.h +++ b/esp32/include/filesystem_ws.h @@ -7,7 +7,7 @@ #include #include -#define FS_MAX_CHUNK_SIZE 16384 +#define FS_MAX_CHUNK_SIZE (1024*64) #define FS_TRANSFER_TIMEOUT_MS 30000 namespace FileSystemWS { diff --git a/esp32/src/filesystem.cpp b/esp32/src/filesystem.cpp index 4a04573..376c7c3 100644 --- a/esp32/src/filesystem.cpp +++ b/esp32/src/filesystem.cpp @@ -6,6 +6,9 @@ #include #include #include +#include "esp_vfs_fat.h" +#include +#include static const char *TAG = "FileSystem"; @@ -79,7 +82,7 @@ void listFilesProto(const std::string &directory, api_FileEntry *entry) { if (path.empty() || path[0] != '/') { path = "/" + directory; } - std::string fullPath = std::string(MOUNT_POINT) + path; + std::string fullPath = path; listFilesProtoRecursive(fullPath, entry); } @@ -108,7 +111,7 @@ esp_err_t getFilesProto(httpd_req_t *request) { bool init() { esp_vfs_littlefs_conf_t conf = { - .base_path = MOUNT_POINT, + .base_path = LITTLEFS_MOUNT_POINT, .partition_label = "spiffs", .format_if_mount_failed = true, .dont_mount = false, @@ -134,6 +137,51 @@ bool init() { mkdirRecursive(FS_CONFIG_DIRECTORY); + + // Optional SD card mounting via SDMMC (1-bit mode for ESP32-S3-CAM) + // Pin definitions - override in build flags if needed +#ifndef SD_CMD_PIN +#define SD_CMD_PIN GPIO_NUM_38 +#endif +#ifndef SD_CLK_PIN +#define SD_CLK_PIN GPIO_NUM_39 +#endif +#ifndef SD_DATA_PIN +#define SD_DATA_PIN GPIO_NUM_40 +#endif + + esp_vfs_fat_sdmmc_mount_config_t sd_mount_config = { + .format_if_mount_failed = false, + .max_files = 4, + .allocation_unit_size = 16 * 1024, + }; + + sdmmc_host_t host = SDMMC_HOST_DEFAULT(); + host.flags = SDMMC_HOST_FLAG_1BIT; // Use 1-bit mode + host.max_freq_khz = SDMMC_FREQ_DEFAULT; + + sdmmc_slot_config_t slot_config = SDMMC_SLOT_CONFIG_DEFAULT(); + slot_config.width = 1; // 1-bit mode + slot_config.clk = SD_CLK_PIN; + slot_config.cmd = SD_CMD_PIN; + slot_config.d0 = SD_DATA_PIN; + slot_config.flags |= SDMMC_SLOT_FLAG_INTERNAL_PULLUP; + + sdmmc_card_t *card = nullptr; + esp_err_t err = esp_vfs_fat_sdmmc_mount(SD_MOUNT_POINT, &host, &slot_config, &sd_mount_config, &card); + if (err != ESP_OK) { + if (err == ESP_FAIL) { + ESP_LOGW(TAG, "Failed to mount SD card filesystem"); + } else { + ESP_LOGW(TAG, "SD card not present or failed to initialize (%s)", esp_err_to_name(err)); + } + // Don't fail - SD card is optional + } else { + ESP_LOGI(TAG, "SD card mounted at %s", SD_MOUNT_POINT); + ESP_LOGI(TAG, "SD card: %s, %lluMB", card->cid.name, + ((uint64_t)card->csd.capacity) * card->csd.sector_size / (1024 * 1024)); + } + return true; } @@ -231,7 +279,7 @@ esp_err_t getFiles(httpd_req_t *request) { esp_err_t getConfigFile(httpd_req_t *request) { const char *uri = request->uri; - std::string path = std::string(MOUNT_POINT) + "/config" + std::string(uri).substr(11); + std::string path = std::string(LITTLEFS_MOUNT_POINT) + "/config" + std::string(uri).substr(11); if (!fileExists(path.c_str())) { return WebServer::sendError(request, 404, "File not found"); @@ -253,7 +301,7 @@ esp_err_t getConfigFile(httpd_req_t *request) { } esp_err_t handleDelete(httpd_req_t *request, const api_FileDeleteRequest &req) { - std::string fullPath = std::string(MOUNT_POINT) + req.path; + std::string fullPath = req.path; ESP_LOGI(TAG, "Deleting file: %s", fullPath.c_str()); api_Response res = api_Response_init_zero; @@ -267,7 +315,7 @@ esp_err_t handleDelete(httpd_req_t *request, const api_FileDeleteRequest &req) { } esp_err_t handleEdit(httpd_req_t *request, const api_FileEditRequest &req) { - std::string fullPath = std::string(MOUNT_POINT) + req.path; + std::string fullPath = req.path; ESP_LOGI(TAG, "Editing file: %s", fullPath.c_str()); api_Response res = api_Response_init_zero; @@ -326,7 +374,7 @@ bool editFile(const char *filename, const uint8_t *content, size_t size) { retur bool editFile(const char *filename, const char *content) { return writeFile(filename, content); } esp_err_t mkdir(httpd_req_t *request, const api_FileMkdirRequest &req) { - std::string fullPath = std::string(MOUNT_POINT) + req.path; + std::string fullPath = req.path; ESP_LOGI(TAG, "Creating directory: %s", fullPath.c_str()); api_Response res = api_Response_init_zero; diff --git a/esp32/src/filesystem_ws.cpp b/esp32/src/filesystem_ws.cpp index f2e48c6..921820f 100644 --- a/esp32/src/filesystem_ws.cpp +++ b/esp32/src/filesystem_ws.cpp @@ -7,6 +7,7 @@ #include #include #include +#include static const char* TAG = "FileSystemWS"; @@ -80,7 +81,7 @@ void FileSystemHandler::cleanupExpiredTransfers() { socket_message_FSDeleteResponse FileSystemHandler::handleDelete(const socket_message_FSDeleteRequest& req) { socket_message_FSDeleteResponse response = socket_message_FSDeleteResponse_init_zero; - std::string path = std::string(MOUNT_POINT) + req.path; + std::string path = req.path; ESP_LOGI(TAG, "Delete request: %s", path.c_str()); struct stat st; @@ -129,7 +130,7 @@ bool FileSystemHandler::deleteRecursive(const std::string& path) { socket_message_FSMkdirResponse FileSystemHandler::handleMkdir(const socket_message_FSMkdirRequest& req) { socket_message_FSMkdirResponse response = socket_message_FSMkdirResponse_init_zero; - std::string path = std::string(MOUNT_POINT) + req.path; + std::string path = req.path; ESP_LOGI(TAG, "Mkdir request: %s", path.c_str()); struct stat st; @@ -150,6 +151,15 @@ socket_message_FSMkdirResponse FileSystemHandler::handleMkdir(const socket_messa } void FileSystemHandler::listDirectory(const std::string& path, socket_message_FSListResponse& response) { + + // Root "/" is virtual - list mount points instead + if (strcmp(path.c_str(), "/") == 0) { + strncpy(response.directories[0].name, LITTLEFS_MOUNT_POINT + 1, sizeof(response.directories[0].name) - 1); + strncpy(response.directories[1].name, SD_MOUNT_POINT + 1, sizeof(response.directories[1].name) - 1); + response.directories_count = 2; + return; + } + DIR* dir = opendir(path.c_str()); if (!dir) { return; @@ -191,15 +201,13 @@ void FileSystemHandler::listDirectory(const std::string& path, socket_message_FS socket_message_FSListResponse FileSystemHandler::handleList(const socket_message_FSListRequest& req) { socket_message_FSListResponse response = socket_message_FSListResponse_init_zero; - std::string path = std::string(MOUNT_POINT); - if (strlen(req.path) > 0 && req.path[0] != '\0') { - path += req.path; - } + std::string path = req.path; ESP_LOGI(TAG, "List request: %s", path.c_str()); struct stat st; - if (stat(path.c_str(), &st) != 0) { + // Make sure that path exists, or that it is a root listing + if (strcmp(path.c_str(), "/") != 0 && stat(path.c_str(), &st) != 0) { response.success = false; strncpy(response.error, "Path not found", sizeof(response.error) - 1); return response; @@ -212,7 +220,7 @@ socket_message_FSListResponse FileSystemHandler::handleList(const socket_message } void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadRequest& req, int clientId) { - std::string path = std::string(MOUNT_POINT) + req.path; + std::string path = req.path; ESP_LOGI(TAG, "Download request: %s", path.c_str()); struct stat st; @@ -268,7 +276,7 @@ void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadReq ESP_LOGI(TAG, "Download started: %s, size=%u, chunks=%u, id=%u", path.c_str(), fileSize, totalChunks, transferId); while (sendNextDownloadChunk(transferId)) { - taskYIELD(); + vTaskDelay(pdMS_TO_TICKS(5)); // Give network time to send (5 ms) } } @@ -308,8 +316,28 @@ bool FileSystemHandler::sendNextDownloadChunk(uint32_t transferId) { bytesToRead = state.fileSize - position; } - size_t bytesRead = fread(data->data.bytes, 1, bytesToRead, state.file); + // Allocate buffer for FT_POINTER data field + data->data = (pb_bytes_array_t*)malloc(PB_BYTES_ARRAY_T_ALLOCSIZE(bytesToRead)); + if (!data->data) { + delete data; + if (sendCompleteCallback_) { + socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero; + complete.transfer_id = transferId; + complete.success = false; + strncpy(complete.error, "Memory allocation failed", sizeof(complete.error) - 1); + complete.total_chunks = state.chunksSent; + complete.file_size = state.fileSize; + sendCompleteCallback_(complete, state.clientId); + } + fclose(state.file); + downloads_.erase(it); + ESP_LOGE(TAG, "Download failed - memory allocation: %u", transferId); + return false; + } + + size_t bytesRead = fread(data->data->bytes, 1, bytesToRead, state.file); if (bytesRead == 0 && bytesToRead > 0) { + free(data->data); delete data; if (sendCompleteCallback_) { socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero; @@ -326,12 +354,13 @@ bool FileSystemHandler::sendNextDownloadChunk(uint32_t transferId) { ESP_LOGE(TAG, "Download failed - read error: %u", transferId); return false; } - data->data.size = bytesRead; + data->data->size = bytesRead; if (sendDataCallback_) { sendDataCallback_(*data, state.clientId); } + free(data->data); delete data; state.chunksSent++; ESP_LOGD(TAG, "Download chunk %u/%u sent: %u bytes", state.chunksSent, state.totalChunks, bytesRead); @@ -343,17 +372,22 @@ socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const int clientId) { socket_message_FSUploadStartResponse response = socket_message_FSUploadStartResponse_init_zero; - std::string path = std::string(MOUNT_POINT) + req.path; + std::string path = req.path; ESP_LOGI(TAG, "Upload start request: %s, size=%u, chunks=%u", path.c_str(), req.file_size, req.total_chunks); - size_t fs_total = 0, fs_used = 0; - esp_littlefs_info("spiffs", &fs_total, &fs_used); - size_t freeSpace = fs_total - fs_used; - if (freeSpace < req.file_size + 4096) { - response.success = false; - strncpy(response.error, "Insufficient storage space", sizeof(response.error) - 1); - return response; + // Check available space on the target filesystem + if (path.find(SD_MOUNT_POINT) != 0) { + // LittleFS path + size_t fs_total = 0, fs_used = 0; + esp_littlefs_info("spiffs", &fs_total, &fs_used); + size_t freeSpace = fs_total - fs_used; + if (freeSpace < req.file_size + 4096) { + response.success = false; + strncpy(response.error, "Insufficient storage space", sizeof(response.error) - 1); + return response; + } } + // TODO: SD card space check skipped - FAT doesn't have a simple API for this size_t lastSlash = path.find_last_of('/'); if (lastSlash != std::string::npos && lastSlash > 0) { @@ -368,8 +402,9 @@ socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const FILE* file = fopen(path.c_str(), "wb"); if (!file) { + ESP_LOGE(TAG, "fopen failed for '%s': %s (errno=%d)", path.c_str(), strerror(errno), errno); response.success = false; - strncpy(response.error, "Cannot open file for writing", sizeof(response.error) - 1); + snprintf(response.error, sizeof(response.error) - 1, "Cannot open file: %s", strerror(errno)); return response; } @@ -416,8 +451,15 @@ void FileSystemHandler::handleUploadData(const socket_message_FSUploadData& req) ESP_LOGW(TAG, "Upload chunk out of order: expected %u, got %u", state.chunksReceived, req.chunk_index); } - size_t bytesWritten = fwrite(req.data.bytes, 1, req.data.size, state.file); - if (bytesWritten != req.data.size) { + if (!req.data || req.data->size == 0) { + state.hasError = true; + state.errorMessage = "Empty or invalid data chunk"; + finalizeUpload(transferId, false, state.errorMessage); + return; + } + + size_t bytesWritten = fwrite(req.data->bytes, 1, req.data->size, state.file); + if (bytesWritten != req.data->size) { state.hasError = true; state.errorMessage = "Failed to write chunk"; finalizeUpload(transferId, false, state.errorMessage); diff --git a/esp32/src/main.cpp b/esp32/src/main.cpp index 591d95c..8ac7330 100644 --- a/esp32/src/main.cpp +++ b/esp32/src/main.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +43,8 @@ WiFiService wifiService; APService apService; void setupServer() { + ESP_LOGI("Main", "Free heap before server: %lu, largest block: %lu", + esp_get_free_heap_size(), heap_caps_get_largest_free_block(MALLOC_CAP_8BIT)); server.config(50 + WWW_ASSETS_COUNT, 16384); server.listen(80); diff --git a/platform_shared/filesystem.options b/platform_shared/filesystem.options index 2daa355..e76faf8 100644 --- a/platform_shared/filesystem.options +++ b/platform_shared/filesystem.options @@ -16,11 +16,11 @@ socket_message.FSListResponse.directories max_count:20 # Streaming download messages socket_message.FSDownloadRequest.path max_size:256 socket_message.FSDownloadMetadata.error max_size:128 -socket_message.FSDownloadData.data max_size:16384 +socket_message.FSDownloadData.data type:FT_POINTER socket_message.FSDownloadComplete.error max_size:128 # Streaming upload messages socket_message.FSUploadStart.path max_size:256 socket_message.FSUploadStartResponse.error max_size:128 -socket_message.FSUploadData.data max_size:16384 +socket_message.FSUploadData.data type:FT_POINTER socket_message.FSUploadComplete.error max_size:128 diff --git a/platformio.ini b/platformio.ini index a1504bc..ccbb375 100644 --- a/platformio.ini +++ b/platformio.ini @@ -82,8 +82,8 @@ platform = espressif32 @ 6.8.1 framework = espidf monitor_speed = 115200 monitor_filters = - direct - esp32_exception_decoder + direct + esp32_exception_decoder build_flags = ${factory_settings.build_flags} ${features.build_flags} @@ -96,6 +96,8 @@ build_flags = -fdata-sections -Wl,--gc-sections -I submodules/nanopb + -D PB_FIELD_32BIT=1 + -D PB_ENABLE_MALLOC=1 -Wno-missing-braces -Wno-format -D CONFIG_HTTPD_WS_SUPPORT=1