From f440fa3973ac6f2fe3b784b50bc5cccc13b7d80e Mon Sep 17 00:00:00 2001 From: Niklas Jensen Date: Mon, 5 Jan 2026 20:33:16 +0100 Subject: [PATCH] Claude: File chunking system --- FILESYSTEM_CHUNKED_TRANSFER.md | 211 ++++++++++ app/env.d.ts | 12 +- .../components/filesystem/FileManager.svelte | 374 ++++++++++++++++++ app/src/lib/filesystem/chunkedTransfer.ts | 326 +++++++++++++++ esp32/include/communication/proto_helpers.h | 16 + esp32/include/filesystem_ws.h | 62 +++ esp32/src/filesystem_ws.cpp | 364 +++++++++++++++++ platform_shared/message.proto | 126 +++++- 8 files changed, 1472 insertions(+), 19 deletions(-) create mode 100644 FILESYSTEM_CHUNKED_TRANSFER.md create mode 100644 app/src/lib/components/filesystem/FileManager.svelte create mode 100644 app/src/lib/filesystem/chunkedTransfer.ts create mode 100644 esp32/include/filesystem_ws.h create mode 100644 esp32/src/filesystem_ws.cpp diff --git a/FILESYSTEM_CHUNKED_TRANSFER.md b/FILESYSTEM_CHUNKED_TRANSFER.md new file mode 100644 index 0000000..c7cae22 --- /dev/null +++ b/FILESYSTEM_CHUNKED_TRANSFER.md @@ -0,0 +1,211 @@ +# Chunked Filesystem Transfer System + +This system enables chunked file uploads and downloads between the client (web browser) and ESP32 over WebSocket, overcoming the 1KB per stream limitation. + +## Architecture + +### Protocol Messages (Protobuf) + +The system uses Protocol Buffers for efficient binary messaging with the following operations: + +1. **Delete**: Remove files or directories +2. **Mkdir**: Create directories +3. **List**: List files and directories +4. **Download**: Transfer files from ESP32 to client (chunked) +5. **Upload**: Transfer files from client to ESP32 (chunked) +6. **Cancel**: Cancel ongoing transfers + +### Chunked Transfer Flow + +#### Download (ESP32 → Client) + +1. Client sends `FSDownloadStartRequest` with file path +2. ESP32 responds with `FSDownloadStartResponse` containing: + - Transfer ID + - File size + - Chunk size (1024 bytes max) + - Total chunks +3. Client requests chunks sequentially using `FSDownloadChunkRequest` +4. ESP32 sends each chunk via `FSDownloadChunkResponse` +5. Transfer completes when last chunk is received + +#### Upload (Client → ESP32) + +1. Client sends `FSUploadStartRequest` with destination path and file size +2. ESP32 responds with `FSUploadStartResponse` containing: + - Transfer ID + - Max chunk size (1024 bytes) +3. Client sends chunks sequentially using `FSUploadChunkRequest` +4. ESP32 responds with `FSUploadChunkResponse` after each chunk +5. Transfer completes when last chunk is written + +## Implementation Details + +### ESP32 Side + +**Files:** +- `esp32/include/filesystem_ws.h` - Header file with handler class definition +- `esp32/src/filesystem_ws.cpp` - Implementation of filesystem operations +- `esp32/include/communication/proto_helpers.h` - Message traits for protobuf + +**Key Features:** +- Transfer state management with automatic cleanup +- Timeout handling (30 seconds of inactivity) +- Recursive directory deletion +- File integrity verification + +**Integration:** +You need to integrate the filesystem handlers into your WebSocket message handling. In your main WebSocket handler, add: + +```cpp +#include + +// In your correlation request handler: +void handleCorrelationRequest(const socket_message_CorrelationRequest& request, int clientId) { + socket_message_CorrelationResponse response; + + // ... existing handlers ... + + if (request.which_request == socket_message_CorrelationRequest_fs_delete_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_delete_response_tag; + response.response.fs_delete_response = + FileSystemWS::fsHandler.handleDelete(request.request.fs_delete_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_mkdir_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_mkdir_response_tag; + response.response.fs_mkdir_response = + FileSystemWS::fsHandler.handleMkdir(request.request.fs_mkdir_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_list_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_list_response_tag; + response.response.fs_list_response = + FileSystemWS::fsHandler.handleList(request.request.fs_list_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_download_start_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_download_start_response_tag; + response.response.fs_download_start_response = + FileSystemWS::fsHandler.handleDownloadStart(request.request.fs_download_start_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_download_chunk_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_download_chunk_response_tag; + response.response.fs_download_chunk_response = + FileSystemWS::fsHandler.handleDownloadChunk(request.request.fs_download_chunk_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_upload_start_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_upload_start_response_tag; + response.response.fs_upload_start_response = + FileSystemWS::fsHandler.handleUploadStart(request.request.fs_upload_start_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_upload_chunk_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_upload_chunk_response_tag; + response.response.fs_upload_chunk_response = + FileSystemWS::fsHandler.handleUploadChunk(request.request.fs_upload_chunk_request); + } + else if (request.which_request == socket_message_CorrelationRequest_fs_cancel_transfer_request_tag) { + response.which_response = socket_message_CorrelationResponse_fs_cancel_transfer_response_tag; + response.response.fs_cancel_transfer_response = + FileSystemWS::fsHandler.handleCancelTransfer(request.request.fs_cancel_transfer_request); + } + + // Send response back to client + sendCorrelationResponse(response, clientId); +} + +// Optionally, in your main loop or timer: +void loop() { + // Clean up expired transfers periodically + FileSystemWS::fsHandler.cleanupExpiredTransfers(); +} +``` + +### Client Side (TypeScript/Svelte) + +**Files:** +- `app/src/lib/filesystem/chunkedTransfer.ts` - Client library for file transfers +- `app/src/lib/components/filesystem/FileManager.svelte` - Example UI component + +**Usage Example:** + +```typescript +import { fileSystemClient } from '$lib/filesystem/chunkedTransfer' + +// Upload a file +const file = new File(['Hello World'], 'test.txt') +const result = await fileSystemClient.uploadFileFromBrowser('/test.txt', file, (progress) => { + console.log(`Upload: ${progress.percentage}%`) +}) + +// Download a file +const download = await fileSystemClient.downloadFileAndSave( + '/test.txt', + 'test.txt', + (progress) => { + console.log(`Download: ${progress.percentage}%`) + } +) + +// List directory +const listing = await fileSystemClient.listDirectory('/') +console.log('Files:', listing.files) +console.log('Directories:', listing.directories) + +// Create directory +await fileSystemClient.createDirectory('/new_folder') + +// Delete file +await fileSystemClient.deleteFile('/old_file.txt') +``` + +## Configuration + +### Maximum Chunk Size + +Currently set to 1024 bytes (`FS_MAX_CHUNK_SIZE`) to work within ESP32 WebSocket frame limitations. Adjust if your setup allows larger frames. + +### Transfer Timeout + +Transfers inactive for 30 seconds (`FS_TRANSFER_TIMEOUT`) are automatically cleaned up. Increase for slower connections. + +## Error Handling + +- Network errors: Transfers are automatically cancelled +- Timeouts: Inactive transfers are cleaned up on ESP32 +- File errors: Detailed error messages returned to client +- Partial uploads: Cancelled uploads delete the partial file on ESP32 + +## Performance Considerations + +- **Sequential Chunks**: Chunks are sent sequentially to ensure order and reliability +- **Memory Usage**: ESP32 keeps one File handle open per active transfer +- **Browser Memory**: Downloads buffer entire file in memory before saving +- **Network**: ~1KB per message overhead due to protobuf encoding + +## Security Notes + +- No authentication/authorization implemented - add as needed +- Path traversal: Validate paths to prevent access outside allowed directories +- File size limits: Consider adding max file size restrictions +- Rate limiting: Consider limiting concurrent transfers per client + +## Testing + +1. Build and flash the ESP32 firmware +2. Run the web application +3. Navigate to the FileManager component +4. Test upload/download with files of various sizes + +## Troubleshooting + +**Transfer fails midway:** +- Check WebSocket connection stability +- Verify ESP32 has sufficient filesystem space +- Check for timeout issues + +**Upload creates corrupted files:** +- Verify chunk order is preserved +- Check for protobuf encoding/decoding errors + +**ESP32 runs out of memory:** +- Reduce number of concurrent transfers +- Close File handles properly after transfers +- Run cleanup more frequently diff --git a/app/env.d.ts b/app/env.d.ts index 8c7f1c8..fe77695 100644 --- a/app/env.d.ts +++ b/app/env.d.ts @@ -1,8 +1,8 @@ -declare module 'app-env' { - interface ENV { - VITE_USE_HOST_NAME: boolean - } +declare module "app-env" { + interface ENV { + VITE_USE_HOST_NAME: boolean; + } - const appEnv: ENV - export default appEnv + const appEnv: ENV; + export default appEnv; } diff --git a/app/src/lib/components/filesystem/FileManager.svelte b/app/src/lib/components/filesystem/FileManager.svelte new file mode 100644 index 0000000..637066a --- /dev/null +++ b/app/src/lib/components/filesystem/FileManager.svelte @@ -0,0 +1,374 @@ + + +
+
+

File Manager

+
Current: {currentPath}
+
+ + + +
+
+ + {#if error} +
{error}
+ {/if} + + {#if uploadProgress} +
+
+ Uploading: {uploadProgress.percentage.toFixed(1)}% ({formatBytes( + uploadProgress.bytesTransferred + )} / {formatBytes(uploadProgress.totalBytes)}) +
+
+
+
+
+ {/if} + + {#if downloadProgress} +
+
+ Downloading: {downloadProgress.percentage.toFixed(1)}% ({formatBytes( + downloadProgress.bytesTransferred + )} / {formatBytes(downloadProgress.totalBytes)}) +
+
+
+
+
+ {/if} + +
+ {#if loading} +
Loading...
+ {:else} + {#if currentPath !== '/'} +
navigateTo('/')}> + 📁 + .. +
+ {/if} + + {#each directories as dir} +
+ 📁 + navigateTo(currentPath + '/' + dir.name)} + >{dir.name} + +
+ {/each} + + {#each files as file} +
+ 📄 + {file.name} + {formatBytes(file.size)} + + +
+ {/each} + + {#if files.length === 0 && directories.length === 0} +
Directory is empty
+ {/if} + {/if} +
+
+ + diff --git a/app/src/lib/filesystem/chunkedTransfer.ts b/app/src/lib/filesystem/chunkedTransfer.ts new file mode 100644 index 0000000..7ad9914 --- /dev/null +++ b/app/src/lib/filesystem/chunkedTransfer.ts @@ -0,0 +1,326 @@ +import { socket } from '$lib/stores/socket' +import * as Messages from '$lib/platform_shared/message' +import type { + FSDeleteRequest, + FSMkdirRequest, + FSListRequest, + FSDownloadStartRequest, + FSDownloadChunkRequest, + FSUploadStartRequest, + FSUploadChunkRequest, + FSCancelTransferRequest, + CorrelationResponse +} from '$lib/platform_shared/message' + +const MAX_CHUNK_SIZE = 1024 + +export interface FileInfo { + name: string + size: number +} + +export interface DirectoryInfo { + name: string +} + +export interface ListResult { + success: boolean + error?: string + files: FileInfo[] + directories: DirectoryInfo[] +} + +export interface TransferProgress { + transferId: string + bytesTransferred: number + totalBytes: number + chunksCompleted: number + totalChunks: number + percentage: number +} + +export type ProgressCallback = (progress: TransferProgress) => void + +export class FileSystemClient { + /** + * Delete a file or directory on the ESP32 + */ + async deleteFile(path: string): Promise<{ success: boolean; error?: string }> { + const request: FSDeleteRequest = { path } + + const response = await socket.request({ + fsDeleteRequest: request + }) + + if (response.fsDeleteResponse) { + return { + success: response.fsDeleteResponse.success, + error: response.fsDeleteResponse.error || undefined + } + } + + return { success: false, error: 'No response received' } + } + + /** + * Create a directory on the ESP32 + */ + async createDirectory(path: string): Promise<{ success: boolean; error?: string }> { + const request: FSMkdirRequest = { path } + + const response = await socket.request({ + fsMkdirRequest: request + }) + + if (response.fsMkdirResponse) { + return { + success: response.fsMkdirResponse.success, + error: response.fsMkdirResponse.error || undefined + } + } + + return { success: false, error: 'No response received' } + } + + /** + * List files and directories at the given path + */ + async listDirectory(path: string = '/'): Promise { + const request: FSListRequest = { path } + + const response = await socket.request({ + fsListRequest: request + }) + + if (response.fsListResponse) { + const resp = response.fsListResponse + return { + success: resp.success, + error: resp.error || undefined, + files: (resp.files || []).map((f) => ({ name: f.name, size: f.size })), + directories: (resp.directories || []).map((d) => ({ name: d.name })) + } + } + + return { success: false, error: 'No response received', files: [], directories: [] } + } + + /** + * Download a file from the ESP32 + */ + async downloadFile( + path: string, + onProgress?: ProgressCallback + ): Promise<{ success: boolean; data?: Uint8Array; error?: string }> { + // Start download + const startRequest: FSDownloadStartRequest = { path } + + const startResponse = await socket.request({ + fsDownloadStartRequest: startRequest + }) + + if (!startResponse.fsDownloadStartResponse) { + return { success: false, error: 'Failed to start download' } + } + + const startResp = startResponse.fsDownloadStartResponse + + if (!startResp.success) { + return { success: false, error: startResp.error || 'Failed to start download' } + } + + const transferId = startResp.transferId + const totalChunks = startResp.totalChunks + const fileSize = startResp.fileSize + + // Allocate buffer for entire file + const buffer = new Uint8Array(fileSize) + let offset = 0 + + // Download chunks sequentially + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const chunkRequest: FSDownloadChunkRequest = { + transferId, + chunkIndex + } + + const chunkResponse = await socket.request({ + fsDownloadChunkRequest: chunkRequest + }) + + if (!chunkResponse.fsDownloadChunkResponse) { + await this.cancelTransfer(transferId) + return { success: false, error: `Failed to download chunk ${chunkIndex}` } + } + + const chunkResp = chunkResponse.fsDownloadChunkResponse + + if (chunkResp.error) { + await this.cancelTransfer(transferId) + return { success: false, error: chunkResp.error } + } + + // Copy chunk data to buffer + if (chunkResp.data) { + buffer.set(chunkResp.data, offset) + offset += chunkResp.data.length + } + + // Report progress + if (onProgress) { + onProgress({ + transferId, + bytesTransferred: offset, + totalBytes: fileSize, + chunksCompleted: chunkIndex + 1, + totalChunks, + percentage: ((chunkIndex + 1) / totalChunks) * 100 + }) + } + } + + return { success: true, data: buffer } + } + + /** + * Upload a file to the ESP32 + */ + async uploadFile( + path: string, + data: Uint8Array, + onProgress?: ProgressCallback + ): Promise<{ success: boolean; error?: string }> { + const fileSize = data.length + const chunkSize = MAX_CHUNK_SIZE + const totalChunks = Math.ceil(fileSize / chunkSize) + + // Start upload + const startRequest: FSUploadStartRequest = { + path, + fileSize, + chunkSize + } + + const startResponse = await socket.request({ + fsUploadStartRequest: startRequest + }) + + if (!startResponse.fsUploadStartResponse) { + return { success: false, error: 'Failed to start upload' } + } + + const startResp = startResponse.fsUploadStartResponse + + if (!startResp.success) { + return { success: false, error: startResp.error || 'Failed to start upload' } + } + + const transferId = startResp.transferId + const maxChunkSize = startResp.maxChunkSize || MAX_CHUNK_SIZE + + // Upload chunks sequentially + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { + const offset = chunkIndex * chunkSize + const end = Math.min(offset + chunkSize, fileSize) + const chunkData = data.slice(offset, end) + const isLast = chunkIndex === totalChunks - 1 + + const chunkRequest: FSUploadChunkRequest = { + transferId, + chunkIndex, + data: chunkData, + isLast + } + + const chunkResponse = await socket.request({ + fsUploadChunkRequest: chunkRequest + }) + + if (!chunkResponse.fsUploadChunkResponse) { + await this.cancelTransfer(transferId) + return { success: false, error: `Failed to upload chunk ${chunkIndex}` } + } + + const chunkResp = chunkResponse.fsUploadChunkResponse + + if (!chunkResp.success) { + await this.cancelTransfer(transferId) + return { success: false, error: chunkResp.error || 'Failed to upload chunk' } + } + + // Report progress + if (onProgress) { + onProgress({ + transferId, + bytesTransferred: end, + totalBytes: fileSize, + chunksCompleted: chunkIndex + 1, + totalChunks, + percentage: ((chunkIndex + 1) / totalChunks) * 100 + }) + } + } + + return { success: true } + } + + /** + * Cancel an ongoing transfer + */ + async cancelTransfer(transferId: string): Promise<{ success: boolean }> { + const request: FSCancelTransferRequest = { transferId } + + const response = await socket.request({ + fsCancelTransferRequest: request + }) + + if (response.fsCancelTransferResponse) { + return { success: response.fsCancelTransferResponse.success } + } + + return { success: false } + } + + /** + * Helper: Upload a File object from browser + */ + async uploadFileFromBrowser( + destinationPath: string, + file: File, + onProgress?: ProgressCallback + ): Promise<{ success: boolean; error?: string }> { + const arrayBuffer = await file.arrayBuffer() + const data = new Uint8Array(arrayBuffer) + return this.uploadFile(destinationPath, data, onProgress) + } + + /** + * Helper: Download a file and save it to browser + */ + async downloadFileAndSave( + path: string, + filename: string, + onProgress?: ProgressCallback + ): Promise<{ success: boolean; error?: string }> { + const result = await this.downloadFile(path, onProgress) + + if (!result.success || !result.data) { + return { success: false, error: result.error } + } + + // Create blob and trigger download + const blob = new Blob([result.data]) + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = filename + document.body.appendChild(a) + a.click() + document.body.removeChild(a) + URL.revokeObjectURL(url) + + return { success: true } + } +} + +export const fileSystemClient = new FileSystemClient() diff --git a/esp32/include/communication/proto_helpers.h b/esp32/include/communication/proto_helpers.h index 3f7004c..3fdf9dc 100644 --- a/esp32/include/communication/proto_helpers.h +++ b/esp32/include/communication/proto_helpers.h @@ -41,6 +41,22 @@ DEFINE_MESSAGE_TRAITS(ServoPWMData, servo_pwm) DEFINE_MESSAGE_TRAITS(ServoStateData, servo_state) DEFINE_MESSAGE_TRAITS(CorrelationRequest, correlation_request) DEFINE_MESSAGE_TRAITS(CorrelationResponse, correlation_response) +DEFINE_MESSAGE_TRAITS(FSDeleteRequest, fs_delete_request) +DEFINE_MESSAGE_TRAITS(FSDeleteResponse, fs_delete_response) +DEFINE_MESSAGE_TRAITS(FSMkdirRequest, fs_mkdir_request) +DEFINE_MESSAGE_TRAITS(FSMkdirResponse, fs_mkdir_response) +DEFINE_MESSAGE_TRAITS(FSListRequest, fs_list_request) +DEFINE_MESSAGE_TRAITS(FSListResponse, fs_list_response) +DEFINE_MESSAGE_TRAITS(FSDownloadStartRequest, fs_download_start_request) +DEFINE_MESSAGE_TRAITS(FSDownloadStartResponse, fs_download_start_response) +DEFINE_MESSAGE_TRAITS(FSDownloadChunkRequest, fs_download_chunk_request) +DEFINE_MESSAGE_TRAITS(FSDownloadChunkResponse, fs_download_chunk_response) +DEFINE_MESSAGE_TRAITS(FSUploadStartRequest, fs_upload_start_request) +DEFINE_MESSAGE_TRAITS(FSUploadStartResponse, fs_upload_start_response) +DEFINE_MESSAGE_TRAITS(FSUploadChunkRequest, fs_upload_chunk_request) +DEFINE_MESSAGE_TRAITS(FSUploadChunkResponse, fs_upload_chunk_response) +DEFINE_MESSAGE_TRAITS(FSCancelTransferRequest, fs_cancel_transfer_request) +DEFINE_MESSAGE_TRAITS(FSCancelTransferResponse, fs_cancel_transfer_response) #undef DEFINE_MESSAGE_TRAITS diff --git a/esp32/include/filesystem_ws.h b/esp32/include/filesystem_ws.h new file mode 100644 index 0000000..58c08d6 --- /dev/null +++ b/esp32/include/filesystem_ws.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include +#include + +#define FS_MAX_CHUNK_SIZE 1024 +#define FS_TRANSFER_TIMEOUT 30000 // 30 seconds + +namespace FileSystemWS { + +struct TransferState { + std::string path; + File file; + uint32_t fileSize; + uint32_t chunkSize; + uint32_t totalChunks; + uint32_t chunksProcessed; + uint32_t lastActivityTime; + bool isUpload; +}; + +class FileSystemHandler { + public: + FileSystemHandler(); + + // Delete file/directory + socket_message_FSDeleteResponse handleDelete(const socket_message_FSDeleteRequest& req); + + // Create directory + socket_message_FSMkdirResponse handleMkdir(const socket_message_FSMkdirRequest& req); + + // List directory + socket_message_FSListResponse handleList(const socket_message_FSListRequest& req); + + // Download operations (ESP -> Client) + socket_message_FSDownloadStartResponse handleDownloadStart(const socket_message_FSDownloadStartRequest& req); + socket_message_FSDownloadChunkResponse handleDownloadChunk(const socket_message_FSDownloadChunkRequest& req); + + // Upload operations (Client -> ESP) + socket_message_FSUploadStartResponse handleUploadStart(const socket_message_FSUploadStartRequest& req); + socket_message_FSUploadChunkResponse handleUploadChunk(const socket_message_FSUploadChunkRequest& req); + + // Cancel transfer + socket_message_FSCancelTransferResponse handleCancelTransfer(const socket_message_FSCancelTransferRequest& req); + + // Cleanup expired transfers + void cleanupExpiredTransfers(); + + private: + std::map transfers_; + uint32_t transferIdCounter_; + + std::string generateTransferId(); + void listDirectory(const std::string& path, socket_message_FSListResponse& response); + bool deleteRecursive(const std::string& path); +}; + +extern FileSystemHandler fsHandler; + +} // namespace FileSystemWS diff --git a/esp32/src/filesystem_ws.cpp b/esp32/src/filesystem_ws.cpp new file mode 100644 index 0000000..fbd939f --- /dev/null +++ b/esp32/src/filesystem_ws.cpp @@ -0,0 +1,364 @@ +#include +#include +#include +#include + +static const char* TAG = "FileSystemWS"; + +namespace FileSystemWS { + +FileSystemHandler fsHandler; + +FileSystemHandler::FileSystemHandler() : transferIdCounter_(0) {} + +std::string FileSystemHandler::generateTransferId() { + return "xfer_" + std::to_string(millis()) + "_" + std::to_string(++transferIdCounter_); +} + +void FileSystemHandler::cleanupExpiredTransfers() { + uint32_t now = millis(); + auto it = transfers_.begin(); + while (it != transfers_.end()) { + if (now - it->second.lastActivityTime > FS_TRANSFER_TIMEOUT) { + if (it->second.file) { + it->second.file.close(); + } + ESP_LOGW(TAG, "Transfer %s timed out", it->first.c_str()); + it = transfers_.erase(it); + } else { + ++it; + } + } +} + +socket_message_FSDeleteResponse FileSystemHandler::handleDelete(const socket_message_FSDeleteRequest& req) { + socket_message_FSDeleteResponse response = socket_message_FSDeleteResponse_init_zero; + + std::string path(req.path); + ESP_LOGI(TAG, "Delete request: %s", path.c_str()); + + if (!ESP_FS.exists(path.c_str())) { + response.success = false; + strncpy(response.error, "File or directory not found", sizeof(response.error) - 1); + return response; + } + + if (deleteRecursive(path)) { + response.success = true; + } else { + response.success = false; + strncpy(response.error, "Failed to delete", sizeof(response.error) - 1); + } + + return response; +} + +bool FileSystemHandler::deleteRecursive(const std::string& path) { + File file = ESP_FS.open(path.c_str()); + if (!file) return false; + + if (file.isDirectory()) { + File child = file.openNextFile(); + while (child) { + std::string childPath = std::string(child.name()); + child.close(); + if (!deleteRecursive(childPath)) { + file.close(); + return false; + } + child = file.openNextFile(); + } + file.close(); + return ESP_FS.rmdir(path.c_str()); + } else { + file.close(); + return ESP_FS.remove(path.c_str()); + } +} + +socket_message_FSMkdirResponse FileSystemHandler::handleMkdir(const socket_message_FSMkdirRequest& req) { + socket_message_FSMkdirResponse response = socket_message_FSMkdirResponse_init_zero; + + std::string path(req.path); + ESP_LOGI(TAG, "Mkdir request: %s", path.c_str()); + + if (ESP_FS.exists(path.c_str())) { + response.success = false; + strncpy(response.error, "Path already exists", sizeof(response.error) - 1); + return response; + } + + if (ESP_FS.mkdir(path.c_str())) { + response.success = true; + } else { + response.success = false; + strncpy(response.error, "Failed to create directory", sizeof(response.error) - 1); + } + + return response; +} + +void FileSystemHandler::listDirectory(const std::string& path, socket_message_FSListResponse& response) { + File dir = ESP_FS.open(path.c_str()); + if (!dir || !dir.isDirectory()) { + return; + } + + File file = dir.openNextFile(); + int fileCount = 0; + int dirCount = 0; + + while (file && fileCount < 50 && dirCount < 50) { // Limit to prevent overflow + if (file.isDirectory()) { + if (dirCount < 50) { + strncpy(response.directories[dirCount].name, file.name(), sizeof(response.directories[dirCount].name) - 1); + dirCount++; + } + } else { + if (fileCount < 50) { + strncpy(response.files[fileCount].name, file.name(), sizeof(response.files[fileCount].name) - 1); + response.files[fileCount].size = file.size(); + fileCount++; + } + } + file = dir.openNextFile(); + } + + response.files_count = fileCount; + response.directories_count = dirCount; +} + +socket_message_FSListResponse FileSystemHandler::handleList(const socket_message_FSListRequest& req) { + socket_message_FSListResponse response = socket_message_FSListResponse_init_zero; + + std::string path(req.path); + if (path.empty()) path = "/"; + + ESP_LOGI(TAG, "List request: %s", path.c_str()); + + if (!ESP_FS.exists(path.c_str())) { + response.success = false; + strncpy(response.error, "Path not found", sizeof(response.error) - 1); + return response; + } + + listDirectory(path, response); + response.success = true; + + return response; +} + +socket_message_FSDownloadStartResponse FileSystemHandler::handleDownloadStart(const socket_message_FSDownloadStartRequest& req) { + socket_message_FSDownloadStartResponse response = socket_message_FSDownloadStartResponse_init_zero; + + std::string path(req.path); + ESP_LOGI(TAG, "Download start request: %s", path.c_str()); + + if (!ESP_FS.exists(path.c_str())) { + response.success = false; + strncpy(response.error, "File not found", sizeof(response.error) - 1); + return response; + } + + File file = ESP_FS.open(path.c_str(), "r"); + if (!file || file.isDirectory()) { + response.success = false; + strncpy(response.error, "Cannot open file for reading", sizeof(response.error) - 1); + return response; + } + + uint32_t fileSize = file.size(); + uint32_t chunkSize = FS_MAX_CHUNK_SIZE; + uint32_t totalChunks = (fileSize + chunkSize - 1) / chunkSize; + + std::string transferId = generateTransferId(); + + TransferState state; + state.path = path; + state.file = file; + state.fileSize = fileSize; + state.chunkSize = chunkSize; + state.totalChunks = totalChunks; + state.chunksProcessed = 0; + state.lastActivityTime = millis(); + state.isUpload = false; + + transfers_[transferId] = state; + + response.success = true; + response.file_size = fileSize; + response.chunk_size = chunkSize; + response.total_chunks = totalChunks; + strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1); + + ESP_LOGI(TAG, "Download started: %s, size=%u, chunks=%u, id=%s", path.c_str(), fileSize, totalChunks, transferId.c_str()); + + return response; +} + +socket_message_FSDownloadChunkResponse FileSystemHandler::handleDownloadChunk(const socket_message_FSDownloadChunkRequest& req) { + socket_message_FSDownloadChunkResponse response = socket_message_FSDownloadChunkResponse_init_zero; + + std::string transferId(req.transfer_id); + strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1); + response.chunk_index = req.chunk_index; + + auto it = transfers_.find(transferId); + if (it == transfers_.end()) { + strncpy(response.error, "Invalid transfer ID", sizeof(response.error) - 1); + return response; + } + + TransferState& state = it->second; + state.lastActivityTime = millis(); + + // Seek to chunk position + uint32_t position = req.chunk_index * state.chunkSize; + if (!state.file.seek(position)) { + strncpy(response.error, "Failed to seek file", sizeof(response.error) - 1); + return response; + } + + // Calculate chunk size (last chunk might be smaller) + uint32_t bytesToRead = state.chunkSize; + if (req.chunk_index == state.totalChunks - 1) { + bytesToRead = state.fileSize - position; + } + + // Read chunk data + size_t bytesRead = state.file.read(response.data.bytes, bytesToRead); + response.data.size = bytesRead; + + response.is_last = (req.chunk_index == state.totalChunks - 1); + + ESP_LOGI(TAG, "Download chunk %u/%u: %u bytes", req.chunk_index + 1, state.totalChunks, bytesRead); + + // Cleanup if last chunk + if (response.is_last) { + state.file.close(); + transfers_.erase(it); + ESP_LOGI(TAG, "Download completed: %s", transferId.c_str()); + } + + return response; +} + +socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const socket_message_FSUploadStartRequest& req) { + socket_message_FSUploadStartResponse response = socket_message_FSUploadStartResponse_init_zero; + + std::string path(req.path); + ESP_LOGI(TAG, "Upload start request: %s, size=%u", path.c_str(), req.file_size); + + // Ensure parent directory exists + size_t lastSlash = path.find_last_of('/'); + if (lastSlash != std::string::npos && lastSlash > 0) { + std::string parentDir = path.substr(0, lastSlash); + if (!ESP_FS.exists(parentDir.c_str())) { + response.success = false; + strncpy(response.error, "Parent directory does not exist", sizeof(response.error) - 1); + return response; + } + } + + File file = ESP_FS.open(path.c_str(), FILE_WRITE); + if (!file) { + response.success = false; + strncpy(response.error, "Cannot open file for writing", sizeof(response.error) - 1); + return response; + } + + std::string transferId = generateTransferId(); + + TransferState state; + state.path = path; + state.file = file; + state.fileSize = req.file_size; + state.chunkSize = req.chunk_size > FS_MAX_CHUNK_SIZE ? FS_MAX_CHUNK_SIZE : req.chunk_size; + state.totalChunks = (req.file_size + state.chunkSize - 1) / state.chunkSize; + state.chunksProcessed = 0; + state.lastActivityTime = millis(); + state.isUpload = true; + + transfers_[transferId] = state; + + response.success = true; + response.max_chunk_size = FS_MAX_CHUNK_SIZE; + strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1); + + ESP_LOGI(TAG, "Upload started: %s, id=%s", path.c_str(), transferId.c_str()); + + return response; +} + +socket_message_FSUploadChunkResponse FileSystemHandler::handleUploadChunk(const socket_message_FSUploadChunkRequest& req) { + socket_message_FSUploadChunkResponse response = socket_message_FSUploadChunkResponse_init_zero; + + std::string transferId(req.transfer_id); + strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1); + response.chunk_index = req.chunk_index; + + auto it = transfers_.find(transferId); + if (it == transfers_.end()) { + response.success = false; + strncpy(response.error, "Invalid transfer ID", sizeof(response.error) - 1); + return response; + } + + TransferState& state = it->second; + state.lastActivityTime = millis(); + + // Write chunk data + size_t bytesWritten = state.file.write(req.data.bytes, req.data.size); + if (bytesWritten != req.data.size) { + response.success = false; + strncpy(response.error, "Failed to write chunk", sizeof(response.error) - 1); + state.file.close(); + transfers_.erase(it); + return response; + } + + state.chunksProcessed++; + response.success = true; + response.transfer_complete = req.is_last; + + ESP_LOGI(TAG, "Upload chunk %u/%u: %u bytes", state.chunksProcessed, state.totalChunks, bytesWritten); + + // Cleanup if last chunk + if (req.is_last) { + state.file.close(); + transfers_.erase(it); + ESP_LOGI(TAG, "Upload completed: %s", state.path.c_str()); + } + + return response; +} + +socket_message_FSCancelTransferResponse FileSystemHandler::handleCancelTransfer(const socket_message_FSCancelTransferRequest& req) { + socket_message_FSCancelTransferResponse response = socket_message_FSCancelTransferResponse_init_zero; + + std::string transferId(req.transfer_id); + auto it = transfers_.find(transferId); + + if (it == transfers_.end()) { + response.success = false; + return response; + } + + if (it->second.file) { + it->second.file.close(); + } + + // Delete partial upload file + if (it->second.isUpload) { + ESP_FS.remove(it->second.path.c_str()); + } + + transfers_.erase(it); + response.success = true; + + ESP_LOGI(TAG, "Transfer cancelled: %s", transferId.c_str()); + + return response; +} + +} // namespace FileSystemWS diff --git a/platform_shared/message.proto b/platform_shared/message.proto index 1e5c4d7..1e9b2d9 100644 --- a/platform_shared/message.proto +++ b/platform_shared/message.proto @@ -11,28 +11,112 @@ message KnownNetworkItem { string ssid = 1; string password = 2; bool static_ip // ----- FILESYSTEM ----- -message File { string name = 10; } // Add permissions? +message File { + string name = 10; + uint32 size = 20; +} + message Directory { string name = 10; repeated File files = 20; repeated Directory directories = 30; } -enum FileRequestOptions { - DELETE = 0; - EDIT = 1; - MKDIR = 2; - GET = 3; - POST = 4; - START_UPLOAD = 10; +// Delete a file or directory +message FSDeleteRequest { + string path = 1; } -message FSDelete { string path = 1; } +message FSDeleteResponse { + bool success = 1; + string error = 2; +} -message FilesystemRequest { - oneof type { - - } +// Create directory +message FSMkdirRequest { + string path = 1; +} + +message FSMkdirResponse { + bool success = 1; + string error = 2; +} + +// List files/directories +message FSListRequest { + string path = 1; +} + +message FSListResponse { + bool success = 1; + string error = 2; + repeated File files = 3; + repeated Directory directories = 4; +} + +// Download from ESP (ESP -> Client) +message FSDownloadStartRequest { + string path = 1; // File path on ESP to download +} + +message FSDownloadStartResponse { + bool success = 1; + string error = 2; + uint32 file_size = 3; // Total file size in bytes + uint32 chunk_size = 4; // Size of each chunk (max 1024) + uint32 total_chunks = 5; // Total number of chunks + string transfer_id = 6; // Unique ID for this transfer +} + +message FSDownloadChunkRequest { + string transfer_id = 1; + uint32 chunk_index = 2; // Which chunk to request (0-based) +} + +message FSDownloadChunkResponse { + string transfer_id = 1; + uint32 chunk_index = 2; + bytes data = 3; // Chunk data (max 1024 bytes) + bool is_last = 4; // True if this is the last chunk + string error = 5; +} + +// Upload to ESP (Client -> ESP) +message FSUploadStartRequest { + string path = 1; // Destination path on ESP + uint32 file_size = 2; // Total file size in bytes + uint32 chunk_size = 3; // Size of each chunk (max 1024) +} + +message FSUploadStartResponse { + bool success = 1; + string error = 2; + string transfer_id = 3; // Unique ID for this transfer + uint32 max_chunk_size = 4; // Server's max chunk size (1024) +} + +message FSUploadChunkRequest { + string transfer_id = 1; + uint32 chunk_index = 2; // Which chunk this is (0-based) + bytes data = 3; // Chunk data (max 1024 bytes) + bool is_last = 4; // True if this is the last chunk +} + +message FSUploadChunkResponse { + string transfer_id = 1; + uint32 chunk_index = 2; + bool success = 3; + string error = 4; + bool transfer_complete = 5; // True when all chunks received +} + +// Cancel transfer +message FSCancelTransferRequest { + string transfer_id = 1; +} + +message FSCancelTransferResponse { + bool success = 1; } // ----- FILESYSTEM ----- @@ -83,6 +167,14 @@ message CorrelationRequest { I2CScanDataRequest i2c_scan_data_request = 20; IMUCalibrateExecute imu_calibrate_execute = 30; SystemInformationRequest system_information_request = 40; + FSDeleteRequest fs_delete_request = 50; + FSMkdirRequest fs_mkdir_request = 60; + FSListRequest fs_list_request = 70; + FSDownloadStartRequest fs_download_start_request = 80; + FSDownloadChunkRequest fs_download_chunk_request = 90; + FSUploadStartRequest fs_upload_start_request = 100; + FSUploadChunkRequest fs_upload_chunk_request = 110; + FSCancelTransferRequest fs_cancel_transfer_request = 120; } } @@ -94,6 +186,14 @@ message CorrelationResponse { I2CScanData i2c_scan_data = 20; IMUCalibrateData imu_calibrate_data = 30; SystemInformationResponse system_information_response = 40; + FSDeleteResponse fs_delete_response = 50; + FSMkdirResponse fs_mkdir_response = 60; + FSListResponse fs_list_response = 70; + FSDownloadStartResponse fs_download_start_response = 80; + FSDownloadChunkResponse fs_download_chunk_response = 90; + FSUploadStartResponse fs_upload_start_response = 100; + FSUploadChunkResponse fs_upload_chunk_response = 110; + FSCancelTransferResponse fs_cancel_transfer_response = 120; } }