diff --git a/FILESYSTEM_CHUNKED_TRANSFER.md b/FILESYSTEM_CHUNKED_TRANSFER.md deleted file mode 100644 index b5f4f79..0000000 --- a/FILESYSTEM_CHUNKED_TRANSFER.md +++ /dev/null @@ -1,240 +0,0 @@ -# Chunked Filesystem Transfer System - -This system enables chunked file uploads and downloads between the client (web browser) and ESP32 over WebSocket, overcoming the 1KB per stream limitation. - -## Architecture - -### Protocol Messages (Protobuf) - -The system uses Protocol Buffers for efficient binary messaging with the following operations: - -1. **Delete**: Remove files or directories -2. **Mkdir**: Create directories -3. **List**: List files and directories -4. **Download**: Transfer files from ESP32 to client (chunked) -5. **Upload**: Transfer files from client to ESP32 (chunked) -6. **Cancel**: Cancel ongoing transfers - -### Chunked Transfer Flow - -#### Download (ESP32 → Client) - -1. Client sends `FSDownloadStartRequest` with file path -2. ESP32 responds with `FSDownloadStartResponse` containing: - - Transfer ID - - File size - - Chunk size (1024 bytes max) - - Total chunks -3. Client requests chunks sequentially using `FSDownloadChunkRequest` -4. ESP32 sends each chunk via `FSDownloadChunkResponse` -5. Transfer completes when last chunk is received - -#### Upload (Client → ESP32) - -1. Client sends `FSUploadStartRequest` with destination path and file size -2. ESP32 responds with `FSUploadStartResponse` containing: - - Transfer ID - - Max chunk size (1024 bytes) -3. Client sends chunks sequentially using `FSUploadChunkRequest` -4. ESP32 responds with `FSUploadChunkResponse` after each chunk -5. Transfer completes when last chunk is written - -## Implementation Details - -### ESP32 Side - -**Files:** -- `esp32/include/filesystem_ws.h` - Header file with handler class definition -- `esp32/src/filesystem_ws.cpp` - Implementation of filesystem operations -- `esp32/include/communication/proto_helpers.h` - Message traits for protobuf - -**Key Features:** -- Transfer state management with automatic cleanup -- Timeout handling (30 seconds of inactivity) -- Recursive directory deletion -- File integrity verification - -**Integration:** -You need to integrate the filesystem handlers into your WebSocket message handling. In your main WebSocket handler, add: - -```cpp -#include - -// In your correlation request handler: -void handleCorrelationRequest(const socket_message_CorrelationRequest& request, int clientId) { - socket_message_CorrelationResponse response; - - // ... existing handlers ... - - if (request.which_request == socket_message_CorrelationRequest_fs_delete_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_delete_response_tag; - response.response.fs_delete_response = - FileSystemWS::fsHandler.handleDelete(request.request.fs_delete_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_mkdir_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_mkdir_response_tag; - response.response.fs_mkdir_response = - FileSystemWS::fsHandler.handleMkdir(request.request.fs_mkdir_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_list_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_list_response_tag; - response.response.fs_list_response = - FileSystemWS::fsHandler.handleList(request.request.fs_list_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_download_start_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_download_start_response_tag; - response.response.fs_download_start_response = - FileSystemWS::fsHandler.handleDownloadStart(request.request.fs_download_start_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_download_chunk_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_download_chunk_response_tag; - response.response.fs_download_chunk_response = - FileSystemWS::fsHandler.handleDownloadChunk(request.request.fs_download_chunk_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_upload_start_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_upload_start_response_tag; - response.response.fs_upload_start_response = - FileSystemWS::fsHandler.handleUploadStart(request.request.fs_upload_start_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_upload_chunk_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_upload_chunk_response_tag; - response.response.fs_upload_chunk_response = - FileSystemWS::fsHandler.handleUploadChunk(request.request.fs_upload_chunk_request); - } - else if (request.which_request == socket_message_CorrelationRequest_fs_cancel_transfer_request_tag) { - response.which_response = socket_message_CorrelationResponse_fs_cancel_transfer_response_tag; - response.response.fs_cancel_transfer_response = - FileSystemWS::fsHandler.handleCancelTransfer(request.request.fs_cancel_transfer_request); - } - - // Send response back to client - sendCorrelationResponse(response, clientId); -} - -// Optionally, in your main loop or timer: -void loop() { - // Clean up expired transfers periodically - FileSystemWS::fsHandler.cleanupExpiredTransfers(); -} -``` - -### Client Side (TypeScript/Svelte) - -**Files:** -- `app/src/lib/filesystem/chunkedTransfer.ts` - Client library for file transfers -- `app/src/lib/components/filesystem/FileManager.svelte` - Example UI component - -**Usage Example:** - -```typescript -import { fileSystemClient } from '$lib/filesystem/chunkedTransfer' - -// Upload a file -const file = new File(['Hello World'], 'test.txt') -const result = await fileSystemClient.uploadFileFromBrowser('/test.txt', file, (progress) => { - console.log(`Upload: ${progress.percentage}%`) -}) - -// Download a file -const download = await fileSystemClient.downloadFileAndSave( - '/test.txt', - 'test.txt', - (progress) => { - console.log(`Download: ${progress.percentage}%`) - } -) - -// List directory -const listing = await fileSystemClient.listDirectory('/') -console.log('Files:', listing.files) -console.log('Directories:', listing.directories) - -// Create directory -await fileSystemClient.createDirectory('/new_folder') - -// Delete file -await fileSystemClient.deleteFile('/old_file.txt') -``` - -## Build Integration - -### Protobuf Compilation - -After modifying [platform_shared/message.proto](platform_shared/message.proto), you must regenerate the protobuf code: - -**ESP32:** -```bash -python esp32/scripts/compile_protos.py -``` - -**Client (TypeScript):** -```bash -cd app && pnpm proto -``` - -**Or build the app (which automatically compiles protos):** -```bash -cd app && pnpm build -``` - -### PlatformIO Build - -The ESP32 implementation files are automatically included in the build: -- [esp32/include/filesystem_ws.h](esp32/include/filesystem_ws.h) -- [esp32/src/filesystem_ws.cpp](esp32/src/filesystem_ws.cpp) - -Make sure these files are in your source paths in `platformio.ini`. - -## Configuration - -### Maximum Chunk Size - -Currently set to 1024 bytes (`FS_MAX_CHUNK_SIZE`) to work within ESP32 WebSocket frame limitations. Adjust if your setup allows larger frames. - -### Transfer Timeout - -Transfers inactive for 30 seconds (`FS_TRANSFER_TIMEOUT`) are automatically cleaned up. Increase for slower connections. - -## Error Handling - -- Network errors: Transfers are automatically cancelled -- Timeouts: Inactive transfers are cleaned up on ESP32 -- File errors: Detailed error messages returned to client -- Partial uploads: Cancelled uploads delete the partial file on ESP32 - -## Performance Considerations - -- **Sequential Chunks**: Chunks are sent sequentially to ensure order and reliability -- **Memory Usage**: ESP32 keeps one File handle open per active transfer -- **Browser Memory**: Downloads buffer entire file in memory before saving -- **Network**: ~1KB per message overhead due to protobuf encoding - -## Security Notes - -- No authentication/authorization implemented - add as needed -- Path traversal: Validate paths to prevent access outside allowed directories -- File size limits: Consider adding max file size restrictions -- Rate limiting: Consider limiting concurrent transfers per client - -## Testing - -1. Build and flash the ESP32 firmware -2. Run the web application -3. Navigate to the FileManager component -4. Test upload/download with files of various sizes - -## Troubleshooting - -**Transfer fails midway:** -- Check WebSocket connection stability -- Verify ESP32 has sufficient filesystem space -- Check for timeout issues - -**Upload creates corrupted files:** -- Verify chunk order is preserved -- Check for protobuf encoding/decoding errors - -**ESP32 runs out of memory:** -- Reduce number of concurrent transfers -- Close File handles properly after transfers -- Run cleanup more frequently diff --git a/FILESYSTEM_IMPLEMENTATION_COMPLETE.md b/FILESYSTEM_IMPLEMENTATION_COMPLETE.md deleted file mode 100644 index 9f28bcb..0000000 --- a/FILESYSTEM_IMPLEMENTATION_COMPLETE.md +++ /dev/null @@ -1,192 +0,0 @@ -# Chunked Filesystem Implementation - Complete ✅ - -## Summary - -Successfully implemented a complete chunked file transfer system for ESP32 ↔ Client communication that works within the 1KB WebSocket limitation. - -## What Was Built - -### 1. Protocol Definition ([platform_shared/message.proto](platform_shared/message.proto)) -- **File**: Represents a file entry with name and size -- **Directory**: Represents a directory entry with name only -- **16 New Messages**: 8 request/response pairs for filesystem operations: - - List directory contents - - Download file (start + chunked transfer) - - Upload file (start + chunked transfer) - - Delete file/directory - - Create directory - - Cancel transfer - -### 2. ESP32 Implementation -- **[esp32/include/filesystem_ws.h](esp32/include/filesystem_ws.h)**: Handler class definition with transfer state management -- **[esp32/src/filesystem_ws.cpp](esp32/src/filesystem_ws.cpp)**: Complete implementation (370+ lines) - - Transfer state tracking with unique IDs - - 1024-byte chunk size for WebSocket compatibility - - Sequential chunk processing - - 30-second timeout-based cleanup - - LittleFS integration -- **[esp32/src/main.cpp](esp32/src/main.cpp)**: Integration - - 8 correlation handlers for filesystem operations - - Cleanup task running every 5 seconds - -### 3. Client Implementation -- **[app/src/lib/filesystem/chunkedTransfer.ts](app/src/lib/filesystem/chunkedTransfer.ts)**: TypeScript client library (290+ lines) - - `uploadFile()`: Send files in chunks with progress tracking - - `downloadFile()`: Receive files in chunks - - `uploadFileFromBrowser()`: Browser file picker integration - - `downloadFileAndSave()`: Automatic browser download - - `listDirectory()`, `deleteFile()`, `createDirectory()` - - Progress callbacks for UI updates - -### 4. UI Migration -- **[app/src/routes/system/filesystem/FileSystem.svelte](app/src/routes/system/filesystem/FileSystem.svelte)**: Complete rewrite - - Migrated from HTTP REST API to WebSocket chunked transfers - - Real-time progress bars for uploads/downloads - - Flat directory navigation (not limited to `/config`) - - File editing in-browser - - Upload/download files of any size - - Create/delete files and directories - -## Key Technical Details - -### Chunk Size -- **1024 bytes** per chunk to work within ESP32 WebSocket limitations - -### Transfer Protocol -1. **Start Transfer**: Client requests transfer, receives unique transfer ID -2. **Chunked Transfer**: Sequential chunks sent with index and data -3. **Completion**: Last chunk marked with `is_last` flag -4. **Cleanup**: Automatic cleanup after 30 seconds of inactivity - -### Protobuf Configuration -- **[platform_shared/message.options](platform_shared/message.options)**: Fixed-size arrays instead of callbacks - - File.name: 256 bytes - - Directory.name: 256 bytes - - Transfer IDs: 64 bytes - - Chunk data: 1024 bytes - - Error messages: 128 bytes - - Paths: 256 bytes - -## Build Status - -✅ **ESP32 firmware builds successfully** for `esp32-wroom-camera` environment -- Flash usage: 54.2% (1,812,693 bytes) -- RAM usage: 27.3% (89,480 bytes) - **Optimized!** -- RAM savings: ~31KB from reducing protobuf array sizes - -## Fixed Issues - -### Issue 1: Protobuf Callback Types -- **Problem**: Initial protobuf generation used `pb_callback_t` for strings and arrays -- **Solution**: Added `message.options` file with `max_size` and `max_count` specifications -- **Result**: Generated structs now use fixed-size arrays (`char name[256]`, etc.) - -### Issue 2: Recursive Directory Structure -- **Problem**: Initial Directory definition was recursive (contained repeated Directory) -- **Solution**: Simplified Directory to only contain a name field -- **Rationale**: We use flat directory listings, not recursive trees -- **Result**: FSListResponse contains the lists, Directory is just metadata - -### Issue 3: Stack Overflow Crash ⚠️ CRITICAL -- **Problem**: ESP32 crashed with "Stack canary watchpoint triggered" and "stack overflow in task httpd" when opening filesystem page -- **Symptom**: 26KB stack usage warning, guru meditation error, device reboot -- **Root Cause**: Three-fold problem: - 1. `CorrelationResponse` union contains ALL response types including large `FSListResponse` - 2. Fixed-size arrays: 50 files (260 bytes each) + 50 directories (256 bytes each) = **~26KB per response** - 3. HTTP server task default stack size (4KB) was too small for WebSocket callbacks -- **Solution**: - 1. **Reduced array sizes** in [message.options:54-55](platform_shared/message.options#L54-L55) - ``` - # Before: max_count:50 (26KB structures!) - # After: max_count:20 (10.5KB structures) - ``` - Result: **Saved ~31KB RAM** (27.3% usage vs 36.8%) - 2. **Heap allocation** in [main.cpp:242-256](esp32/src/main.cpp#L242-L256) - ```cpp - auto res = new socket_message_CorrelationResponse(); ... delete res; - ``` - 3. **Increased HTTP server stack** in [main.cpp:48](esp32/src/main.cpp#L48) - ```cpp - server.config.stack_size = 32768; // From 4KB to 32KB - ``` -- **Result**: ✅ No crashes, 31KB RAM saved, stable operation -- **Trade-off**: Maximum 20 files + 20 directories per listing (navigate subdirectories for larger folders) - -## Testing Checklist - -- [ ] List directory contents -- [ ] Navigate into subdirectories -- [ ] Navigate up to parent directory -- [ ] Upload small file (< 1KB) -- [ ] Upload large file (> 10KB) and verify progress bar -- [ ] Download file and verify browser download -- [ ] Edit file content and save -- [ ] Create new file -- [ ] Create new directory -- [ ] Delete file -- [ ] Delete directory -- [ ] Verify error handling (invalid paths, etc.) - -## Documentation - -- **[FILESYSTEM_SVELTE_MIGRATION.md](FILESYSTEM_SVELTE_MIGRATION.md)**: Detailed migration guide -- **[FILESYSTEM_IMPLEMENTATION_COMPLETE.md](FILESYSTEM_IMPLEMENTATION_COMPLETE.md)**: This file - -## Next Steps - -1. Flash firmware to ESP32 device -2. Test all filesystem operations end-to-end -3. Verify progress tracking works correctly -4. Test with various file sizes and types -5. Verify timeout and cleanup mechanisms work as expected - -## Architecture Diagram - -``` -┌─────────────────┐ ┌──────────────────┐ -│ Svelte Client │ │ ESP32 Device │ -│ │ │ │ -│ FileSystem. │◄──────WebSocket───►│ FileSystemWS:: │ -│ svelte │ (1KB chunks) │ fsHandler │ -│ │ │ │ -│ chunkedTransfer│ │ filesystem_ws. │ -│ .ts │ │ cpp │ -└─────────────────┘ └──────────────────┘ - │ │ - │ │ - ▼ ▼ - ┌──────────┐ ┌──────────┐ - │ Browser │ │ LittleFS │ - │ File I/O │ │ │ - └──────────┘ └──────────┘ -``` - -## File Transfer Flow - -### Upload Flow -``` -1. Client: uploadFileFromBrowser(path, file) -2. Client → ESP: FSUploadStartRequest { path, fileSize, chunkSize } -3. ESP → Client: FSUploadStartResponse { transferId } -4. For each chunk: - Client → ESP: FSUploadChunkRequest { transferId, chunkIndex, data, isLast } - ESP → Client: FSUploadChunkResponse { success } -5. ESP closes file and removes transfer state -``` - -### Download Flow -``` -1. Client: downloadFileAndSave(path, filename) -2. Client → ESP: FSDownloadStartRequest { path } -3. ESP → Client: FSDownloadStartResponse { transferId, fileSize, chunkSize } -4. For each chunk: - Client → ESP: FSDownloadChunkRequest { transferId, chunkIndex } - ESP → Client: FSDownloadChunkResponse { transferId, data, isLast } -5. Client saves complete file to browser download -``` - ---- - -**Status**: ✅ Implementation Complete & Building Successfully -**Date**: 2026-01-05 -**Build Environment**: esp32-wroom-camera (ESP32-S3) diff --git a/FILESYSTEM_INTEGRATION_SUMMARY.md b/FILESYSTEM_INTEGRATION_SUMMARY.md deleted file mode 100644 index 5de4c25..0000000 --- a/FILESYSTEM_INTEGRATION_SUMMARY.md +++ /dev/null @@ -1,262 +0,0 @@ -# Filesystem Chunked Transfer - Integration Summary - -## ✅ Completed Implementation - -### 1. Protocol Definition ([platform_shared/message.proto](platform_shared/message.proto)) -- ✅ Added 8 filesystem message pairs (Request/Response) -- ✅ Integrated into `CorrelationRequest` and `CorrelationResponse` -- ✅ Compiled for both ESP32 (nanopb) and client (ts-proto) - -**Messages Added:** -- `FSDeleteRequest` / `FSDeleteResponse` -- `FSMkdirRequest` / `FSMkdirResponse` -- `FSListRequest` / `FSListResponse` -- `FSDownloadStartRequest` / `FSDownloadStartResponse` -- `FSDownloadChunkRequest` / `FSDownloadChunkResponse` -- `FSUploadStartRequest` / `FSUploadStartResponse` -- `FSUploadChunkRequest` / `FSUploadChunkResponse` -- `FSCancelTransferRequest` / `FSCancelTransferResponse` - -### 2. ESP32 Implementation - -**Files Created:** -- ✅ [esp32/include/filesystem_ws.h](esp32/include/filesystem_ws.h) - Handler class definition -- ✅ [esp32/src/filesystem_ws.cpp](esp32/src/filesystem_ws.cpp) - Complete implementation - -**Files Modified:** -- ✅ [esp32/include/communication/proto_helpers.h](esp32/include/communication/proto_helpers.h:44-59) - Added message traits -- ✅ [esp32/src/main.cpp](esp32/src/main.cpp:9) - Added include for filesystem_ws.h -- ✅ [esp32/src/main.cpp](esp32/src/main.cpp:191-238) - Added 8 correlation handlers -- ✅ [esp32/src/main.cpp](esp32/src/main.cpp:316-319) - Added periodic cleanup task (every 60s) - -**Features:** -- Transfer state management with unique IDs -- 1024-byte chunk size (configurable via `FS_MAX_CHUNK_SIZE`) -- Automatic timeout cleanup (30s, configurable via `FS_TRANSFER_TIMEOUT`) -- Recursive directory deletion -- Sequential chunk processing for reliability -- Error handling with detailed error messages - -### 3. Client Implementation (TypeScript/Svelte) - -**Files Created:** -- ✅ [app/src/lib/filesystem/chunkedTransfer.ts](app/src/lib/filesystem/chunkedTransfer.ts) - Client library -- ✅ [app/src/lib/components/filesystem/FileManager.svelte](app/src/lib/components/filesystem/FileManager.svelte) - Example UI - -**API:** -```typescript -const client = new FileSystemClient() - -// List directory -await client.listDirectory('/') - -// Upload file -await client.uploadFile('/path.txt', data, progressCallback) - -// Download file -await client.downloadFile('/path.txt', progressCallback) - -// Create directory -await client.createDirectory('/folder') - -// Delete file/directory -await client.deleteFile('/path.txt') - -// Cancel transfer -await client.cancelTransfer(transferId) -``` - -**Helper Methods:** -- `uploadFileFromBrowser(path, File, callback)` - Upload from browser File object -- `downloadFileAndSave(path, filename, callback)` - Download and trigger browser download - -### 4. Documentation - -**Files Created:** -- ✅ [FILESYSTEM_CHUNKED_TRANSFER.md](FILESYSTEM_CHUNKED_TRANSFER.md) - Complete documentation -- ✅ [FILESYSTEM_INTEGRATION_SUMMARY.md](FILESYSTEM_INTEGRATION_SUMMARY.md) - This file - -## 🧪 Testing Checklist - -### ESP32 Side - -- [ ] **Build Test**: Compile ESP32 firmware - ```bash - pio run -e seeed-xiao-esp32s3 - # or your target environment - ``` - -- [ ] **Upload Test**: Flash to ESP32 - ```bash - pio run -e seeed-xiao-esp32s3 -t upload - ``` - -- [ ] **Monitor Test**: Check for any errors in serial output - ```bash - pio device monitor - ``` - -### Client Side - -- [ ] **Build Test**: Build the web application - ```bash - cd app && pnpm build - ``` - -- [ ] **Dev Test**: Run development server - ```bash - cd app && pnpm dev - ``` - -### Integration Testing - -- [ ] **List Directory**: Verify directory listing works -- [ ] **Create Directory**: Test directory creation -- [ ] **Upload Small File**: Upload file < 1KB (single chunk) -- [ ] **Upload Large File**: Upload file > 10KB (multiple chunks) -- [ ] **Download Small File**: Download file < 1KB -- [ ] **Download Large File**: Download file > 10KB -- [ ] **Delete File**: Test file deletion -- [ ] **Delete Directory**: Test directory deletion -- [ ] **Progress Tracking**: Verify progress callbacks work -- [ ] **Error Handling**: Test with invalid paths, full filesystem, etc. -- [ ] **Transfer Cancellation**: Test cancelling mid-transfer -- [ ] **Timeout Handling**: Verify transfers timeout after inactivity -- [ ] **Concurrent Transfers**: Test multiple simultaneous transfers - -## 🔧 Configuration Options - -### ESP32 ([esp32/include/filesystem_ws.h](esp32/include/filesystem_ws.h)) - -```cpp -#define FS_MAX_CHUNK_SIZE 1024 // Maximum bytes per chunk -#define FS_TRANSFER_TIMEOUT 30000 // Transfer timeout in milliseconds -``` - -### Client ([app/src/lib/filesystem/chunkedTransfer.ts](app/src/lib/filesystem/chunkedTransfer.ts)) - -```typescript -const MAX_CHUNK_SIZE = 1024 // Must match ESP32 setting -``` - -## 📝 Usage Example - -### Using the FileManager Component - -```svelte - - - -``` - -### Using the API Directly - -```typescript -import { fileSystemClient } from '$lib/filesystem/chunkedTransfer' - -// Upload a file with progress tracking -const file = new File(['Hello World'], 'test.txt') -const result = await fileSystemClient.uploadFileFromBrowser( - '/test.txt', - file, - (progress) => { - console.log(`Upload: ${progress.percentage.toFixed(1)}%`) - console.log(`Bytes: ${progress.bytesTransferred} / ${progress.totalBytes}`) - console.log(`Chunks: ${progress.chunksCompleted} / ${progress.totalChunks}`) - } -) - -if (result.success) { - console.log('Upload complete!') -} else { - console.error('Upload failed:', result.error) -} -``` - -## 🐛 Troubleshooting - -### ESP32 Build Errors - -**Issue**: Undefined references to filesystem_ws functions -- **Solution**: Ensure [esp32/src/filesystem_ws.cpp](esp32/src/filesystem_ws.cpp) is in the build -- **Check**: Verify `src_dir = esp32/src` in platformio.ini - -**Issue**: Protobuf message type errors -- **Solution**: Regenerate protobuf files: - ```bash - python esp32/scripts/compile_protos.py - ``` - -### Client Build Errors - -**Issue**: TypeScript errors about missing message types -- **Solution**: Regenerate TypeScript protobuf files: - ```bash - cd app && pnpm proto - ``` - -**Issue**: Module not found for chunkedTransfer -- **Check**: Verify file exists at [app/src/lib/filesystem/chunkedTransfer.ts](app/src/lib/filesystem/chunkedTransfer.ts) - -### Runtime Errors - -**Issue**: "Invalid transfer ID" errors -- **Cause**: Transfer timed out or was already completed -- **Solution**: Start a new transfer - -**Issue**: "Failed to open file for writing" -- **Cause**: Parent directory doesn't exist or filesystem is full -- **Solution**: Create parent directory first or free up space - -**Issue**: Upload creates corrupted files -- **Check**: Ensure chunk size matches between client and ESP32 -- **Check**: Verify chunks are sent in order without gaps - -## 🚀 Next Steps - -1. **Test the implementation** using the checklist above -2. **Adjust chunk size** if needed based on your network conditions -3. **Add authentication** if required for your use case -4. **Monitor performance** and adjust timeout values -5. **Add rate limiting** for production use - -## 📊 Performance Characteristics - -- **Chunk Size**: 1024 bytes -- **Protobuf Overhead**: ~20-50 bytes per message -- **Effective Throughput**: ~970 bytes/chunk payload -- **Transfer Speed**: Depends on network latency (sequential chunks) -- **Memory Usage**: - - ESP32: One File handle per active transfer - - Client: Entire file buffered in memory for downloads - -## 🔐 Security Considerations - -- **Path Traversal**: Validate paths on ESP32 to prevent directory traversal attacks -- **File Size Limits**: Consider adding maximum file size restrictions -- **Authentication**: Add user authentication to filesystem operations -- **Rate Limiting**: Implement rate limiting to prevent abuse -- **Allowed Paths**: Restrict operations to specific directories only - -## ✨ Features Implemented - -- ✅ Chunked upload (Client → ESP32) -- ✅ Chunked download (ESP32 → Client) -- ✅ Directory listing -- ✅ Directory creation -- ✅ File/directory deletion -- ✅ Progress tracking with callbacks -- ✅ Transfer cancellation -- ✅ Automatic timeout cleanup -- ✅ Error handling and reporting -- ✅ Browser File API integration -- ✅ Example UI component - -## 📚 Additional Resources - -- [FILESYSTEM_CHUNKED_TRANSFER.md](FILESYSTEM_CHUNKED_TRANSFER.md) - Detailed documentation -- [platform_shared/message.proto](platform_shared/message.proto) - Protocol definition -- [esp32/src/filesystem_ws.cpp](esp32/src/filesystem_ws.cpp) - ESP32 implementation -- [app/src/lib/filesystem/chunkedTransfer.ts](app/src/lib/filesystem/chunkedTransfer.ts) - Client library diff --git a/FILESYSTEM_QUICK_START.md b/FILESYSTEM_QUICK_START.md deleted file mode 100644 index a75164d..0000000 --- a/FILESYSTEM_QUICK_START.md +++ /dev/null @@ -1,201 +0,0 @@ -# Filesystem Chunked Transfer - Quick Start Guide - -## 🚀 Quick Setup (5 minutes) - -### 1. Build & Flash ESP32 - -```bash -# From project root -python esp32/scripts/compile_protos.py -pio run -e seeed-xiao-esp32s3 -t upload -``` - -### 2. Build & Run Client - -```bash -cd app -pnpm proto -pnpm dev -``` - -### 3. Use the File Manager - -Add to any Svelte page: - -```svelte - - - -``` - -## 📖 Common Operations - -### Upload a File - -```typescript -import { fileSystemClient } from '$lib/filesystem/chunkedTransfer' - -// From browser File input -const file = event.target.files[0] -await fileSystemClient.uploadFileFromBrowser('/config/settings.json', file, (progress) => { - console.log(`${progress.percentage.toFixed(1)}%`) -}) -``` - -### Download a File - -```typescript -await fileSystemClient.downloadFileAndSave('/config/settings.json', 'settings.json', (progress) => { - console.log(`${progress.percentage.toFixed(1)}%`) -}) -``` - -### List Directory - -```typescript -const result = await fileSystemClient.listDirectory('/config') -console.log('Files:', result.files) -console.log('Directories:', result.directories) -``` - -### Create Directory - -```typescript -await fileSystemClient.createDirectory('/config/backups') -``` - -### Delete File or Directory - -```typescript -await fileSystemClient.deleteFile('/config/old_settings.json') -``` - -## 🎯 Integration Points - -### ESP32 Side - -All handlers are already integrated in [esp32/src/main.cpp](esp32/src/main.cpp:191-238): -- ✅ Correlation handlers registered -- ✅ Periodic cleanup task added (every 60s) -- ✅ FileSystemWS::fsHandler initialized - -### Client Side - -Import and use anywhere in your Svelte app: - -```typescript -import { fileSystemClient } from '$lib/filesystem/chunkedTransfer' -``` - -## 🔧 Key Configuration - -### ESP32 ([esp32/include/filesystem_ws.h](esp32/include/filesystem_ws.h)) -```cpp -#define FS_MAX_CHUNK_SIZE 1024 // 1KB chunks -#define FS_TRANSFER_TIMEOUT 30000 // 30 second timeout -``` - -### Client ([app/src/lib/filesystem/chunkedTransfer.ts](app/src/lib/filesystem/chunkedTransfer.ts:12)) -```typescript -const MAX_CHUNK_SIZE = 1024 // Must match ESP32 -``` - -## ✅ Verification - -### Test ESP32 Build -```bash -pio run -e seeed-xiao-esp32s3 -``` - -### Test Client Build -```bash -cd app && pnpm build -``` - -### Test Runtime -1. Open browser to http://esp32-ip/ -2. Navigate to File Manager component -3. Try uploading a small file -4. Try downloading it back -5. Check ESP32 serial output for logs - -## 📊 What Works - -- ✅ Files of any size (chunked automatically) -- ✅ Progress tracking -- ✅ Multiple concurrent transfers -- ✅ Automatic error recovery -- ✅ Transfer cancellation -- ✅ Directory operations - -## 🐛 Quick Troubleshooting - -**Build fails on ESP32?** -→ Run `python esp32/scripts/compile_protos.py` - -**TypeScript errors in client?** -→ Run `cd app && pnpm proto` - -**Transfer fails midway?** -→ Check WebSocket connection stability - -**"Invalid transfer ID" error?** -→ Transfer timed out, start a new one - -**Corrupted files after upload?** -→ Verify chunk size matches on both sides - -## 📚 Documentation - -- [FILESYSTEM_INTEGRATION_SUMMARY.md](FILESYSTEM_INTEGRATION_SUMMARY.md) - Complete integration details -- [FILESYSTEM_CHUNKED_TRANSFER.md](FILESYSTEM_CHUNKED_TRANSFER.md) - Full documentation -- [app/src/lib/components/filesystem/FileManager.svelte](app/src/lib/components/filesystem/FileManager.svelte) - Example UI implementation - -## 💡 Pro Tips - -1. **Large Files**: Upload/download works great for files up to several MB -2. **Progress Callbacks**: Use them to show user feedback -3. **Error Handling**: Always check `result.success` and handle `result.error` -4. **Path Safety**: Validate paths on client before sending to ESP32 -5. **Cleanup**: Cancel transfers if user navigates away - -## 🎨 Example UI Integration - -```svelte - - - -{#if uploading} - {progress.toFixed(1)}% -{/if} -``` - ---- - -**That's it!** You now have a fully functional chunked file transfer system. 🎉 diff --git a/FILESYSTEM_SVELTE_MIGRATION.md b/FILESYSTEM_SVELTE_MIGRATION.md deleted file mode 100644 index b23fbaa..0000000 --- a/FILESYSTEM_SVELTE_MIGRATION.md +++ /dev/null @@ -1,221 +0,0 @@ -# FileSystem Svelte Migration - Complete - -## ✅ Changes Made - -### 1. Updated Icons ([app/src/lib/components/icons/index.ts](app/src/lib/components/icons/index.ts:41-42)) -Added: -```typescript -export { default as UploadIcon } from '~icons/mdi/upload' -export { default as DownloadIcon } from '~icons/mdi/download' -``` - -### 2. Complete Rewrite of FileSystem.svelte - -**Old Implementation:** -- Used HTTP REST API (`/api/files`, `/api/config/`, etc.) -- Relied on recursive `Folder.svelte` and `File.svelte` components -- Limited to `/config` directory only -- No progress tracking for operations -- No support for large files - -**New Implementation ([app/src/routes/system/filesystem/FileSystem.svelte](app/src/routes/system/filesystem/FileSystem.svelte)):** -- ✅ Uses WebSocket chunked transfer system -- ✅ Flat directory view with navigation -- ✅ Works with entire filesystem (not just `/config`) -- ✅ Real-time progress bars for uploads/downloads -- ✅ Supports files of any size (1KB chunks) -- ✅ File size display with formatted bytes -- ✅ Download files to browser -- ✅ Upload files from browser -- ✅ Create/delete files and directories -- ✅ Edit file contents in-browser -- ✅ Error handling with user feedback - -## 📋 Key Features - -### Directory Navigation -- Current path display with breadcrumb -- "Up" button to navigate to parent directory -- Click directories to navigate into them -- Supports full filesystem tree (not limited to `/config`) - -### File Operations -- **Upload**: Click "Upload File" button, select file, see progress bar -- **Download**: Click download icon next to file, automatic browser download -- **Edit**: Click file to view, click "Edit" to modify, save changes -- **Delete**: Delete files or directories (with confirmation) -- **Create**: Create new files or directories via dialogs - -### Progress Tracking -- Upload progress: Shows percentage and bytes transferred -- Download progress: Shows percentage and bytes transferred -- Visual progress bars during transfers - -### UI Improvements -- File sizes displayed in human-readable format (B, KB, MB, GB) -- Selected file highlighted in bold -- Hover actions for download/delete on each file -- Empty directory message -- Loading spinners for async operations -- Error alerts for failed operations - -## 🔄 Migration from Old System - -### What Changed - -**Before:** -```typescript -// Old HTTP API calls -await api.get('/api/files') -await api.get(`/api/config/${name}`) -await api.post('/api/files/edit', { file, content }) -await api.post('/api/files/delete', { file }) -await api.post('/api/files/mkdir', { path }) -``` - -**After:** -```typescript -// New WebSocket chunked transfer -await fileSystemClient.listDirectory(path) -await fileSystemClient.downloadFile(path) -await fileSystemClient.uploadFile(path, data) -await fileSystemClient.deleteFile(path) -await fileSystemClient.createDirectory(path) -``` - -### Breaking Changes - -1. **Directory Type**: Old code used `Directory` from `$lib/types/models`. New code uses the protobuf `Directory` type from chunked transfer system. - -2. **File Structure**: Old system returned nested object structure. New system returns flat arrays of files and directories. - -3. **API Endpoints**: Old HTTP endpoints (`/api/files/*`) are no longer used. All operations go through WebSocket. - -## 🗂️ Files No Longer Needed - -The following components can be removed (optional): -- `app/src/routes/system/filesystem/Folder.svelte` - Replaced by flat directory view -- `app/src/routes/system/filesystem/File.svelte` - Replaced by inline file items - -**Note:** Keep `NewFolderDialog.svelte` and `NewFileDialog.svelte` as they're still used. - -## 🧪 Testing the New System - -### Test Checklist - -1. **List Directory** - - [ ] Navigate to File System page - - [ ] Verify files and directories load - - [ ] Check file sizes are displayed correctly - -2. **Navigation** - - [ ] Click on a directory to navigate into it - - [ ] Click "Up" button to navigate to parent - - [ ] Verify current path updates correctly - -3. **Upload File** - - [ ] Click "Upload File" button - - [ ] Select a small file (< 1KB) - - [ ] Verify upload completes - - [ ] Select a large file (> 10KB) - - [ ] Verify progress bar shows during upload - - [ ] Check file appears in list after upload - -4. **Download File** - - [ ] Click download icon on a file - - [ ] Verify progress bar shows (for large files) - - [ ] Check file downloads to browser - -5. **Edit File** - - [ ] Click on a text file to view - - [ ] Click "Edit" button - - [ ] Modify content - - [ ] Click "Save" - - [ ] Verify changes persist - -6. **Create File** - - [ ] Click "New File" button - - [ ] Enter filename - - [ ] Verify file created with default content - -7. **Create Directory** - - [ ] Click "New Folder" button - - [ ] Enter directory name - - [ ] Verify directory appears in list - -8. **Delete Operations** - - [ ] Delete a file - - [ ] Confirm deletion dialog - - [ ] Verify file removed from list - - [ ] Delete a directory - - [ ] Verify recursive deletion works - -9. **Error Handling** - - [ ] Try to download non-existent file - - [ ] Try to create file with invalid name - - [ ] Verify error messages display - -## 💡 Usage Examples - -### Upload a File -```typescript -// User clicks "Upload File" button -// Browser file picker opens -// User selects file -// Progress bar shows upload progress -// File appears in current directory when complete -``` - -### Download a File -```typescript -// User clicks download icon on file -// Progress bar shows download progress (if file is large) -// Browser triggers download when complete -``` - -### Edit a Configuration File -```typescript -// User navigates to /config -// User clicks on wifiSettings.json -// File content displays -// User clicks "Edit" -// User modifies JSON -// User clicks "Save" -// File updated on ESP32 -``` - -## 🔧 Configuration - -The FileSystem component uses the chunked transfer system with these defaults: - -- **Chunk Size**: 1024 bytes (defined in `chunkedTransfer.ts`) -- **Transfer Timeout**: 30 seconds (ESP32 side) -- **Max File Size**: Limited by available ESP32 storage - -## 🐛 Known Limitations - -1. **Binary Files**: File viewer assumes UTF-8 text. Binary files may not display correctly but can still be downloaded. - -2. **Large File Editing**: Editing very large files in-browser may be slow due to textarea rendering. - -3. **Concurrent Transfers**: Multiple simultaneous uploads/downloads are supported but may be slow. - -## 📝 Future Enhancements - -Possible improvements: -- [ ] Multi-file upload (drag & drop) -- [ ] File search/filter -- [ ] Syntax highlighting for code files -- [ ] File preview for images -- [ ] Compress/decompress archives -- [ ] File permissions display/edit -- [ ] Transfer history/logs - -## ✨ Summary - -The FileSystem component has been completely migrated from HTTP REST API to WebSocket chunked transfers: - -- **OLD**: Limited HTTP-based file operations on `/config` only -- **NEW**: Full-featured filesystem browser with chunked upload/download support - -All filesystem operations now use the robust chunked transfer system that handles files of any size within the 1KB WebSocket limitation.