Claude: File chunking system

This commit is contained in:
Niklas Jensen
2026-01-05 20:33:16 +01:00
committed by Rune Harlyk
parent 725d62747d
commit f440fa3973
8 changed files with 1472 additions and 19 deletions
+211
View File
@@ -0,0 +1,211 @@
# Chunked Filesystem Transfer System
This system enables chunked file uploads and downloads between the client (web browser) and ESP32 over WebSocket, overcoming the 1KB per stream limitation.
## Architecture
### Protocol Messages (Protobuf)
The system uses Protocol Buffers for efficient binary messaging with the following operations:
1. **Delete**: Remove files or directories
2. **Mkdir**: Create directories
3. **List**: List files and directories
4. **Download**: Transfer files from ESP32 to client (chunked)
5. **Upload**: Transfer files from client to ESP32 (chunked)
6. **Cancel**: Cancel ongoing transfers
### Chunked Transfer Flow
#### Download (ESP32 → Client)
1. Client sends `FSDownloadStartRequest` with file path
2. ESP32 responds with `FSDownloadStartResponse` containing:
- Transfer ID
- File size
- Chunk size (1024 bytes max)
- Total chunks
3. Client requests chunks sequentially using `FSDownloadChunkRequest`
4. ESP32 sends each chunk via `FSDownloadChunkResponse`
5. Transfer completes when last chunk is received
#### Upload (Client → ESP32)
1. Client sends `FSUploadStartRequest` with destination path and file size
2. ESP32 responds with `FSUploadStartResponse` containing:
- Transfer ID
- Max chunk size (1024 bytes)
3. Client sends chunks sequentially using `FSUploadChunkRequest`
4. ESP32 responds with `FSUploadChunkResponse` after each chunk
5. Transfer completes when last chunk is written
## Implementation Details
### ESP32 Side
**Files:**
- `esp32/include/filesystem_ws.h` - Header file with handler class definition
- `esp32/src/filesystem_ws.cpp` - Implementation of filesystem operations
- `esp32/include/communication/proto_helpers.h` - Message traits for protobuf
**Key Features:**
- Transfer state management with automatic cleanup
- Timeout handling (30 seconds of inactivity)
- Recursive directory deletion
- File integrity verification
**Integration:**
You need to integrate the filesystem handlers into your WebSocket message handling. In your main WebSocket handler, add:
```cpp
#include <filesystem_ws.h>
// In your correlation request handler:
void handleCorrelationRequest(const socket_message_CorrelationRequest& request, int clientId) {
socket_message_CorrelationResponse response;
// ... existing handlers ...
if (request.which_request == socket_message_CorrelationRequest_fs_delete_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_delete_response_tag;
response.response.fs_delete_response =
FileSystemWS::fsHandler.handleDelete(request.request.fs_delete_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_mkdir_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_mkdir_response_tag;
response.response.fs_mkdir_response =
FileSystemWS::fsHandler.handleMkdir(request.request.fs_mkdir_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_list_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_list_response_tag;
response.response.fs_list_response =
FileSystemWS::fsHandler.handleList(request.request.fs_list_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_download_start_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_download_start_response_tag;
response.response.fs_download_start_response =
FileSystemWS::fsHandler.handleDownloadStart(request.request.fs_download_start_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_download_chunk_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_download_chunk_response_tag;
response.response.fs_download_chunk_response =
FileSystemWS::fsHandler.handleDownloadChunk(request.request.fs_download_chunk_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_upload_start_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_upload_start_response_tag;
response.response.fs_upload_start_response =
FileSystemWS::fsHandler.handleUploadStart(request.request.fs_upload_start_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_upload_chunk_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_upload_chunk_response_tag;
response.response.fs_upload_chunk_response =
FileSystemWS::fsHandler.handleUploadChunk(request.request.fs_upload_chunk_request);
}
else if (request.which_request == socket_message_CorrelationRequest_fs_cancel_transfer_request_tag) {
response.which_response = socket_message_CorrelationResponse_fs_cancel_transfer_response_tag;
response.response.fs_cancel_transfer_response =
FileSystemWS::fsHandler.handleCancelTransfer(request.request.fs_cancel_transfer_request);
}
// Send response back to client
sendCorrelationResponse(response, clientId);
}
// Optionally, in your main loop or timer:
void loop() {
// Clean up expired transfers periodically
FileSystemWS::fsHandler.cleanupExpiredTransfers();
}
```
### Client Side (TypeScript/Svelte)
**Files:**
- `app/src/lib/filesystem/chunkedTransfer.ts` - Client library for file transfers
- `app/src/lib/components/filesystem/FileManager.svelte` - Example UI component
**Usage Example:**
```typescript
import { fileSystemClient } from '$lib/filesystem/chunkedTransfer'
// Upload a file
const file = new File(['Hello World'], 'test.txt')
const result = await fileSystemClient.uploadFileFromBrowser('/test.txt', file, (progress) => {
console.log(`Upload: ${progress.percentage}%`)
})
// Download a file
const download = await fileSystemClient.downloadFileAndSave(
'/test.txt',
'test.txt',
(progress) => {
console.log(`Download: ${progress.percentage}%`)
}
)
// List directory
const listing = await fileSystemClient.listDirectory('/')
console.log('Files:', listing.files)
console.log('Directories:', listing.directories)
// Create directory
await fileSystemClient.createDirectory('/new_folder')
// Delete file
await fileSystemClient.deleteFile('/old_file.txt')
```
## Configuration
### Maximum Chunk Size
Currently set to 1024 bytes (`FS_MAX_CHUNK_SIZE`) to work within ESP32 WebSocket frame limitations. Adjust if your setup allows larger frames.
### Transfer Timeout
Transfers inactive for 30 seconds (`FS_TRANSFER_TIMEOUT`) are automatically cleaned up. Increase for slower connections.
## Error Handling
- Network errors: Transfers are automatically cancelled
- Timeouts: Inactive transfers are cleaned up on ESP32
- File errors: Detailed error messages returned to client
- Partial uploads: Cancelled uploads delete the partial file on ESP32
## Performance Considerations
- **Sequential Chunks**: Chunks are sent sequentially to ensure order and reliability
- **Memory Usage**: ESP32 keeps one File handle open per active transfer
- **Browser Memory**: Downloads buffer entire file in memory before saving
- **Network**: ~1KB per message overhead due to protobuf encoding
## Security Notes
- No authentication/authorization implemented - add as needed
- Path traversal: Validate paths to prevent access outside allowed directories
- File size limits: Consider adding max file size restrictions
- Rate limiting: Consider limiting concurrent transfers per client
## Testing
1. Build and flash the ESP32 firmware
2. Run the web application
3. Navigate to the FileManager component
4. Test upload/download with files of various sizes
## Troubleshooting
**Transfer fails midway:**
- Check WebSocket connection stability
- Verify ESP32 has sufficient filesystem space
- Check for timeout issues
**Upload creates corrupted files:**
- Verify chunk order is preserved
- Check for protobuf encoding/decoding errors
**ESP32 runs out of memory:**
- Reduce number of concurrent transfers
- Close File handles properly after transfers
- Run cleanup more frequently
+6 -6
View File
@@ -1,8 +1,8 @@
declare module 'app-env' {
interface ENV {
VITE_USE_HOST_NAME: boolean
}
declare module "app-env" {
interface ENV {
VITE_USE_HOST_NAME: boolean;
}
const appEnv: ENV
export default appEnv
const appEnv: ENV;
export default appEnv;
}
@@ -0,0 +1,374 @@
<script lang="ts">
import { fileSystemClient, type TransferProgress } from '$lib/filesystem/chunkedTransfer'
import { onMount } from 'svelte'
let currentPath = '/'
let files: Array<{ name: string; size: number }> = []
let directories: Array<{ name: string }> = []
let loading = false
let error = ''
let uploadProgress: TransferProgress | null = null
let downloadProgress: TransferProgress | null = null
async function loadDirectory() {
loading = true
error = ''
try {
const result = await fileSystemClient.listDirectory(currentPath)
if (result.success) {
files = result.files
directories = result.directories
} else {
error = result.error || 'Failed to load directory'
}
} catch (e) {
error = e instanceof Error ? e.message : 'Unknown error'
} finally {
loading = false
}
}
async function navigateTo(path: string) {
currentPath = path
await loadDirectory()
}
async function handleFileUpload(event: Event) {
const input = event.target as HTMLInputElement
const file = input.files?.[0]
if (!file) return
uploadProgress = null
error = ''
const destinationPath = currentPath.endsWith('/')
? currentPath + file.name
: currentPath + '/' + file.name
try {
const result = await fileSystemClient.uploadFileFromBrowser(
destinationPath,
file,
(progress) => {
uploadProgress = progress
}
)
if (result.success) {
await loadDirectory()
} else {
error = result.error || 'Upload failed'
}
} catch (e) {
error = e instanceof Error ? e.message : 'Upload error'
} finally {
uploadProgress = null
input.value = ''
}
}
async function handleDownload(filename: string) {
downloadProgress = null
error = ''
const filePath = currentPath.endsWith('/')
? currentPath + filename
: currentPath + '/' + filename
try {
const result = await fileSystemClient.downloadFileAndSave(filePath, filename, (progress) => {
downloadProgress = progress
})
if (!result.success) {
error = result.error || 'Download failed'
}
} catch (e) {
error = e instanceof Error ? e.message : 'Download error'
} finally {
downloadProgress = null
}
}
async function handleDelete(name: string, isDirectory: boolean) {
if (!confirm(`Delete ${isDirectory ? 'directory' : 'file'} "${name}"?`)) return
error = ''
const path = currentPath.endsWith('/') ? currentPath + name : currentPath + '/' + name
try {
const result = await fileSystemClient.deleteFile(path)
if (result.success) {
await loadDirectory()
} else {
error = result.error || 'Delete failed'
}
} catch (e) {
error = e instanceof Error ? e.message : 'Delete error'
}
}
async function handleCreateDirectory() {
const name = prompt('Enter directory name:')
if (!name) return
error = ''
const path = currentPath.endsWith('/') ? currentPath + name : currentPath + '/' + name
try {
const result = await fileSystemClient.createDirectory(path)
if (result.success) {
await loadDirectory()
} else {
error = result.error || 'Failed to create directory'
}
} catch (e) {
error = e instanceof Error ? e.message : 'Error creating directory'
}
}
function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]
}
onMount(() => {
loadDirectory()
})
</script>
<div class="file-manager">
<div class="toolbar">
<h2>File Manager</h2>
<div class="path">Current: {currentPath}</div>
<div class="actions">
<button on:click={handleCreateDirectory}>New Folder</button>
<label class="upload-btn">
Upload File
<input type="file" on:change={handleFileUpload} style="display: none;" />
</label>
<button on:click={loadDirectory}>Refresh</button>
</div>
</div>
{#if error}
<div class="error">{error}</div>
{/if}
{#if uploadProgress}
<div class="progress">
<div class="progress-label">
Uploading: {uploadProgress.percentage.toFixed(1)}% ({formatBytes(
uploadProgress.bytesTransferred
)} / {formatBytes(uploadProgress.totalBytes)})
</div>
<div class="progress-bar">
<div class="progress-fill" style="width: {uploadProgress.percentage}%"></div>
</div>
</div>
{/if}
{#if downloadProgress}
<div class="progress">
<div class="progress-label">
Downloading: {downloadProgress.percentage.toFixed(1)}% ({formatBytes(
downloadProgress.bytesTransferred
)} / {formatBytes(downloadProgress.totalBytes)})
</div>
<div class="progress-bar">
<div class="progress-fill" style="width: {downloadProgress.percentage}%"></div>
</div>
</div>
{/if}
<div class="file-list">
{#if loading}
<div class="loading">Loading...</div>
{:else}
{#if currentPath !== '/'}
<div class="file-item directory" on:click={() => navigateTo('/')}>
<span class="icon">📁</span>
<span class="name">..</span>
</div>
{/if}
{#each directories as dir}
<div class="file-item directory">
<span class="icon">📁</span>
<span class="name" on:click={() => navigateTo(currentPath + '/' + dir.name)}
>{dir.name}</span
>
<button class="delete-btn" on:click={() => handleDelete(dir.name, true)}>Delete</button>
</div>
{/each}
{#each files as file}
<div class="file-item">
<span class="icon">📄</span>
<span class="name">{file.name}</span>
<span class="size">{formatBytes(file.size)}</span>
<button class="download-btn" on:click={() => handleDownload(file.name)}>Download</button>
<button class="delete-btn" on:click={() => handleDelete(file.name, false)}>Delete</button>
</div>
{/each}
{#if files.length === 0 && directories.length === 0}
<div class="empty">Directory is empty</div>
{/if}
{/if}
</div>
</div>
<style>
.file-manager {
max-width: 800px;
margin: 2rem auto;
padding: 1rem;
border: 1px solid #ddd;
border-radius: 8px;
background: white;
}
.toolbar {
margin-bottom: 1rem;
}
.toolbar h2 {
margin: 0 0 0.5rem 0;
}
.path {
font-family: monospace;
background: #f5f5f5;
padding: 0.5rem;
border-radius: 4px;
margin-bottom: 0.5rem;
}
.actions {
display: flex;
gap: 0.5rem;
}
.actions button,
.upload-btn {
padding: 0.5rem 1rem;
background: #007bff;
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
}
.actions button:hover,
.upload-btn:hover {
background: #0056b3;
}
.error {
background: #f8d7da;
color: #721c24;
padding: 0.75rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.progress {
margin-bottom: 1rem;
}
.progress-label {
margin-bottom: 0.5rem;
font-size: 0.9rem;
}
.progress-bar {
height: 20px;
background: #e9ecef;
border-radius: 4px;
overflow: hidden;
}
.progress-fill {
height: 100%;
background: #28a745;
transition: width 0.3s ease;
}
.file-list {
border: 1px solid #ddd;
border-radius: 4px;
min-height: 200px;
}
.file-item {
display: flex;
align-items: center;
padding: 0.75rem;
border-bottom: 1px solid #eee;
gap: 0.5rem;
}
.file-item:last-child {
border-bottom: none;
}
.file-item.directory {
background: #f8f9fa;
}
.icon {
font-size: 1.5rem;
}
.name {
flex: 1;
cursor: pointer;
}
.name:hover {
text-decoration: underline;
}
.size {
color: #6c757d;
font-size: 0.9rem;
}
.download-btn,
.delete-btn {
padding: 0.25rem 0.75rem;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 0.9rem;
}
.download-btn {
background: #28a745;
color: white;
}
.download-btn:hover {
background: #218838;
}
.delete-btn {
background: #dc3545;
color: white;
}
.delete-btn:hover {
background: #c82333;
}
.loading,
.empty {
text-align: center;
padding: 2rem;
color: #6c757d;
}
</style>
+326
View File
@@ -0,0 +1,326 @@
import { socket } from '$lib/stores/socket'
import * as Messages from '$lib/platform_shared/message'
import type {
FSDeleteRequest,
FSMkdirRequest,
FSListRequest,
FSDownloadStartRequest,
FSDownloadChunkRequest,
FSUploadStartRequest,
FSUploadChunkRequest,
FSCancelTransferRequest,
CorrelationResponse
} from '$lib/platform_shared/message'
const MAX_CHUNK_SIZE = 1024
export interface FileInfo {
name: string
size: number
}
export interface DirectoryInfo {
name: string
}
export interface ListResult {
success: boolean
error?: string
files: FileInfo[]
directories: DirectoryInfo[]
}
export interface TransferProgress {
transferId: string
bytesTransferred: number
totalBytes: number
chunksCompleted: number
totalChunks: number
percentage: number
}
export type ProgressCallback = (progress: TransferProgress) => void
export class FileSystemClient {
/**
* Delete a file or directory on the ESP32
*/
async deleteFile(path: string): Promise<{ success: boolean; error?: string }> {
const request: FSDeleteRequest = { path }
const response = await socket.request({
fsDeleteRequest: request
})
if (response.fsDeleteResponse) {
return {
success: response.fsDeleteResponse.success,
error: response.fsDeleteResponse.error || undefined
}
}
return { success: false, error: 'No response received' }
}
/**
* Create a directory on the ESP32
*/
async createDirectory(path: string): Promise<{ success: boolean; error?: string }> {
const request: FSMkdirRequest = { path }
const response = await socket.request({
fsMkdirRequest: request
})
if (response.fsMkdirResponse) {
return {
success: response.fsMkdirResponse.success,
error: response.fsMkdirResponse.error || undefined
}
}
return { success: false, error: 'No response received' }
}
/**
* List files and directories at the given path
*/
async listDirectory(path: string = '/'): Promise<ListResult> {
const request: FSListRequest = { path }
const response = await socket.request({
fsListRequest: request
})
if (response.fsListResponse) {
const resp = response.fsListResponse
return {
success: resp.success,
error: resp.error || undefined,
files: (resp.files || []).map((f) => ({ name: f.name, size: f.size })),
directories: (resp.directories || []).map((d) => ({ name: d.name }))
}
}
return { success: false, error: 'No response received', files: [], directories: [] }
}
/**
* Download a file from the ESP32
*/
async downloadFile(
path: string,
onProgress?: ProgressCallback
): Promise<{ success: boolean; data?: Uint8Array; error?: string }> {
// Start download
const startRequest: FSDownloadStartRequest = { path }
const startResponse = await socket.request({
fsDownloadStartRequest: startRequest
})
if (!startResponse.fsDownloadStartResponse) {
return { success: false, error: 'Failed to start download' }
}
const startResp = startResponse.fsDownloadStartResponse
if (!startResp.success) {
return { success: false, error: startResp.error || 'Failed to start download' }
}
const transferId = startResp.transferId
const totalChunks = startResp.totalChunks
const fileSize = startResp.fileSize
// Allocate buffer for entire file
const buffer = new Uint8Array(fileSize)
let offset = 0
// Download chunks sequentially
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
const chunkRequest: FSDownloadChunkRequest = {
transferId,
chunkIndex
}
const chunkResponse = await socket.request({
fsDownloadChunkRequest: chunkRequest
})
if (!chunkResponse.fsDownloadChunkResponse) {
await this.cancelTransfer(transferId)
return { success: false, error: `Failed to download chunk ${chunkIndex}` }
}
const chunkResp = chunkResponse.fsDownloadChunkResponse
if (chunkResp.error) {
await this.cancelTransfer(transferId)
return { success: false, error: chunkResp.error }
}
// Copy chunk data to buffer
if (chunkResp.data) {
buffer.set(chunkResp.data, offset)
offset += chunkResp.data.length
}
// Report progress
if (onProgress) {
onProgress({
transferId,
bytesTransferred: offset,
totalBytes: fileSize,
chunksCompleted: chunkIndex + 1,
totalChunks,
percentage: ((chunkIndex + 1) / totalChunks) * 100
})
}
}
return { success: true, data: buffer }
}
/**
* Upload a file to the ESP32
*/
async uploadFile(
path: string,
data: Uint8Array,
onProgress?: ProgressCallback
): Promise<{ success: boolean; error?: string }> {
const fileSize = data.length
const chunkSize = MAX_CHUNK_SIZE
const totalChunks = Math.ceil(fileSize / chunkSize)
// Start upload
const startRequest: FSUploadStartRequest = {
path,
fileSize,
chunkSize
}
const startResponse = await socket.request({
fsUploadStartRequest: startRequest
})
if (!startResponse.fsUploadStartResponse) {
return { success: false, error: 'Failed to start upload' }
}
const startResp = startResponse.fsUploadStartResponse
if (!startResp.success) {
return { success: false, error: startResp.error || 'Failed to start upload' }
}
const transferId = startResp.transferId
const maxChunkSize = startResp.maxChunkSize || MAX_CHUNK_SIZE
// Upload chunks sequentially
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
const offset = chunkIndex * chunkSize
const end = Math.min(offset + chunkSize, fileSize)
const chunkData = data.slice(offset, end)
const isLast = chunkIndex === totalChunks - 1
const chunkRequest: FSUploadChunkRequest = {
transferId,
chunkIndex,
data: chunkData,
isLast
}
const chunkResponse = await socket.request({
fsUploadChunkRequest: chunkRequest
})
if (!chunkResponse.fsUploadChunkResponse) {
await this.cancelTransfer(transferId)
return { success: false, error: `Failed to upload chunk ${chunkIndex}` }
}
const chunkResp = chunkResponse.fsUploadChunkResponse
if (!chunkResp.success) {
await this.cancelTransfer(transferId)
return { success: false, error: chunkResp.error || 'Failed to upload chunk' }
}
// Report progress
if (onProgress) {
onProgress({
transferId,
bytesTransferred: end,
totalBytes: fileSize,
chunksCompleted: chunkIndex + 1,
totalChunks,
percentage: ((chunkIndex + 1) / totalChunks) * 100
})
}
}
return { success: true }
}
/**
* Cancel an ongoing transfer
*/
async cancelTransfer(transferId: string): Promise<{ success: boolean }> {
const request: FSCancelTransferRequest = { transferId }
const response = await socket.request({
fsCancelTransferRequest: request
})
if (response.fsCancelTransferResponse) {
return { success: response.fsCancelTransferResponse.success }
}
return { success: false }
}
/**
* Helper: Upload a File object from browser
*/
async uploadFileFromBrowser(
destinationPath: string,
file: File,
onProgress?: ProgressCallback
): Promise<{ success: boolean; error?: string }> {
const arrayBuffer = await file.arrayBuffer()
const data = new Uint8Array(arrayBuffer)
return this.uploadFile(destinationPath, data, onProgress)
}
/**
* Helper: Download a file and save it to browser
*/
async downloadFileAndSave(
path: string,
filename: string,
onProgress?: ProgressCallback
): Promise<{ success: boolean; error?: string }> {
const result = await this.downloadFile(path, onProgress)
if (!result.success || !result.data) {
return { success: false, error: result.error }
}
// Create blob and trigger download
const blob = new Blob([result.data])
const url = URL.createObjectURL(blob)
const a = document.createElement('a')
a.href = url
a.download = filename
document.body.appendChild(a)
a.click()
document.body.removeChild(a)
URL.revokeObjectURL(url)
return { success: true }
}
}
export const fileSystemClient = new FileSystemClient()
@@ -41,6 +41,22 @@ DEFINE_MESSAGE_TRAITS(ServoPWMData, servo_pwm)
DEFINE_MESSAGE_TRAITS(ServoStateData, servo_state)
DEFINE_MESSAGE_TRAITS(CorrelationRequest, correlation_request)
DEFINE_MESSAGE_TRAITS(CorrelationResponse, correlation_response)
DEFINE_MESSAGE_TRAITS(FSDeleteRequest, fs_delete_request)
DEFINE_MESSAGE_TRAITS(FSDeleteResponse, fs_delete_response)
DEFINE_MESSAGE_TRAITS(FSMkdirRequest, fs_mkdir_request)
DEFINE_MESSAGE_TRAITS(FSMkdirResponse, fs_mkdir_response)
DEFINE_MESSAGE_TRAITS(FSListRequest, fs_list_request)
DEFINE_MESSAGE_TRAITS(FSListResponse, fs_list_response)
DEFINE_MESSAGE_TRAITS(FSDownloadStartRequest, fs_download_start_request)
DEFINE_MESSAGE_TRAITS(FSDownloadStartResponse, fs_download_start_response)
DEFINE_MESSAGE_TRAITS(FSDownloadChunkRequest, fs_download_chunk_request)
DEFINE_MESSAGE_TRAITS(FSDownloadChunkResponse, fs_download_chunk_response)
DEFINE_MESSAGE_TRAITS(FSUploadStartRequest, fs_upload_start_request)
DEFINE_MESSAGE_TRAITS(FSUploadStartResponse, fs_upload_start_response)
DEFINE_MESSAGE_TRAITS(FSUploadChunkRequest, fs_upload_chunk_request)
DEFINE_MESSAGE_TRAITS(FSUploadChunkResponse, fs_upload_chunk_response)
DEFINE_MESSAGE_TRAITS(FSCancelTransferRequest, fs_cancel_transfer_request)
DEFINE_MESSAGE_TRAITS(FSCancelTransferResponse, fs_cancel_transfer_response)
#undef DEFINE_MESSAGE_TRAITS
+62
View File
@@ -0,0 +1,62 @@
#pragma once
#include <LittleFS.h>
#include <platform_shared/message.pb.h>
#include <map>
#include <string>
#define FS_MAX_CHUNK_SIZE 1024
#define FS_TRANSFER_TIMEOUT 30000 // 30 seconds
namespace FileSystemWS {
struct TransferState {
std::string path;
File file;
uint32_t fileSize;
uint32_t chunkSize;
uint32_t totalChunks;
uint32_t chunksProcessed;
uint32_t lastActivityTime;
bool isUpload;
};
class FileSystemHandler {
public:
FileSystemHandler();
// Delete file/directory
socket_message_FSDeleteResponse handleDelete(const socket_message_FSDeleteRequest& req);
// Create directory
socket_message_FSMkdirResponse handleMkdir(const socket_message_FSMkdirRequest& req);
// List directory
socket_message_FSListResponse handleList(const socket_message_FSListRequest& req);
// Download operations (ESP -> Client)
socket_message_FSDownloadStartResponse handleDownloadStart(const socket_message_FSDownloadStartRequest& req);
socket_message_FSDownloadChunkResponse handleDownloadChunk(const socket_message_FSDownloadChunkRequest& req);
// Upload operations (Client -> ESP)
socket_message_FSUploadStartResponse handleUploadStart(const socket_message_FSUploadStartRequest& req);
socket_message_FSUploadChunkResponse handleUploadChunk(const socket_message_FSUploadChunkRequest& req);
// Cancel transfer
socket_message_FSCancelTransferResponse handleCancelTransfer(const socket_message_FSCancelTransferRequest& req);
// Cleanup expired transfers
void cleanupExpiredTransfers();
private:
std::map<std::string, TransferState> transfers_;
uint32_t transferIdCounter_;
std::string generateTransferId();
void listDirectory(const std::string& path, socket_message_FSListResponse& response);
bool deleteRecursive(const std::string& path);
};
extern FileSystemHandler fsHandler;
} // namespace FileSystemWS
+364
View File
@@ -0,0 +1,364 @@
#include <filesystem_ws.h>
#include <filesystem.h>
#include <pb_encode.h>
#include <pb_decode.h>
static const char* TAG = "FileSystemWS";
namespace FileSystemWS {
FileSystemHandler fsHandler;
FileSystemHandler::FileSystemHandler() : transferIdCounter_(0) {}
std::string FileSystemHandler::generateTransferId() {
return "xfer_" + std::to_string(millis()) + "_" + std::to_string(++transferIdCounter_);
}
void FileSystemHandler::cleanupExpiredTransfers() {
uint32_t now = millis();
auto it = transfers_.begin();
while (it != transfers_.end()) {
if (now - it->second.lastActivityTime > FS_TRANSFER_TIMEOUT) {
if (it->second.file) {
it->second.file.close();
}
ESP_LOGW(TAG, "Transfer %s timed out", it->first.c_str());
it = transfers_.erase(it);
} else {
++it;
}
}
}
socket_message_FSDeleteResponse FileSystemHandler::handleDelete(const socket_message_FSDeleteRequest& req) {
socket_message_FSDeleteResponse response = socket_message_FSDeleteResponse_init_zero;
std::string path(req.path);
ESP_LOGI(TAG, "Delete request: %s", path.c_str());
if (!ESP_FS.exists(path.c_str())) {
response.success = false;
strncpy(response.error, "File or directory not found", sizeof(response.error) - 1);
return response;
}
if (deleteRecursive(path)) {
response.success = true;
} else {
response.success = false;
strncpy(response.error, "Failed to delete", sizeof(response.error) - 1);
}
return response;
}
bool FileSystemHandler::deleteRecursive(const std::string& path) {
File file = ESP_FS.open(path.c_str());
if (!file) return false;
if (file.isDirectory()) {
File child = file.openNextFile();
while (child) {
std::string childPath = std::string(child.name());
child.close();
if (!deleteRecursive(childPath)) {
file.close();
return false;
}
child = file.openNextFile();
}
file.close();
return ESP_FS.rmdir(path.c_str());
} else {
file.close();
return ESP_FS.remove(path.c_str());
}
}
socket_message_FSMkdirResponse FileSystemHandler::handleMkdir(const socket_message_FSMkdirRequest& req) {
socket_message_FSMkdirResponse response = socket_message_FSMkdirResponse_init_zero;
std::string path(req.path);
ESP_LOGI(TAG, "Mkdir request: %s", path.c_str());
if (ESP_FS.exists(path.c_str())) {
response.success = false;
strncpy(response.error, "Path already exists", sizeof(response.error) - 1);
return response;
}
if (ESP_FS.mkdir(path.c_str())) {
response.success = true;
} else {
response.success = false;
strncpy(response.error, "Failed to create directory", sizeof(response.error) - 1);
}
return response;
}
void FileSystemHandler::listDirectory(const std::string& path, socket_message_FSListResponse& response) {
File dir = ESP_FS.open(path.c_str());
if (!dir || !dir.isDirectory()) {
return;
}
File file = dir.openNextFile();
int fileCount = 0;
int dirCount = 0;
while (file && fileCount < 50 && dirCount < 50) { // Limit to prevent overflow
if (file.isDirectory()) {
if (dirCount < 50) {
strncpy(response.directories[dirCount].name, file.name(), sizeof(response.directories[dirCount].name) - 1);
dirCount++;
}
} else {
if (fileCount < 50) {
strncpy(response.files[fileCount].name, file.name(), sizeof(response.files[fileCount].name) - 1);
response.files[fileCount].size = file.size();
fileCount++;
}
}
file = dir.openNextFile();
}
response.files_count = fileCount;
response.directories_count = dirCount;
}
socket_message_FSListResponse FileSystemHandler::handleList(const socket_message_FSListRequest& req) {
socket_message_FSListResponse response = socket_message_FSListResponse_init_zero;
std::string path(req.path);
if (path.empty()) path = "/";
ESP_LOGI(TAG, "List request: %s", path.c_str());
if (!ESP_FS.exists(path.c_str())) {
response.success = false;
strncpy(response.error, "Path not found", sizeof(response.error) - 1);
return response;
}
listDirectory(path, response);
response.success = true;
return response;
}
socket_message_FSDownloadStartResponse FileSystemHandler::handleDownloadStart(const socket_message_FSDownloadStartRequest& req) {
socket_message_FSDownloadStartResponse response = socket_message_FSDownloadStartResponse_init_zero;
std::string path(req.path);
ESP_LOGI(TAG, "Download start request: %s", path.c_str());
if (!ESP_FS.exists(path.c_str())) {
response.success = false;
strncpy(response.error, "File not found", sizeof(response.error) - 1);
return response;
}
File file = ESP_FS.open(path.c_str(), "r");
if (!file || file.isDirectory()) {
response.success = false;
strncpy(response.error, "Cannot open file for reading", sizeof(response.error) - 1);
return response;
}
uint32_t fileSize = file.size();
uint32_t chunkSize = FS_MAX_CHUNK_SIZE;
uint32_t totalChunks = (fileSize + chunkSize - 1) / chunkSize;
std::string transferId = generateTransferId();
TransferState state;
state.path = path;
state.file = file;
state.fileSize = fileSize;
state.chunkSize = chunkSize;
state.totalChunks = totalChunks;
state.chunksProcessed = 0;
state.lastActivityTime = millis();
state.isUpload = false;
transfers_[transferId] = state;
response.success = true;
response.file_size = fileSize;
response.chunk_size = chunkSize;
response.total_chunks = totalChunks;
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
ESP_LOGI(TAG, "Download started: %s, size=%u, chunks=%u, id=%s", path.c_str(), fileSize, totalChunks, transferId.c_str());
return response;
}
socket_message_FSDownloadChunkResponse FileSystemHandler::handleDownloadChunk(const socket_message_FSDownloadChunkRequest& req) {
socket_message_FSDownloadChunkResponse response = socket_message_FSDownloadChunkResponse_init_zero;
std::string transferId(req.transfer_id);
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
response.chunk_index = req.chunk_index;
auto it = transfers_.find(transferId);
if (it == transfers_.end()) {
strncpy(response.error, "Invalid transfer ID", sizeof(response.error) - 1);
return response;
}
TransferState& state = it->second;
state.lastActivityTime = millis();
// Seek to chunk position
uint32_t position = req.chunk_index * state.chunkSize;
if (!state.file.seek(position)) {
strncpy(response.error, "Failed to seek file", sizeof(response.error) - 1);
return response;
}
// Calculate chunk size (last chunk might be smaller)
uint32_t bytesToRead = state.chunkSize;
if (req.chunk_index == state.totalChunks - 1) {
bytesToRead = state.fileSize - position;
}
// Read chunk data
size_t bytesRead = state.file.read(response.data.bytes, bytesToRead);
response.data.size = bytesRead;
response.is_last = (req.chunk_index == state.totalChunks - 1);
ESP_LOGI(TAG, "Download chunk %u/%u: %u bytes", req.chunk_index + 1, state.totalChunks, bytesRead);
// Cleanup if last chunk
if (response.is_last) {
state.file.close();
transfers_.erase(it);
ESP_LOGI(TAG, "Download completed: %s", transferId.c_str());
}
return response;
}
socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const socket_message_FSUploadStartRequest& req) {
socket_message_FSUploadStartResponse response = socket_message_FSUploadStartResponse_init_zero;
std::string path(req.path);
ESP_LOGI(TAG, "Upload start request: %s, size=%u", path.c_str(), req.file_size);
// Ensure parent directory exists
size_t lastSlash = path.find_last_of('/');
if (lastSlash != std::string::npos && lastSlash > 0) {
std::string parentDir = path.substr(0, lastSlash);
if (!ESP_FS.exists(parentDir.c_str())) {
response.success = false;
strncpy(response.error, "Parent directory does not exist", sizeof(response.error) - 1);
return response;
}
}
File file = ESP_FS.open(path.c_str(), FILE_WRITE);
if (!file) {
response.success = false;
strncpy(response.error, "Cannot open file for writing", sizeof(response.error) - 1);
return response;
}
std::string transferId = generateTransferId();
TransferState state;
state.path = path;
state.file = file;
state.fileSize = req.file_size;
state.chunkSize = req.chunk_size > FS_MAX_CHUNK_SIZE ? FS_MAX_CHUNK_SIZE : req.chunk_size;
state.totalChunks = (req.file_size + state.chunkSize - 1) / state.chunkSize;
state.chunksProcessed = 0;
state.lastActivityTime = millis();
state.isUpload = true;
transfers_[transferId] = state;
response.success = true;
response.max_chunk_size = FS_MAX_CHUNK_SIZE;
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
ESP_LOGI(TAG, "Upload started: %s, id=%s", path.c_str(), transferId.c_str());
return response;
}
socket_message_FSUploadChunkResponse FileSystemHandler::handleUploadChunk(const socket_message_FSUploadChunkRequest& req) {
socket_message_FSUploadChunkResponse response = socket_message_FSUploadChunkResponse_init_zero;
std::string transferId(req.transfer_id);
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
response.chunk_index = req.chunk_index;
auto it = transfers_.find(transferId);
if (it == transfers_.end()) {
response.success = false;
strncpy(response.error, "Invalid transfer ID", sizeof(response.error) - 1);
return response;
}
TransferState& state = it->second;
state.lastActivityTime = millis();
// Write chunk data
size_t bytesWritten = state.file.write(req.data.bytes, req.data.size);
if (bytesWritten != req.data.size) {
response.success = false;
strncpy(response.error, "Failed to write chunk", sizeof(response.error) - 1);
state.file.close();
transfers_.erase(it);
return response;
}
state.chunksProcessed++;
response.success = true;
response.transfer_complete = req.is_last;
ESP_LOGI(TAG, "Upload chunk %u/%u: %u bytes", state.chunksProcessed, state.totalChunks, bytesWritten);
// Cleanup if last chunk
if (req.is_last) {
state.file.close();
transfers_.erase(it);
ESP_LOGI(TAG, "Upload completed: %s", state.path.c_str());
}
return response;
}
socket_message_FSCancelTransferResponse FileSystemHandler::handleCancelTransfer(const socket_message_FSCancelTransferRequest& req) {
socket_message_FSCancelTransferResponse response = socket_message_FSCancelTransferResponse_init_zero;
std::string transferId(req.transfer_id);
auto it = transfers_.find(transferId);
if (it == transfers_.end()) {
response.success = false;
return response;
}
if (it->second.file) {
it->second.file.close();
}
// Delete partial upload file
if (it->second.isUpload) {
ESP_FS.remove(it->second.path.c_str());
}
transfers_.erase(it);
response.success = true;
ESP_LOGI(TAG, "Transfer cancelled: %s", transferId.c_str());
return response;
}
} // namespace FileSystemWS
+113 -13
View File
@@ -11,28 +11,112 @@ message KnownNetworkItem { string ssid = 1; string password = 2; bool static_ip
// ----- FILESYSTEM -----
message File { string name = 10; } // Add permissions?
message File {
string name = 10;
uint32 size = 20;
}
message Directory {
string name = 10;
repeated File files = 20;
repeated Directory directories = 30;
}
enum FileRequestOptions {
DELETE = 0;
EDIT = 1;
MKDIR = 2;
GET = 3;
POST = 4;
START_UPLOAD = 10;
// Delete a file or directory
message FSDeleteRequest {
string path = 1;
}
message FSDelete { string path = 1; }
message FSDeleteResponse {
bool success = 1;
string error = 2;
}
message FilesystemRequest {
oneof type {
}
// Create directory
message FSMkdirRequest {
string path = 1;
}
message FSMkdirResponse {
bool success = 1;
string error = 2;
}
// List files/directories
message FSListRequest {
string path = 1;
}
message FSListResponse {
bool success = 1;
string error = 2;
repeated File files = 3;
repeated Directory directories = 4;
}
// Download from ESP (ESP -> Client)
message FSDownloadStartRequest {
string path = 1; // File path on ESP to download
}
message FSDownloadStartResponse {
bool success = 1;
string error = 2;
uint32 file_size = 3; // Total file size in bytes
uint32 chunk_size = 4; // Size of each chunk (max 1024)
uint32 total_chunks = 5; // Total number of chunks
string transfer_id = 6; // Unique ID for this transfer
}
message FSDownloadChunkRequest {
string transfer_id = 1;
uint32 chunk_index = 2; // Which chunk to request (0-based)
}
message FSDownloadChunkResponse {
string transfer_id = 1;
uint32 chunk_index = 2;
bytes data = 3; // Chunk data (max 1024 bytes)
bool is_last = 4; // True if this is the last chunk
string error = 5;
}
// Upload to ESP (Client -> ESP)
message FSUploadStartRequest {
string path = 1; // Destination path on ESP
uint32 file_size = 2; // Total file size in bytes
uint32 chunk_size = 3; // Size of each chunk (max 1024)
}
message FSUploadStartResponse {
bool success = 1;
string error = 2;
string transfer_id = 3; // Unique ID for this transfer
uint32 max_chunk_size = 4; // Server's max chunk size (1024)
}
message FSUploadChunkRequest {
string transfer_id = 1;
uint32 chunk_index = 2; // Which chunk this is (0-based)
bytes data = 3; // Chunk data (max 1024 bytes)
bool is_last = 4; // True if this is the last chunk
}
message FSUploadChunkResponse {
string transfer_id = 1;
uint32 chunk_index = 2;
bool success = 3;
string error = 4;
bool transfer_complete = 5; // True when all chunks received
}
// Cancel transfer
message FSCancelTransferRequest {
string transfer_id = 1;
}
message FSCancelTransferResponse {
bool success = 1;
}
// ----- FILESYSTEM -----
@@ -83,6 +167,14 @@ message CorrelationRequest {
I2CScanDataRequest i2c_scan_data_request = 20;
IMUCalibrateExecute imu_calibrate_execute = 30;
SystemInformationRequest system_information_request = 40;
FSDeleteRequest fs_delete_request = 50;
FSMkdirRequest fs_mkdir_request = 60;
FSListRequest fs_list_request = 70;
FSDownloadStartRequest fs_download_start_request = 80;
FSDownloadChunkRequest fs_download_chunk_request = 90;
FSUploadStartRequest fs_upload_start_request = 100;
FSUploadChunkRequest fs_upload_chunk_request = 110;
FSCancelTransferRequest fs_cancel_transfer_request = 120;
}
}
@@ -94,6 +186,14 @@ message CorrelationResponse {
I2CScanData i2c_scan_data = 20;
IMUCalibrateData imu_calibrate_data = 30;
SystemInformationResponse system_information_response = 40;
FSDeleteResponse fs_delete_response = 50;
FSMkdirResponse fs_mkdir_response = 60;
FSListResponse fs_list_response = 70;
FSDownloadStartResponse fs_download_start_response = 80;
FSDownloadChunkResponse fs_download_chunk_response = 90;
FSUploadStartResponse fs_upload_start_response = 100;
FSUploadChunkResponse fs_upload_chunk_response = 110;
FSCancelTransferResponse fs_cancel_transfer_response = 120;
}
}