diff --git a/app/src/lib/filesystem/chunkedTransfer.ts b/app/src/lib/filesystem/chunkedTransfer.ts index 0709988..b879c31 100644 --- a/app/src/lib/filesystem/chunkedTransfer.ts +++ b/app/src/lib/filesystem/chunkedTransfer.ts @@ -5,6 +5,7 @@ import type { FSMkdirRequest, FSListRequest, FSDownloadRequest, + FSDownloadMetadata, FSDownloadData, FSDownloadComplete, FSUploadStart, @@ -70,6 +71,16 @@ interface ActiveUpload { export class FileSystemClient { private activeDownloads = new Map() private activeUploads = new Map() + private pendingDownloads = new Map< + string, + { + resolve: (result: { success: boolean; data?: Uint8Array; error?: string }) => void + reject: (error: Error) => void + onProgress?: ProgressCallback + timeoutId: ReturnType + } + >() + private metadataListenerCleanup: (() => void) | null = null private downloadListenerCleanup: (() => void) | null = null private completeListenerCleanup: (() => void) | null = null private uploadCompleteListenerCleanup: (() => void) | null = null @@ -80,6 +91,14 @@ export class FileSystemClient { } private setupListeners() { + // Listen for download metadata (sent first with file size) + this.metadataListenerCleanup = socket.on( + Messages.FSDownloadMetadata, + (metadata: FSDownloadMetadata) => { + this.handleDownloadMetadata(metadata) + } + ) + // Listen for download data chunks this.downloadListenerCleanup = socket.on( Messages.FSDownloadData, @@ -105,6 +124,51 @@ export class FileSystemClient { ) } + private handleDownloadMetadata(metadata: FSDownloadMetadata) { + // Find the pending download by path (we don't have transferId yet) + // The metadata arrives in response to a download request + const pending = this.pendingDownloads.values().next().value + if (!pending) { + console.warn(`Received download metadata but no pending download`) + return + } + + // Clear initial timeout + clearTimeout(pending.timeoutId) + + // Get the path from the pending downloads (first one) + const [path] = this.pendingDownloads.keys() + this.pendingDownloads.delete(path) + + if (!metadata.success) { + pending.resolve({ success: false, error: metadata.error || 'Download failed' }) + return + } + + const transferId = metadata.transferId + + // Now we know the exact file size - allocate properly sized buffer + const buffer = new Uint8Array(metadata.fileSize) + + const download: ActiveDownload = { + path, + buffer, + fileSize: metadata.fileSize, + totalChunks: metadata.totalChunks, + chunksReceived: 0, + bytesReceived: 0, + resolve: pending.resolve, + reject: pending.reject, + onProgress: pending.onProgress, + timeoutId: setTimeout(() => { + this.activeDownloads.delete(transferId) + pending.reject(new Error('Download timeout')) + }, this.transferTimeout) + } + + this.activeDownloads.set(transferId, download) + } + private handleDownloadData(data: FSDownloadData) { const download = this.activeDownloads.get(data.transferId) if (!download) { @@ -244,80 +308,34 @@ export class FileSystemClient { /** * Download a file from the ESP32 using streaming transfer - * Server streams all chunks without waiting for ACKs + * Server sends metadata first (with file size), then streams all chunks */ async downloadFile( path: string, onProgress?: ProgressCallback ): Promise<{ success: boolean; data?: Uint8Array; error?: string }> { return new Promise((resolve, reject) => { - // Send download request - server will stream chunks back + // Send download request - server will send metadata first, then stream chunks const request: FSDownloadRequest = { path } - // We need to set up tracking before sending the request - // The server will generate a transfer ID and include it in all responses - // We'll capture the first chunk to get the transfer ID - - // Set up timeout for initial response + // Set up timeout for initial metadata response const initialTimeout = setTimeout(() => { - reject(new Error('Download request timeout - no data received')) + this.pendingDownloads.delete(path) + reject(new Error('Download request timeout - no metadata received')) }, this.transferTimeout) - // One-time listener for the first chunk to get transfer details - const firstChunkHandler = (data: FSDownloadData) => { - clearTimeout(initialTimeout) + // Track this pending download - will be converted to active when metadata arrives + this.pendingDownloads.set(path, { + resolve, + reject, + onProgress, + timeoutId: initialTimeout + }) - // Now we have the real transfer ID - const transferId = data.transferId - - // Estimate total size from first chunk (server sends file_size in complete message) - // For now, allocate a large buffer and resize later - const estimatedSize = 10 * 1024 * 1024 // 10MB max - const buffer = new Uint8Array(estimatedSize) - - const download: ActiveDownload = { - path, - buffer, - fileSize: estimatedSize, // Will be updated on completion - totalChunks: Math.ceil(estimatedSize / MAX_CHUNK_SIZE), - chunksReceived: 0, - bytesReceived: 0, - resolve, - reject, - onProgress, - timeoutId: setTimeout(() => { - this.activeDownloads.delete(transferId) - reject(new Error('Download timeout')) - }, this.transferTimeout) - } - - this.activeDownloads.set(transferId, download) - - // Process this first chunk - this.handleDownloadData(data) - - // Remove the first chunk handler - subsequent chunks go through normal listener - firstChunkCleanup() - } - - // Error handler for if download fails immediately - const errorHandler = (complete: FSDownloadComplete) => { - if (!complete.success && !complete.transferId) { - clearTimeout(initialTimeout) - firstChunkCleanup() - errorCleanup() - resolve({ success: false, error: complete.error || 'Download failed' }) - } - } - - const firstChunkCleanup = socket.on(Messages.FSDownloadData, firstChunkHandler) - const errorCleanup = socket.on(Messages.FSDownloadComplete, errorHandler) - - // Send the download request (no response expected, server streams data) + // Send the download request (server will respond with metadata, then stream data) socket.request({ fsDownloadRequest: request }).catch((err) => { clearTimeout(initialTimeout) - firstChunkCleanup() - errorCleanup() + this.pendingDownloads.delete(path) reject(err) }) }) diff --git a/esp32/include/communication/proto_helpers.h b/esp32/include/communication/proto_helpers.h index da43ddf..053eb37 100644 --- a/esp32/include/communication/proto_helpers.h +++ b/esp32/include/communication/proto_helpers.h @@ -43,6 +43,7 @@ DEFINE_MESSAGE_TRAITS(CorrelationRequest, correlation_request) DEFINE_MESSAGE_TRAITS(CorrelationResponse, correlation_response) // Streaming file transfer messages +DEFINE_MESSAGE_TRAITS(FSDownloadMetadata, fs_download_metadata) DEFINE_MESSAGE_TRAITS(FSDownloadData, fs_download_data) DEFINE_MESSAGE_TRAITS(FSDownloadComplete, fs_download_complete) DEFINE_MESSAGE_TRAITS(FSUploadData, fs_upload_data) diff --git a/esp32/include/filesystem_ws.h b/esp32/include/filesystem_ws.h index 9e19814..c008bc9 100644 --- a/esp32/include/filesystem_ws.h +++ b/esp32/include/filesystem_ws.h @@ -37,6 +37,7 @@ struct UploadState { }; // Callback type for sending messages to clients +using SendMetadataCallback = std::function; using SendCallback = std::function; using SendCompleteCallback = std::function; using SendUploadCompleteCallback = std::function; @@ -47,6 +48,7 @@ class FileSystemHandler { // Set callbacks for sending streaming data void setSendCallbacks( + SendMetadataCallback sendMetadata, SendCallback sendData, SendCompleteCallback sendComplete, SendUploadCompleteCallback sendUploadComplete @@ -84,6 +86,7 @@ class FileSystemHandler { std::map uploads_; uint32_t transferIdCounter_; + SendMetadataCallback sendMetadataCallback_; SendCallback sendDataCallback_; SendCompleteCallback sendCompleteCallback_; SendUploadCompleteCallback sendUploadCompleteCallback_; diff --git a/esp32/src/filesystem_ws.cpp b/esp32/src/filesystem_ws.cpp index fed3b0e..c37b368 100644 --- a/esp32/src/filesystem_ws.cpp +++ b/esp32/src/filesystem_ws.cpp @@ -13,10 +13,12 @@ FileSystemHandler fsHandler; FileSystemHandler::FileSystemHandler() : transferIdCounter_(0) {} void FileSystemHandler::setSendCallbacks( + SendMetadataCallback sendMetadata, SendCallback sendData, SendCompleteCallback sendComplete, SendUploadCompleteCallback sendUploadComplete ) { + sendMetadataCallback_ = sendMetadata; sendDataCallback_ = sendData; sendCompleteCallback_ = sendComplete; sendUploadCompleteCallback_ = sendUploadComplete; @@ -208,22 +210,22 @@ void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadReq // Validate file exists if (!ESP_FS.exists(path.c_str())) { - if (sendCompleteCallback_) { - socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero; - complete.success = false; - strncpy(complete.error, "File not found", sizeof(complete.error) - 1); - sendCompleteCallback_(complete, clientId); + if (sendMetadataCallback_) { + socket_message_FSDownloadMetadata metadata = socket_message_FSDownloadMetadata_init_zero; + metadata.success = false; + strncpy(metadata.error, "File not found", sizeof(metadata.error) - 1); + sendMetadataCallback_(metadata, clientId); } return; } File file = ESP_FS.open(path.c_str(), "r"); if (!file || file.isDirectory()) { - if (sendCompleteCallback_) { - socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero; - complete.success = false; - strncpy(complete.error, "Cannot open file for reading", sizeof(complete.error) - 1); - sendCompleteCallback_(complete, clientId); + if (sendMetadataCallback_) { + socket_message_FSDownloadMetadata metadata = socket_message_FSDownloadMetadata_init_zero; + metadata.success = false; + strncpy(metadata.error, "Cannot open file for reading", sizeof(metadata.error) - 1); + sendMetadataCallback_(metadata, clientId); } return; } @@ -235,6 +237,16 @@ void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadReq std::string transferId = generateTransferId(); + // Send metadata first so client knows exact file size and can allocate buffer + if (sendMetadataCallback_) { + socket_message_FSDownloadMetadata metadata = socket_message_FSDownloadMetadata_init_zero; + strncpy(metadata.transfer_id, transferId.c_str(), sizeof(metadata.transfer_id) - 1); + metadata.success = true; + metadata.file_size = fileSize; + metadata.total_chunks = totalChunks; + sendMetadataCallback_(metadata, clientId); + } + DownloadState state; state.path = path; state.file = file; diff --git a/esp32/src/main.cpp b/esp32/src/main.cpp index 8f5d37a..4987fba 100644 --- a/esp32/src/main.cpp +++ b/esp32/src/main.cpp @@ -137,6 +137,10 @@ void setupServer() { void setupEventSocket() { // Set up filesystem handler callbacks for streaming transfers FileSystemWS::fsHandler.setSendCallbacks( + // Send download metadata (file size, total chunks) + [](const socket_message_FSDownloadMetadata& metadata, int clientId) { + socket.emit(metadata, clientId); + }, // Send download data chunk [](const socket_message_FSDownloadData& data, int clientId) { socket.emit(data, clientId); diff --git a/platform_shared/message.options b/platform_shared/message.options index 2f4b18b..c4c8702 100644 --- a/platform_shared/message.options +++ b/platform_shared/message.options @@ -56,6 +56,8 @@ socket_message.FSListResponse.directories max_count:20 # Streaming download messages socket_message.FSDownloadRequest.path max_size:256 +socket_message.FSDownloadMetadata.transfer_id max_size:64 +socket_message.FSDownloadMetadata.error max_size:128 socket_message.FSDownloadData.transfer_id max_size:64 socket_message.FSDownloadData.data max_size:16384 socket_message.FSDownloadComplete.transfer_id max_size:64 diff --git a/platform_shared/message.proto b/platform_shared/message.proto index c8501ec..3b175ea 100644 --- a/platform_shared/message.proto +++ b/platform_shared/message.proto @@ -54,12 +54,20 @@ message FSListResponse { } // ===== STREAMING DOWNLOAD (ESP -> Client) ===== -// Flow: Client sends FSDownloadRequest -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete +// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete message FSDownloadRequest { string path = 1; // File path on ESP to download } +message FSDownloadMetadata { + string transfer_id = 1; // Transfer identifier + bool success = 2; // True if file exists and is readable + string error = 3; // Error message if failed + uint32 file_size = 4; // Total file size in bytes + uint32 total_chunks = 5; // Total number of chunks to expect +} + message FSDownloadData { string transfer_id = 1; // Transfer identifier uint32 chunk_index = 2; // Which chunk this is (0-based) @@ -312,6 +320,7 @@ message Message { PingMsg pingmsg = 30; PongMsg pongmsg = 31; // Streaming file transfer messages (fire-and-forget, no correlation) + FSDownloadMetadata fs_download_metadata = 39; FSDownloadData fs_download_data = 40; FSDownloadComplete fs_download_complete = 41; FSUploadData fs_upload_data = 42;