Remaking the upload and download scheme to rapid streaming (WIP)

This commit is contained in:
Niklas Jensen
2026-01-16 21:18:16 +01:00
committed by Rune Harlyk
parent 50ef91ab22
commit f0c4f0f929
7 changed files with 733 additions and 353 deletions
+276 -122
View File
@@ -12,22 +12,73 @@ FileSystemHandler fsHandler;
FileSystemHandler::FileSystemHandler() : transferIdCounter_(0) {}
void FileSystemHandler::setSendCallbacks(
SendCallback sendData,
SendCompleteCallback sendComplete,
SendUploadCompleteCallback sendUploadComplete
) {
sendDataCallback_ = sendData;
sendCompleteCallback_ = sendComplete;
sendUploadCompleteCallback_ = sendUploadComplete;
}
std::string FileSystemHandler::generateTransferId() {
return "xfer_" + std::to_string(millis()) + "_" + std::to_string(++transferIdCounter_);
}
void FileSystemHandler::cleanupExpiredTransfers() {
uint32_t now = millis();
auto it = transfers_.begin();
while (it != transfers_.end()) {
if (now - it->second.lastActivityTime > FS_TRANSFER_TIMEOUT) {
if (it->second.file) {
it->second.file.close();
// Cleanup expired downloads
auto dlIt = downloads_.begin();
while (dlIt != downloads_.end()) {
if (now - dlIt->second.lastActivityTime > FS_TRANSFER_TIMEOUT) {
if (dlIt->second.file) {
dlIt->second.file.close();
}
ESP_LOGW(TAG, "Transfer %s timed out", it->first.c_str());
it = transfers_.erase(it);
ESP_LOGW(TAG, "Download %s timed out", dlIt->first.c_str());
// Send error completion
if (sendCompleteCallback_) {
socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
strncpy(complete.transfer_id, dlIt->first.c_str(), sizeof(complete.transfer_id) - 1);
complete.success = false;
strncpy(complete.error, "Transfer timed out", sizeof(complete.error) - 1);
complete.total_chunks = dlIt->second.chunksSent;
complete.file_size = dlIt->second.fileSize;
sendCompleteCallback_(complete, dlIt->second.clientId);
}
dlIt = downloads_.erase(dlIt);
} else {
++it;
++dlIt;
}
}
// Cleanup expired uploads
auto ulIt = uploads_.begin();
while (ulIt != uploads_.end()) {
if (now - ulIt->second.lastActivityTime > FS_TRANSFER_TIMEOUT) {
if (ulIt->second.file) {
ulIt->second.file.close();
}
// Delete partial file
ESP_FS.remove(ulIt->second.path.c_str());
ESP_LOGW(TAG, "Upload %s timed out, deleted partial file", ulIt->first.c_str());
// Send error completion
if (sendUploadCompleteCallback_) {
socket_message_FSUploadComplete complete = socket_message_FSUploadComplete_init_zero;
strncpy(complete.transfer_id, ulIt->first.c_str(), sizeof(complete.transfer_id) - 1);
complete.success = false;
strncpy(complete.error, "Transfer timed out", sizeof(complete.error) - 1);
complete.chunks_received = ulIt->second.chunksReceived;
sendUploadCompleteCallback_(complete, ulIt->second.clientId);
}
ulIt = uploads_.erase(ulIt);
} else {
++ulIt;
}
}
}
@@ -149,106 +200,161 @@ socket_message_FSListResponse FileSystemHandler::handleList(const socket_message
return response;
}
socket_message_FSDownloadStartResponse FileSystemHandler::handleDownloadStart(const socket_message_FSDownloadStartRequest& req) {
socket_message_FSDownloadStartResponse response = socket_message_FSDownloadStartResponse_init_zero;
// ===== STREAMING DOWNLOAD =====
void FileSystemHandler::handleDownloadRequest(const socket_message_FSDownloadRequest& req, int clientId) {
std::string path(req.path);
ESP_LOGI(TAG, "Download start request: %s", path.c_str());
ESP_LOGI(TAG, "Download request: %s", path.c_str());
// Validate file exists
if (!ESP_FS.exists(path.c_str())) {
response.success = false;
strncpy(response.error, "File not found", sizeof(response.error) - 1);
return response;
if (sendCompleteCallback_) {
socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
complete.success = false;
strncpy(complete.error, "File not found", sizeof(complete.error) - 1);
sendCompleteCallback_(complete, clientId);
}
return;
}
File file = ESP_FS.open(path.c_str(), "r");
if (!file || file.isDirectory()) {
response.success = false;
strncpy(response.error, "Cannot open file for reading", sizeof(response.error) - 1);
return response;
if (sendCompleteCallback_) {
socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
complete.success = false;
strncpy(complete.error, "Cannot open file for reading", sizeof(complete.error) - 1);
sendCompleteCallback_(complete, clientId);
}
return;
}
uint32_t fileSize = file.size();
uint32_t chunkSize = FS_MAX_CHUNK_SIZE;
uint32_t totalChunks = (fileSize + chunkSize - 1) / chunkSize;
if (totalChunks == 0) totalChunks = 1; // Handle empty files
std::string transferId = generateTransferId();
TransferState state;
DownloadState state;
state.path = path;
state.file = file;
state.fileSize = fileSize;
state.chunkSize = chunkSize;
state.totalChunks = totalChunks;
state.chunksProcessed = 0;
state.chunksSent = 0;
state.lastActivityTime = millis();
state.isUpload = false;
state.clientId = clientId;
transfers_[transferId] = state;
downloads_[transferId] = state;
response.success = true;
response.file_size = fileSize;
response.chunk_size = chunkSize;
response.total_chunks = totalChunks;
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
ESP_LOGI(TAG, "Download started: %s, size=%u, chunks=%u, id=%s",
path.c_str(), fileSize, totalChunks, transferId.c_str());
ESP_LOGI(TAG, "Download started: %s, size=%u, chunks=%u, id=%s", path.c_str(), fileSize, totalChunks, transferId.c_str());
return response;
// Start streaming chunks immediately
while (sendNextDownloadChunk(transferId)) {
// Keep sending until done or error
taskYIELD(); // Allow other tasks to run
}
}
socket_message_FSDownloadChunkResponse FileSystemHandler::handleDownloadChunk(const socket_message_FSDownloadChunkRequest& req) {
socket_message_FSDownloadChunkResponse response = socket_message_FSDownloadChunkResponse_init_zero;
std::string transferId(req.transfer_id);
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
response.chunk_index = req.chunk_index;
auto it = transfers_.find(transferId);
if (it == transfers_.end()) {
strncpy(response.error, "Invalid transfer ID", sizeof(response.error) - 1);
return response;
bool FileSystemHandler::sendNextDownloadChunk(const std::string& transferId) {
auto it = downloads_.find(transferId);
if (it == downloads_.end()) {
return false;
}
TransferState& state = it->second;
DownloadState& state = it->second;
state.lastActivityTime = millis();
// Seek to chunk position
uint32_t position = req.chunk_index * state.chunkSize;
if (!state.file.seek(position)) {
strncpy(response.error, "Failed to seek file", sizeof(response.error) - 1);
return response;
// Check if we're done
if (state.chunksSent >= state.totalChunks) {
// Send completion message
if (sendCompleteCallback_) {
socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
strncpy(complete.transfer_id, transferId.c_str(), sizeof(complete.transfer_id) - 1);
complete.success = true;
complete.total_chunks = state.totalChunks;
complete.file_size = state.fileSize;
sendCompleteCallback_(complete, state.clientId);
}
state.file.close();
downloads_.erase(it);
ESP_LOGI(TAG, "Download completed: %s", transferId.c_str());
return false;
}
// Allocate data struct on heap to avoid stack overflow (it contains 16KB buffer)
auto data = new socket_message_FSDownloadData();
memset(data, 0, sizeof(socket_message_FSDownloadData));
strncpy(data->transfer_id, transferId.c_str(), sizeof(data->transfer_id) - 1);
data->chunk_index = state.chunksSent;
// Calculate chunk size (last chunk might be smaller)
uint32_t bytesToRead = state.chunkSize;
if (req.chunk_index == state.totalChunks - 1) {
uint32_t position = state.chunksSent * state.chunkSize;
if (position + bytesToRead > state.fileSize) {
bytesToRead = state.fileSize - position;
}
// Read chunk data
size_t bytesRead = state.file.read(response.data.bytes, bytesToRead);
response.data.size = bytesRead;
size_t bytesRead = state.file.read(data->data.bytes, bytesToRead);
if (bytesRead == 0 && bytesToRead > 0) {
// Read error - send error completion
delete data;
if (sendCompleteCallback_) {
socket_message_FSDownloadComplete complete = socket_message_FSDownloadComplete_init_zero;
strncpy(complete.transfer_id, transferId.c_str(), sizeof(complete.transfer_id) - 1);
complete.success = false;
strncpy(complete.error, "Failed to read file", sizeof(complete.error) - 1);
complete.total_chunks = state.chunksSent;
complete.file_size = state.fileSize;
sendCompleteCallback_(complete, state.clientId);
}
response.is_last = (req.chunk_index == state.totalChunks - 1);
ESP_LOGI(TAG, "Download chunk %u/%u: %u bytes", req.chunk_index + 1, state.totalChunks, bytesRead);
// Cleanup if last chunk
if (response.is_last) {
state.file.close();
transfers_.erase(it);
ESP_LOGI(TAG, "Download completed: %s", transferId.c_str());
downloads_.erase(it);
ESP_LOGE(TAG, "Download failed - read error: %s", transferId.c_str());
return false;
}
data->data.size = bytesRead;
// Send chunk
if (sendDataCallback_) {
sendDataCallback_(*data, state.clientId);
}
return response;
delete data;
state.chunksSent++;
ESP_LOGD(TAG, "Download chunk %u/%u sent: %u bytes", state.chunksSent, state.totalChunks, bytesRead);
return true;
}
socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const socket_message_FSUploadStartRequest& req) {
void FileSystemHandler::processPendingDownloads() {
// Process any pending downloads (in case we want non-blocking downloads)
// Currently downloads are synchronous in handleDownloadRequest
}
// ===== STREAMING UPLOAD =====
socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(
const socket_message_FSUploadStart& req, int clientId
) {
socket_message_FSUploadStartResponse response = socket_message_FSUploadStartResponse_init_zero;
std::string path(req.path);
ESP_LOGI(TAG, "Upload start request: %s, size=%u", path.c_str(), req.file_size);
ESP_LOGI(TAG, "Upload start request: %s, size=%u, chunks=%u", path.c_str(), req.file_size, req.total_chunks);
// Check available space
size_t fs_total = 0, fs_used = 0;
esp_littlefs_info("spiffs", &fs_total, &fs_used);
size_t freeSpace = fs_total - fs_used;
if (freeSpace < req.file_size + 4096) { // 4KB safety margin
response.success = false;
strncpy(response.error, "Insufficient storage space", sizeof(response.error) - 1);
return response;
}
// Ensure parent directory exists
size_t lastSlash = path.find_last_of('/');
@@ -270,20 +376,20 @@ socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const
std::string transferId = generateTransferId();
TransferState state;
UploadState state;
state.path = path;
state.file = file;
state.fileSize = req.file_size;
state.chunkSize = req.chunk_size > FS_MAX_CHUNK_SIZE ? FS_MAX_CHUNK_SIZE : req.chunk_size;
state.totalChunks = (req.file_size + state.chunkSize - 1) / state.chunkSize;
state.chunksProcessed = 0;
state.totalChunks = req.total_chunks;
state.chunksReceived = 0;
state.bytesReceived = 0;
state.lastActivityTime = millis();
state.isUpload = true;
state.clientId = clientId;
state.hasError = false;
transfers_[transferId] = state;
uploads_[transferId] = state;
response.success = true;
response.max_chunk_size = FS_MAX_CHUNK_SIZE;
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
ESP_LOGI(TAG, "Upload started: %s, id=%s", path.c_str(), transferId.c_str());
@@ -291,91 +397,139 @@ socket_message_FSUploadStartResponse FileSystemHandler::handleUploadStart(const
return response;
}
socket_message_FSUploadChunkResponse FileSystemHandler::handleUploadChunk(const socket_message_FSUploadChunkRequest& req) {
socket_message_FSUploadChunkResponse fs_up_response = socket_message_FSUploadChunkResponse_init_zero;
void FileSystemHandler::handleUploadData(const socket_message_FSUploadData& req) {
std::string transferId(req.transfer_id);
strncpy(fs_up_response.transfer_id, transferId.c_str(), sizeof(fs_up_response.transfer_id) - 1);
fs_up_response.chunk_index = req.chunk_index;
auto it = transfers_.find(transferId);
if (it == transfers_.end()) {
fs_up_response.success = false;
strncpy(fs_up_response.error, "Invalid transfer ID", sizeof(fs_up_response.error) - 1);
return fs_up_response;
auto it = uploads_.find(transferId);
if (it == uploads_.end()) {
ESP_LOGW(TAG, "Upload data for unknown transfer: %s", transferId.c_str());
return;
}
TransferState& state = it->second;
UploadState& state = it->second;
state.lastActivityTime = millis();
// Skip if we already have an error
if (state.hasError) {
return;
}
// Validate chunk index (allow out-of-order but warn)
if (req.chunk_index != state.chunksReceived) {
ESP_LOGW(TAG, "Upload chunk out of order: expected %u, got %u", state.chunksReceived, req.chunk_index);
// For now, we'll accept it anyway and write sequentially
// A more robust implementation would buffer out-of-order chunks
}
// Check available space before writing
size_t fs_total = 0, fs_used = 0;
esp_littlefs_info("spiffs", &fs_total, &fs_used);
size_t freeSpace = fs_total - fs_used;
if (freeSpace < req.data.size + 4096) { // 4KB safety margin
fs_up_response.success = false;
strncpy(fs_up_response.error, "Filesystem full", sizeof(fs_up_response.error) - 1);
state.file.close();
transfers_.erase(it);
return fs_up_response;
size_t freeSpace = fs_total - fs_used;
if (freeSpace < req.data.size + 4096) {
state.hasError = true;
state.errorMessage = "Filesystem full";
finalizeUpload(transferId, false, state.errorMessage);
return;
}
// Write chunk data
size_t bytesWritten = state.file.write(req.data.bytes, req.data.size);
if (bytesWritten != req.data.size) {
fs_up_response.success = false;
strncpy(fs_up_response.error, "Failed to write chunk", sizeof(fs_up_response.error) - 1);
state.file.close();
transfers_.erase(it);
return fs_up_response;
state.hasError = true;
state.errorMessage = "Failed to write chunk";
finalizeUpload(transferId, false, state.errorMessage);
return;
}
// Flush periodically to prevent LittleFS buffer issues (every 64 chunks = ~64KB with 1KB chunks)
if (state.chunksProcessed > 0 && state.chunksProcessed % 64 == 0) {
ESP_LOGI(TAG, "Flushing file at chunk %u", state.chunksProcessed);
state.chunksReceived++;
state.bytesReceived += bytesWritten;
// Flush periodically to prevent LittleFS buffer issues
if (state.chunksReceived > 0 && state.chunksReceived % 64 == 0) {
ESP_LOGD(TAG, "Flushing file at chunk %u", state.chunksReceived);
state.file.flush();
}
state.chunksProcessed++;
fs_up_response.success = true;
fs_up_response.transfer_complete = req.is_last;
ESP_LOGD(TAG, "Upload chunk %u/%u: %u bytes", state.chunksReceived, state.totalChunks, bytesWritten);
ESP_LOGI(TAG, "Upload chunk %u/%u: %u bytes", state.chunksProcessed, state.totalChunks, bytesWritten);
// Cleanup if last chunk
if (req.is_last) {
state.file.close();
transfers_.erase(it);
ESP_LOGI(TAG, "Upload completed: %s", state.path.c_str());
// Check if upload is complete
if (state.chunksReceived >= state.totalChunks) {
finalizeUpload(transferId, true);
}
return fs_up_response;
}
socket_message_FSCancelTransferResponse FileSystemHandler::handleCancelTransfer(const socket_message_FSCancelTransferRequest& req) {
void FileSystemHandler::finalizeUpload(const std::string& transferId, bool success, const std::string& error) {
auto it = uploads_.find(transferId);
if (it == uploads_.end()) {
return;
}
UploadState& state = it->second;
// Close file
if (state.file) {
state.file.close();
}
// Delete file on error
if (!success) {
ESP_FS.remove(state.path.c_str());
ESP_LOGW(TAG, "Upload failed, deleted partial file: %s", state.path.c_str());
} else {
ESP_LOGI(TAG, "Upload completed: %s (%u bytes)", state.path.c_str(), state.bytesReceived);
}
// Send completion message
if (sendUploadCompleteCallback_) {
socket_message_FSUploadComplete complete = socket_message_FSUploadComplete_init_zero;
strncpy(complete.transfer_id, transferId.c_str(), sizeof(complete.transfer_id) - 1);
complete.success = success;
if (!error.empty()) {
strncpy(complete.error, error.c_str(), sizeof(complete.error) - 1);
}
complete.chunks_received = state.chunksReceived;
sendUploadCompleteCallback_(complete, state.clientId);
}
uploads_.erase(it);
}
// ===== TRANSFER CONTROL =====
socket_message_FSCancelTransferResponse FileSystemHandler::handleCancelTransfer(
const socket_message_FSCancelTransfer& req
) {
socket_message_FSCancelTransferResponse response = socket_message_FSCancelTransferResponse_init_zero;
std::string transferId(req.transfer_id);
auto it = transfers_.find(transferId);
strncpy(response.transfer_id, transferId.c_str(), sizeof(response.transfer_id) - 1);
if (it == transfers_.end()) {
response.success = false;
// Check downloads
auto dlIt = downloads_.find(transferId);
if (dlIt != downloads_.end()) {
if (dlIt->second.file) {
dlIt->second.file.close();
}
downloads_.erase(dlIt);
response.success = true;
ESP_LOGI(TAG, "Download cancelled: %s", transferId.c_str());
return response;
}
if (it->second.file) {
it->second.file.close();
// Check uploads
auto ulIt = uploads_.find(transferId);
if (ulIt != uploads_.end()) {
if (ulIt->second.file) {
ulIt->second.file.close();
}
// Delete partial upload file
ESP_FS.remove(ulIt->second.path.c_str());
uploads_.erase(ulIt);
response.success = true;
ESP_LOGI(TAG, "Upload cancelled: %s", transferId.c_str());
return response;
}
// Delete partial upload file
if (it->second.isUpload) {
ESP_FS.remove(it->second.path.c_str());
}
transfers_.erase(it);
response.success = true;
ESP_LOGI(TAG, "Transfer cancelled: %s", transferId.c_str());
response.success = false;
return response;
}