diff --git a/app/scripts/compile_protos.js b/app/scripts/compile_protos.js index 500458e..1886935 100644 --- a/app/scripts/compile_protos.js +++ b/app/scripts/compile_protos.js @@ -17,7 +17,7 @@ const pluginPath = path.join(projectRoot, 'node_modules', '.bin', 'protoc-gen-ts_proto.cmd') : path.join(projectRoot, 'node_modules', '.bin', 'protoc-gen-ts_proto') -const protoFiles = ['message.proto'] +const protoFiles = ['filesystem.proto', 'message.proto'] const tsProtoOpts = ['useExactTypes=false', 'outputExtensions=true', 'outputSchema=true'].join(',') diff --git a/app/src/lib/filesystem/chunkedTransfer.ts b/app/src/lib/filesystem/chunkedTransfer.ts index b879c31..ed4436a 100644 --- a/app/src/lib/filesystem/chunkedTransfer.ts +++ b/app/src/lib/filesystem/chunkedTransfer.ts @@ -1,5 +1,5 @@ import { socket } from '$lib/stores/socket' -import * as Messages from '$lib/platform_shared/message' +import * as FSMessages from '$lib/platform_shared/filesystem' import type { FSDeleteRequest, FSMkdirRequest, @@ -12,7 +12,7 @@ import type { FSUploadData, FSUploadComplete, FSCancelTransfer -} from '$lib/platform_shared/message' +} from '$lib/platform_shared/filesystem' const MAX_CHUNK_SIZE = 2 ** 14 // ~= 16 kb @@ -93,7 +93,7 @@ export class FileSystemClient { private setupListeners() { // Listen for download metadata (sent first with file size) this.metadataListenerCleanup = socket.on( - Messages.FSDownloadMetadata, + FSMessages.FSDownloadMetadata, (metadata: FSDownloadMetadata) => { this.handleDownloadMetadata(metadata) } @@ -101,7 +101,7 @@ export class FileSystemClient { // Listen for download data chunks this.downloadListenerCleanup = socket.on( - Messages.FSDownloadData, + FSMessages.FSDownloadData, (data: FSDownloadData) => { this.handleDownloadData(data) } @@ -109,7 +109,7 @@ export class FileSystemClient { // Listen for download completion this.completeListenerCleanup = socket.on( - Messages.FSDownloadComplete, + FSMessages.FSDownloadComplete, (complete: FSDownloadComplete) => { this.handleDownloadComplete(complete) } @@ -117,7 +117,7 @@ export class FileSystemClient { // Listen for upload completion this.uploadCompleteListenerCleanup = socket.on( - Messages.FSUploadComplete, + FSMessages.FSUploadComplete, (complete: FSUploadComplete) => { this.handleUploadComplete(complete) } @@ -408,7 +408,7 @@ export class FileSystemClient { } // Send chunk as fire-and-forget message - socket.emit(Messages.FSUploadData, uploadData) + socket.emit(FSMessages.FSUploadData, uploadData) upload.chunksSent++ diff --git a/app/src/lib/stores/socket.ts b/app/src/lib/stores/socket.ts index c24d88b..54fb050 100644 --- a/app/src/lib/stores/socket.ts +++ b/app/src/lib/stores/socket.ts @@ -7,6 +7,7 @@ import { type MessageFns } from '$lib/platform_shared/message' import * as Messages from '$lib/platform_shared/message' +import { protoMetadata as filesystemProtoMetadata } from '$lib/platform_shared/filesystem' export const MESSAGE_TYPE_TO_KEY = new Map, string>() export const MESSAGE_TYPE_TO_TAG = new Map, number>() @@ -20,6 +21,12 @@ type PendingRequest = { timeoutId: ReturnType } +// Combine references from both message.proto and filesystem.proto +const combinedReferences: Record> = { + ...protoMetadata.references, + ...filesystemProtoMetadata.references +} + const MessageType = protoMetadata.fileDescriptor.messageType?.find( (msg: { name: string }) => msg.name === 'Message' ) @@ -27,7 +34,7 @@ const MessageType = protoMetadata.fileDescriptor.messageType?.find( if (MessageType?.field) { for (const field of MessageType.field) { if (field.typeName) { - const messageFns = protoMetadata.references[field.typeName] + const messageFns = combinedReferences[field.typeName] if (messageFns && field.jsonName && field.number) { MESSAGE_TYPE_TO_KEY.set(messageFns, field.jsonName) MESSAGE_TYPE_TO_TAG.set(messageFns, field.number) diff --git a/esp32/scripts/compile_protos.py b/esp32/scripts/compile_protos.py index 6c616d3..0a23fe9 100644 --- a/esp32/scripts/compile_protos.py +++ b/esp32/scripts/compile_protos.py @@ -34,7 +34,7 @@ def compile_nanopb(): output_dir.mkdir(parents=True, exist_ok=True) - proto_files = [proto_dir / "message.proto"] + proto_files = [proto_dir / "filesystem.proto", proto_dir / "message.proto"] print(f"Compiling protobuf files with nanopb...") print(f" Proto dir: {proto_dir}") diff --git a/platform_shared/ADDING_PROTO_FILES.md b/platform_shared/ADDING_PROTO_FILES.md new file mode 100644 index 0000000..cbc181e --- /dev/null +++ b/platform_shared/ADDING_PROTO_FILES.md @@ -0,0 +1,140 @@ +# Adding New Proto Files + +## Step-by-Step Guide + +### 1. Create the new .proto file + +Create `platform_shared/myfeature.proto`: + +```protobuf +syntax = "proto3"; + +package socket_message; + +message MyFeatureRequest { + string name = 1; +} + +message MyFeatureResponse { + bool success = 1; + string error = 2; +} +``` + +### 2. Create the .options file (for nanopb size constraints) + +Create `platform_shared/myfeature.options`: + +``` +socket_message.MyFeatureRequest.name max_size:64 +socket_message.MyFeatureResponse.error max_size:128 +``` + +### 3. Import in message.proto + +Add the import at the top of `platform_shared/message.proto`: + +```protobuf +import "myfeature.proto"; +``` + +### 4. Add to Message oneof (if needed for streaming/pub-sub) + +If your message needs to be sent directly (not via correlation request/response), add it to the `Message` oneof in `message.proto`: + +```protobuf +message Message { + oneof message { + // ... existing fields ... + MyFeatureRequest my_feature_request = 300; // Pick unused tag number + MyFeatureResponse my_feature_response = 301; + } +} +``` + +### 5. Add to CorrelationRequest/Response (if using request/response pattern) + +For request/response messages, add to the correlation oneofs in `message.proto`: + +```protobuf +message CorrelationRequest { + oneof request { + // ... existing fields ... + MyFeatureRequest my_feature_request = 200; // Pick unused tag number + } +} + +message CorrelationResponse { + oneof response { + // ... existing fields ... + MyFeatureResponse my_feature_response = 200; + } +} +``` + +### 6. Update compile scripts + +**ESP32 (esp32/scripts/compile_protos.py):** + +```python +proto_files = [proto_dir / "filesystem.proto", proto_dir / "myfeature.proto", proto_dir / "message.proto"] +``` + +**TypeScript (app/scripts/compile_protos.js):** + +```javascript +const protoFiles = ['filesystem.proto', 'myfeature.proto', 'message.proto'] +``` + +### 7. Update socket.ts (for TypeScript - only if added to Message oneof) + +If you added messages to the `Message` oneof, import the protoMetadata in `app/src/lib/stores/socket.ts`: + +```typescript +import { protoMetadata as myfeatureProtoMetadata } from '$lib/platform_shared/myfeature' + +// Add to combinedReferences +const combinedReferences: Record> = { + ...protoMetadata.references, + ...filesystemProtoMetadata.references, + ...myfeatureProtoMetadata.references // Add this +} +``` + +### 8. Add MessageTraits (for ESP32 - only if added to Message oneof) + +If you added messages to the `Message` oneof, add traits in `esp32/include/communication/proto_helpers.h`: + +```cpp +// Before #undef DEFINE_MESSAGE_TRAITS +DEFINE_MESSAGE_TRAITS(MyFeatureRequest, my_feature_request) +DEFINE_MESSAGE_TRAITS(MyFeatureResponse, my_feature_response) +``` + +### 9. Build and test + +```bash +# ESP32 +pio run + +# TypeScript +cd app && pnpm proto +``` + +## Quick Reference + +| File | Purpose | +|------|---------| +| `platform_shared/*.proto` | Protocol definitions | +| `platform_shared/*.options` | Nanopb size constraints | +| `esp32/scripts/compile_protos.py` | ESP32 proto compilation | +| `app/scripts/compile_protos.js` | TypeScript proto compilation | +| `app/src/lib/stores/socket.ts` | Tag mapping for socket.on/emit | +| `esp32/include/communication/proto_helpers.h` | MessageTraits for ESP32 emit | + +## Notes + +- Messages in `CorrelationRequest/Response` don't need MessageTraits or socket.ts updates +- Messages in `Message` oneof (for streaming/pub-sub) need both +- Always use the same `package socket_message;` in all proto files +- Tag numbers in oneofs must be unique across all fields diff --git a/platform_shared/filesystem.options b/platform_shared/filesystem.options new file mode 100644 index 0000000..d1372b1 --- /dev/null +++ b/platform_shared/filesystem.options @@ -0,0 +1,36 @@ +# Filesystem message options +socket_message.File.name max_size:256 +socket_message.Directory.name max_size:256 + +socket_message.FSDeleteRequest.path max_size:256 +socket_message.FSDeleteResponse.error max_size:128 + +socket_message.FSMkdirRequest.path max_size:256 +socket_message.FSMkdirResponse.error max_size:128 + +socket_message.FSListRequest.path max_size:256 +socket_message.FSListResponse.error max_size:128 +socket_message.FSListResponse.files max_count:20 +socket_message.FSListResponse.directories max_count:20 + +# Streaming download messages +socket_message.FSDownloadRequest.path max_size:256 +socket_message.FSDownloadMetadata.transfer_id max_size:64 +socket_message.FSDownloadMetadata.error max_size:128 +socket_message.FSDownloadData.transfer_id max_size:64 +socket_message.FSDownloadData.data max_size:16384 +socket_message.FSDownloadComplete.transfer_id max_size:64 +socket_message.FSDownloadComplete.error max_size:128 + +# Streaming upload messages +socket_message.FSUploadStart.path max_size:256 +socket_message.FSUploadStartResponse.error max_size:128 +socket_message.FSUploadStartResponse.transfer_id max_size:64 +socket_message.FSUploadData.transfer_id max_size:64 +socket_message.FSUploadData.data max_size:16384 +socket_message.FSUploadComplete.transfer_id max_size:64 +socket_message.FSUploadComplete.error max_size:128 + +# Transfer control +socket_message.FSCancelTransfer.transfer_id max_size:64 +socket_message.FSCancelTransferResponse.transfer_id max_size:64 diff --git a/platform_shared/filesystem.proto b/platform_shared/filesystem.proto new file mode 100644 index 0000000..cc3d8f0 --- /dev/null +++ b/platform_shared/filesystem.proto @@ -0,0 +1,115 @@ +syntax = "proto3"; + +package socket_message; + +// ----- FILESYSTEM ----- + +message File { + string name = 10; + uint32 size = 20; +} + +// Represents a single directory entry (metadata only) +message Directory { + string name = 1; +} + +// Delete a file or directory +message FSDeleteRequest { + string path = 1; +} + +message FSDeleteResponse { + bool success = 1; + string error = 2; +} + +// Create directory +message FSMkdirRequest { + string path = 1; +} + +message FSMkdirResponse { + bool success = 1; + string error = 2; +} + +// List files/directories +message FSListRequest { + string path = 1; +} + +message FSListResponse { + bool success = 1; + string error = 2; + repeated File files = 3; + repeated Directory directories = 4; +} + +// ===== STREAMING DOWNLOAD (ESP -> Client) ===== +// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete + +message FSDownloadRequest { + string path = 1; // File path on ESP to download +} + +message FSDownloadMetadata { + string transfer_id = 1; // Transfer identifier + bool success = 2; // True if file exists and is readable + string error = 3; // Error message if failed + uint32 file_size = 4; // Total file size in bytes + uint32 total_chunks = 5; // Total number of chunks to expect +} + +message FSDownloadData { + string transfer_id = 1; // Transfer identifier + uint32 chunk_index = 2; // Which chunk this is (0-based) + bytes data = 3; // Chunk data (up to 16KB) +} + +message FSDownloadComplete { + string transfer_id = 1; + bool success = 2; + string error = 3; // Error message if failed + uint32 total_chunks = 4; // Total chunks that were sent + uint32 file_size = 5; // Total file size in bytes +} + +// ===== STREAMING UPLOAD (Client -> ESP) ===== +// Flow: Client sends FSUploadStart -> Server responds with transfer_id -> Client streams FSUploadData chunks -> Server sends FSUploadComplete + +message FSUploadStart { + string path = 1; // Destination path on ESP + uint32 file_size = 2; // Total file size in bytes + uint32 total_chunks = 3; // Total number of chunks to expect +} + +message FSUploadStartResponse { + bool success = 1; + string error = 2; + string transfer_id = 3; // Unique ID for this transfer +} + +message FSUploadData { + string transfer_id = 1; // Transfer identifier + uint32 chunk_index = 2; // Which chunk this is (0-based) + bytes data = 3; // Chunk data (up to 16KB) +} + +message FSUploadComplete { + string transfer_id = 1; + bool success = 2; + string error = 3; // Error message if failed + uint32 chunks_received = 4; // Number of chunks actually received +} + +// ===== TRANSFER CONTROL ===== + +message FSCancelTransfer { + string transfer_id = 1; +} + +message FSCancelTransferResponse { + string transfer_id = 1; + bool success = 2; +} diff --git a/platform_shared/message.options b/platform_shared/message.options index c4c8702..e98365c 100644 --- a/platform_shared/message.options +++ b/platform_shared/message.options @@ -37,41 +37,4 @@ socket_message.SonarData.dummy_field max_size:16 socket_message.FeaturesDataResponse.variant type:FT_POINTER socket_message.FeaturesDataResponse.firmware_built_target type:FT_POINTER socket_message.FeaturesDataResponse.firmware_name type:FT_POINTER -socket_message.FeaturesDataResponse.firmware_version type:FT_POINTER - -# Filesystem message options -socket_message.File.name max_size:256 -socket_message.Directory.name max_size:256 - -socket_message.FSDeleteRequest.path max_size:256 -socket_message.FSDeleteResponse.error max_size:128 - -socket_message.FSMkdirRequest.path max_size:256 -socket_message.FSMkdirResponse.error max_size:128 - -socket_message.FSListRequest.path max_size:256 -socket_message.FSListResponse.error max_size:128 -socket_message.FSListResponse.files max_count:20 -socket_message.FSListResponse.directories max_count:20 - -# Streaming download messages -socket_message.FSDownloadRequest.path max_size:256 -socket_message.FSDownloadMetadata.transfer_id max_size:64 -socket_message.FSDownloadMetadata.error max_size:128 -socket_message.FSDownloadData.transfer_id max_size:64 -socket_message.FSDownloadData.data max_size:16384 -socket_message.FSDownloadComplete.transfer_id max_size:64 -socket_message.FSDownloadComplete.error max_size:128 - -# Streaming upload messages -socket_message.FSUploadStart.path max_size:256 -socket_message.FSUploadStartResponse.error max_size:128 -socket_message.FSUploadStartResponse.transfer_id max_size:64 -socket_message.FSUploadData.transfer_id max_size:64 -socket_message.FSUploadData.data max_size:16384 -socket_message.FSUploadComplete.transfer_id max_size:64 -socket_message.FSUploadComplete.error max_size:128 - -# Transfer control -socket_message.FSCancelTransfer.transfer_id max_size:64 -socket_message.FSCancelTransferResponse.transfer_id max_size:64 \ No newline at end of file +socket_message.FeaturesDataResponse.firmware_version type:FT_POINTER \ No newline at end of file diff --git a/platform_shared/message.proto b/platform_shared/message.proto index 3b175ea..56bf2fc 100644 --- a/platform_shared/message.proto +++ b/platform_shared/message.proto @@ -2,127 +2,14 @@ syntax = "proto3"; package socket_message; +import "filesystem.proto"; + // Partial data types message Vector { float x = 1; float y = 2; } message I2CDevice { int32 address = 1; string part_number = 2; string name = 3; } message PinConfig { int32 pin = 1; string mode = 2; string type = 3; string role = 4; } message KnownNetworkItem { string ssid = 1; string password = 2; bool static_ip = 3; uint32 local_ip = 4; uint32 subnet_mask = 5; uint32 gateway_ip = 6; uint32 dns_ip_1 = 7; uint32 dns_ip_2 = 8; } - -// ----- FILESYSTEM ----- - -message File { - string name = 10; - uint32 size = 20; -} - -// Represents a single directory entry (metadata only) -message Directory { - string name = 1; -} - -// Delete a file or directory -message FSDeleteRequest { - string path = 1; -} - -message FSDeleteResponse { - bool success = 1; - string error = 2; -} - -// Create directory -message FSMkdirRequest { - string path = 1; -} - -message FSMkdirResponse { - bool success = 1; - string error = 2; -} - -// List files/directories -message FSListRequest { - string path = 1; -} - -message FSListResponse { - bool success = 1; - string error = 2; - repeated File files = 3; - repeated Directory directories = 4; -} - -// ===== STREAMING DOWNLOAD (ESP -> Client) ===== -// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete - -message FSDownloadRequest { - string path = 1; // File path on ESP to download -} - -message FSDownloadMetadata { - string transfer_id = 1; // Transfer identifier - bool success = 2; // True if file exists and is readable - string error = 3; // Error message if failed - uint32 file_size = 4; // Total file size in bytes - uint32 total_chunks = 5; // Total number of chunks to expect -} - -message FSDownloadData { - string transfer_id = 1; // Transfer identifier - uint32 chunk_index = 2; // Which chunk this is (0-based) - bytes data = 3; // Chunk data (up to 16KB) -} - -message FSDownloadComplete { - string transfer_id = 1; - bool success = 2; - string error = 3; // Error message if failed - uint32 total_chunks = 4; // Total chunks that were sent - uint32 file_size = 5; // Total file size in bytes -} - -// ===== STREAMING UPLOAD (Client -> ESP) ===== -// Flow: Client sends FSUploadStart -> Server responds with transfer_id -> Client streams FSUploadData chunks -> Server sends FSUploadComplete - -message FSUploadStart { - string path = 1; // Destination path on ESP - uint32 file_size = 2; // Total file size in bytes - uint32 total_chunks = 3; // Total number of chunks to expect -} - -message FSUploadStartResponse { - bool success = 1; - string error = 2; - string transfer_id = 3; // Unique ID for this transfer -} - -message FSUploadData { - string transfer_id = 1; // Transfer identifier - uint32 chunk_index = 2; // Which chunk this is (0-based) - bytes data = 3; // Chunk data (up to 16KB) -} - -message FSUploadComplete { - string transfer_id = 1; - bool success = 2; - string error = 3; // Error message if failed - uint32 chunks_received = 4; // Number of chunks actually received -} - -// ===== TRANSFER CONTROL ===== - -message FSCancelTransfer { - string transfer_id = 1; -} - -message FSCancelTransferResponse { - string transfer_id = 1; - bool success = 2; -} - -// ----- FILESYSTEM ----- - // Individual message data types message IMUData { float x = 1;