Moved FS proto objects to own file, and MD tutorial

app/scripts/compile_protos.js

@@ -17,7 +17,7 @@ const pluginPath =
         path.join(projectRoot, 'node_modules', '.bin', 'protoc-gen-ts_proto.cmd')
     :   path.join(projectRoot, 'node_modules', '.bin', 'protoc-gen-ts_proto')
 
-const protoFiles = ['message.proto']
+const protoFiles = ['filesystem.proto', 'message.proto']
 
 const tsProtoOpts = ['useExactTypes=false', 'outputExtensions=true', 'outputSchema=true'].join(',')
 

app/src/lib/filesystem/chunkedTransfer.ts

@@ -1,5 +1,5 @@
 import { socket } from '$lib/stores/socket'
-import * as Messages from '$lib/platform_shared/message'
+import * as FSMessages from '$lib/platform_shared/filesystem'
 import type {
 	FSDeleteRequest,
 	FSMkdirRequest,
@@ -12,7 +12,7 @@ import type {
 	FSUploadData,
 	FSUploadComplete,
 	FSCancelTransfer
-} from '$lib/platform_shared/message'
+} from '$lib/platform_shared/filesystem'
 
 const MAX_CHUNK_SIZE = 2 ** 14 // ~= 16 kb
 
@@ -93,7 +93,7 @@ export class FileSystemClient {
 	private setupListeners() {
 		// Listen for download metadata (sent first with file size)
 		this.metadataListenerCleanup = socket.on(
-			Messages.FSDownloadMetadata,
+			FSMessages.FSDownloadMetadata,
 			(metadata: FSDownloadMetadata) => {
 				this.handleDownloadMetadata(metadata)
 			}
@@ -101,7 +101,7 @@ export class FileSystemClient {
 
 		// Listen for download data chunks
 		this.downloadListenerCleanup = socket.on(
-			Messages.FSDownloadData,
+			FSMessages.FSDownloadData,
 			(data: FSDownloadData) => {
 				this.handleDownloadData(data)
 			}
@@ -109,7 +109,7 @@ export class FileSystemClient {
 
 		// Listen for download completion
 		this.completeListenerCleanup = socket.on(
-			Messages.FSDownloadComplete,
+			FSMessages.FSDownloadComplete,
 			(complete: FSDownloadComplete) => {
 				this.handleDownloadComplete(complete)
 			}
@@ -117,7 +117,7 @@ export class FileSystemClient {
 
 		// Listen for upload completion
 		this.uploadCompleteListenerCleanup = socket.on(
-			Messages.FSUploadComplete,
+			FSMessages.FSUploadComplete,
 			(complete: FSUploadComplete) => {
 				this.handleUploadComplete(complete)
 			}
@@ -408,7 +408,7 @@ export class FileSystemClient {
 				}
 
 				// Send chunk as fire-and-forget message
-				socket.emit(Messages.FSUploadData, uploadData)
+				socket.emit(FSMessages.FSUploadData, uploadData)
 
 				upload.chunksSent++
 

app/src/lib/stores/socket.ts

@@ -7,6 +7,7 @@ import {
     type MessageFns
 } from '$lib/platform_shared/message'
 import * as Messages from '$lib/platform_shared/message'
+import { protoMetadata as filesystemProtoMetadata } from '$lib/platform_shared/filesystem'
 
 export const MESSAGE_TYPE_TO_KEY = new Map<MessageFns<unknown>, string>()
 export const MESSAGE_TYPE_TO_TAG = new Map<MessageFns<unknown>, number>()
@@ -20,6 +21,12 @@ type PendingRequest = {
     timeoutId: ReturnType<typeof setTimeout>
 }
 
+// Combine references from both message.proto and filesystem.proto
+const combinedReferences: Record<string, MessageFns<unknown>> = {
+    ...protoMetadata.references,
+    ...filesystemProtoMetadata.references
+}
+
 const MessageType = protoMetadata.fileDescriptor.messageType?.find(
     (msg: { name: string }) => msg.name === 'Message'
 )
@@ -27,7 +34,7 @@ const MessageType = protoMetadata.fileDescriptor.messageType?.find(
 if (MessageType?.field) {
     for (const field of MessageType.field) {
         if (field.typeName) {
-            const messageFns = protoMetadata.references[field.typeName]
+            const messageFns = combinedReferences[field.typeName]
             if (messageFns && field.jsonName && field.number) {
                 MESSAGE_TYPE_TO_KEY.set(messageFns, field.jsonName)
                 MESSAGE_TYPE_TO_TAG.set(messageFns, field.number)

esp32/scripts/compile_protos.py

@@ -34,7 +34,7 @@ def compile_nanopb():
     
     output_dir.mkdir(parents=True, exist_ok=True)
     
-    proto_files = [proto_dir / "message.proto"]
+    proto_files = [proto_dir / "filesystem.proto", proto_dir / "message.proto"]
     
     print(f"Compiling protobuf files with nanopb...")
     print(f"  Proto dir: {proto_dir}")

platform_shared/ADDING_PROTO_FILES.md

@@ -0,0 +1,140 @@
+# Adding New Proto Files
+
+## Step-by-Step Guide
+
+### 1. Create the new .proto file
+
+Create `platform_shared/myfeature.proto`:
+
+```protobuf
+syntax = "proto3";
+
+package socket_message;
+
+message MyFeatureRequest {
+    string name = 1;
+}
+
+message MyFeatureResponse {
+    bool success = 1;
+    string error = 2;
+}
+```
+
+### 2. Create the .options file (for nanopb size constraints)
+
+Create `platform_shared/myfeature.options`:
+
+```
+socket_message.MyFeatureRequest.name max_size:64
+socket_message.MyFeatureResponse.error max_size:128
+```
+
+### 3. Import in message.proto
+
+Add the import at the top of `platform_shared/message.proto`:
+
+```protobuf
+import "myfeature.proto";
+```
+
+### 4. Add to Message oneof (if needed for streaming/pub-sub)
+
+If your message needs to be sent directly (not via correlation request/response), add it to the `Message` oneof in `message.proto`:
+
+```protobuf
+message Message {
+    oneof message {
+        // ... existing fields ...
+        MyFeatureRequest my_feature_request = 300;  // Pick unused tag number
+        MyFeatureResponse my_feature_response = 301;
+    }
+}
+```
+
+### 5. Add to CorrelationRequest/Response (if using request/response pattern)
+
+For request/response messages, add to the correlation oneofs in `message.proto`:
+
+```protobuf
+message CorrelationRequest {
+    oneof request {
+        // ... existing fields ...
+        MyFeatureRequest my_feature_request = 200;  // Pick unused tag number
+    }
+}
+
+message CorrelationResponse {
+    oneof response {
+        // ... existing fields ...
+        MyFeatureResponse my_feature_response = 200;
+    }
+}
+```
+
+### 6. Update compile scripts
+
+**ESP32 (esp32/scripts/compile_protos.py):**
+
+```python
+proto_files = [proto_dir / "filesystem.proto", proto_dir / "myfeature.proto", proto_dir / "message.proto"]
+```
+
+**TypeScript (app/scripts/compile_protos.js):**
+
+```javascript
+const protoFiles = ['filesystem.proto', 'myfeature.proto', 'message.proto']
+```
+
+### 7. Update socket.ts (for TypeScript - only if added to Message oneof)
+
+If you added messages to the `Message` oneof, import the protoMetadata in `app/src/lib/stores/socket.ts`:
+
+```typescript
+import { protoMetadata as myfeatureProtoMetadata } from '$lib/platform_shared/myfeature'
+
+// Add to combinedReferences
+const combinedReferences: Record<string, MessageFns<unknown>> = {
+    ...protoMetadata.references,
+    ...filesystemProtoMetadata.references,
+    ...myfeatureProtoMetadata.references  // Add this
+}
+```
+
+### 8. Add MessageTraits (for ESP32 - only if added to Message oneof)
+
+If you added messages to the `Message` oneof, add traits in `esp32/include/communication/proto_helpers.h`:
+
+```cpp
+// Before #undef DEFINE_MESSAGE_TRAITS
+DEFINE_MESSAGE_TRAITS(MyFeatureRequest, my_feature_request)
+DEFINE_MESSAGE_TRAITS(MyFeatureResponse, my_feature_response)
+```
+
+### 9. Build and test
+
+```bash
+# ESP32
+pio run
+
+# TypeScript
+cd app && pnpm proto
+```
+
+## Quick Reference
+
+| File | Purpose |
+|------|---------|
+| `platform_shared/*.proto` | Protocol definitions |
+| `platform_shared/*.options` | Nanopb size constraints |
+| `esp32/scripts/compile_protos.py` | ESP32 proto compilation |
+| `app/scripts/compile_protos.js` | TypeScript proto compilation |
+| `app/src/lib/stores/socket.ts` | Tag mapping for socket.on/emit |
+| `esp32/include/communication/proto_helpers.h` | MessageTraits for ESP32 emit |
+
+## Notes
+
+- Messages in `CorrelationRequest/Response` don't need MessageTraits or socket.ts updates
+- Messages in `Message` oneof (for streaming/pub-sub) need both
+- Always use the same `package socket_message;` in all proto files
+- Tag numbers in oneofs must be unique across all fields

platform_shared/filesystem.options

@@ -0,0 +1,36 @@
+# Filesystem message options
+socket_message.File.name max_size:256
+socket_message.Directory.name max_size:256
+
+socket_message.FSDeleteRequest.path max_size:256
+socket_message.FSDeleteResponse.error max_size:128
+
+socket_message.FSMkdirRequest.path max_size:256
+socket_message.FSMkdirResponse.error max_size:128
+
+socket_message.FSListRequest.path max_size:256
+socket_message.FSListResponse.error max_size:128
+socket_message.FSListResponse.files max_count:20
+socket_message.FSListResponse.directories max_count:20
+
+# Streaming download messages
+socket_message.FSDownloadRequest.path max_size:256
+socket_message.FSDownloadMetadata.transfer_id max_size:64
+socket_message.FSDownloadMetadata.error max_size:128
+socket_message.FSDownloadData.transfer_id max_size:64
+socket_message.FSDownloadData.data max_size:16384
+socket_message.FSDownloadComplete.transfer_id max_size:64
+socket_message.FSDownloadComplete.error max_size:128
+
+# Streaming upload messages
+socket_message.FSUploadStart.path max_size:256
+socket_message.FSUploadStartResponse.error max_size:128
+socket_message.FSUploadStartResponse.transfer_id max_size:64
+socket_message.FSUploadData.transfer_id max_size:64
+socket_message.FSUploadData.data max_size:16384
+socket_message.FSUploadComplete.transfer_id max_size:64
+socket_message.FSUploadComplete.error max_size:128
+
+# Transfer control
+socket_message.FSCancelTransfer.transfer_id max_size:64
+socket_message.FSCancelTransferResponse.transfer_id max_size:64

platform_shared/filesystem.proto

@@ -0,0 +1,115 @@
+syntax = "proto3";
+
+package socket_message;
+
+// ----- FILESYSTEM -----
+
+message File {
+    string name = 10;
+    uint32 size = 20;
+}
+
+// Represents a single directory entry (metadata only)
+message Directory {
+    string name = 1;
+}
+
+// Delete a file or directory
+message FSDeleteRequest {
+    string path = 1;
+}
+
+message FSDeleteResponse {
+    bool success = 1;
+    string error = 2;
+}
+
+// Create directory
+message FSMkdirRequest {
+    string path = 1;
+}
+
+message FSMkdirResponse {
+    bool success = 1;
+    string error = 2;
+}
+
+// List files/directories
+message FSListRequest {
+    string path = 1;
+}
+
+message FSListResponse {
+    bool success = 1;
+    string error = 2;
+    repeated File files = 3;
+    repeated Directory directories = 4;
+}
+
+// ===== STREAMING DOWNLOAD (ESP -> Client) =====
+// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete
+
+message FSDownloadRequest {
+    string path = 1;           // File path on ESP to download
+}
+
+message FSDownloadMetadata {
+    string transfer_id = 1;    // Transfer identifier
+    bool success = 2;          // True if file exists and is readable
+    string error = 3;          // Error message if failed
+    uint32 file_size = 4;      // Total file size in bytes
+    uint32 total_chunks = 5;   // Total number of chunks to expect
+}
+
+message FSDownloadData {
+    string transfer_id = 1;    // Transfer identifier
+    uint32 chunk_index = 2;    // Which chunk this is (0-based)
+    bytes data = 3;            // Chunk data (up to 16KB)
+}
+
+message FSDownloadComplete {
+    string transfer_id = 1;
+    bool success = 2;
+    string error = 3;          // Error message if failed
+    uint32 total_chunks = 4;   // Total chunks that were sent
+    uint32 file_size = 5;      // Total file size in bytes
+}
+
+// ===== STREAMING UPLOAD (Client -> ESP) =====
+// Flow: Client sends FSUploadStart -> Server responds with transfer_id -> Client streams FSUploadData chunks -> Server sends FSUploadComplete
+
+message FSUploadStart {
+    string path = 1;           // Destination path on ESP
+    uint32 file_size = 2;      // Total file size in bytes
+    uint32 total_chunks = 3;   // Total number of chunks to expect
+}
+
+message FSUploadStartResponse {
+    bool success = 1;
+    string error = 2;
+    string transfer_id = 3;    // Unique ID for this transfer
+}
+
+message FSUploadData {
+    string transfer_id = 1;    // Transfer identifier
+    uint32 chunk_index = 2;    // Which chunk this is (0-based)
+    bytes data = 3;            // Chunk data (up to 16KB)
+}
+
+message FSUploadComplete {
+    string transfer_id = 1;
+    bool success = 2;
+    string error = 3;          // Error message if failed
+    uint32 chunks_received = 4; // Number of chunks actually received
+}
+
+// ===== TRANSFER CONTROL =====
+
+message FSCancelTransfer {
+    string transfer_id = 1;
+}
+
+message FSCancelTransferResponse {
+    string transfer_id = 1;
+    bool success = 2;
+}

platform_shared/message.options

@@ -38,40 +38,3 @@ socket_message.FeaturesDataResponse.variant type:FT_POINTER
 socket_message.FeaturesDataResponse.firmware_built_target type:FT_POINTER
 socket_message.FeaturesDataResponse.firmware_name type:FT_POINTER
 socket_message.FeaturesDataResponse.firmware_version type:FT_POINTER
-
-# Filesystem message options
-socket_message.File.name max_size:256
-socket_message.Directory.name max_size:256
-
-socket_message.FSDeleteRequest.path max_size:256
-socket_message.FSDeleteResponse.error max_size:128
-
-socket_message.FSMkdirRequest.path max_size:256
-socket_message.FSMkdirResponse.error max_size:128
-
-socket_message.FSListRequest.path max_size:256
-socket_message.FSListResponse.error max_size:128
-socket_message.FSListResponse.files max_count:20
-socket_message.FSListResponse.directories max_count:20
-
-# Streaming download messages
-socket_message.FSDownloadRequest.path max_size:256
-socket_message.FSDownloadMetadata.transfer_id max_size:64
-socket_message.FSDownloadMetadata.error max_size:128
-socket_message.FSDownloadData.transfer_id max_size:64
-socket_message.FSDownloadData.data max_size:16384
-socket_message.FSDownloadComplete.transfer_id max_size:64
-socket_message.FSDownloadComplete.error max_size:128
-
-# Streaming upload messages
-socket_message.FSUploadStart.path max_size:256
-socket_message.FSUploadStartResponse.error max_size:128
-socket_message.FSUploadStartResponse.transfer_id max_size:64
-socket_message.FSUploadData.transfer_id max_size:64
-socket_message.FSUploadData.data max_size:16384
-socket_message.FSUploadComplete.transfer_id max_size:64
-socket_message.FSUploadComplete.error max_size:128
-
-# Transfer control
-socket_message.FSCancelTransfer.transfer_id max_size:64
-socket_message.FSCancelTransferResponse.transfer_id max_size:64

platform_shared/message.proto

@@ -2,127 +2,14 @@ syntax = "proto3";
 
 package socket_message;
 
+import "filesystem.proto";
+
 // Partial data types
 message Vector { float x = 1; float y = 2; }
 message I2CDevice { int32 address = 1; string part_number = 2; string name = 3; }
 message PinConfig { int32 pin = 1; string mode = 2; string type = 3; string role = 4; }
 message KnownNetworkItem { string ssid = 1; string password = 2; bool static_ip = 3; uint32 local_ip = 4; uint32 subnet_mask = 5; uint32 gateway_ip = 6; uint32 dns_ip_1 = 7; uint32 dns_ip_2 = 8; }
 
-
-// ----- FILESYSTEM -----
-
-message File {
-    string name = 10;
-    uint32 size = 20;
-}
-
-// Represents a single directory entry (metadata only)
-message Directory {
-    string name = 1;
-}
-
-// Delete a file or directory
-message FSDeleteRequest {
-    string path = 1;
-}
-
-message FSDeleteResponse {
-    bool success = 1;
-    string error = 2;
-}
-
-// Create directory
-message FSMkdirRequest {
-    string path = 1;
-}
-
-message FSMkdirResponse {
-    bool success = 1;
-    string error = 2;
-}
-
-// List files/directories
-message FSListRequest {
-    string path = 1;
-}
-
-message FSListResponse {
-    bool success = 1;
-    string error = 2;
-    repeated File files = 3;
-    repeated Directory directories = 4;
-}
-
-// ===== STREAMING DOWNLOAD (ESP -> Client) =====
-// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete
-
-message FSDownloadRequest {
-    string path = 1;           // File path on ESP to download
-}
-
-message FSDownloadMetadata {
-    string transfer_id = 1;    // Transfer identifier
-    bool success = 2;          // True if file exists and is readable
-    string error = 3;          // Error message if failed
-    uint32 file_size = 4;      // Total file size in bytes
-    uint32 total_chunks = 5;   // Total number of chunks to expect
-}
-
-message FSDownloadData {
-    string transfer_id = 1;    // Transfer identifier
-    uint32 chunk_index = 2;    // Which chunk this is (0-based)
-    bytes data = 3;            // Chunk data (up to 16KB)
-}
-
-message FSDownloadComplete {
-    string transfer_id = 1;
-    bool success = 2;
-    string error = 3;          // Error message if failed
-    uint32 total_chunks = 4;   // Total chunks that were sent
-    uint32 file_size = 5;      // Total file size in bytes
-}
-
-// ===== STREAMING UPLOAD (Client -> ESP) =====
-// Flow: Client sends FSUploadStart -> Server responds with transfer_id -> Client streams FSUploadData chunks -> Server sends FSUploadComplete
-
-message FSUploadStart {
-    string path = 1;           // Destination path on ESP
-    uint32 file_size = 2;      // Total file size in bytes
-    uint32 total_chunks = 3;   // Total number of chunks to expect
-}
-
-message FSUploadStartResponse {
-    bool success = 1;
-    string error = 2;
-    string transfer_id = 3;    // Unique ID for this transfer
-}
-
-message FSUploadData {
-    string transfer_id = 1;    // Transfer identifier
-    uint32 chunk_index = 2;    // Which chunk this is (0-based)
-    bytes data = 3;            // Chunk data (up to 16KB)
-}
-
-message FSUploadComplete {
-    string transfer_id = 1;
-    bool success = 2;
-    string error = 3;          // Error message if failed
-    uint32 chunks_received = 4; // Number of chunks actually received
-}
-
-// ===== TRANSFER CONTROL =====
-
-message FSCancelTransfer {
-    string transfer_id = 1;
-}
-
-message FSCancelTransferResponse {
-    string transfer_id = 1;
-    bool success = 2;
-}
-
-// ----- FILESYSTEM -----
-
 // Individual message data types
 message IMUData {
     float x = 1;