Darkages/SpotMicroESP32-Leika
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moved FS proto objects to own file, and MD tutorial

Browse Source
This commit is contained in:
Niklas Jensen
2026-01-17 21:50:11 +01:00
committed by Rune Harlyk
parent a799af360f
commit 485ecb7547
9 changed files with 311 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/scripts/compile_protos.js
+1 -1
View File
@@ -17,7 +17,7 @@ const pluginPath =
path.join(projectRoot, 'node_modules', '.bin', 'protoc-gen-ts_proto.cmd')
: path.join(projectRoot, 'node_modules', '.bin', 'protoc-gen-ts_proto')
const protoFiles = ['message.proto']
const protoFiles = ['filesystem.proto', 'message.proto']
const tsProtoOpts = ['useExactTypes=false', 'outputExtensions=true', 'outputSchema=true'].join(',')
app/src/lib/filesystem/chunkedTransfer.ts
+7 -7
View File
@@ -1,5 +1,5 @@
import { socket } from '$lib/stores/socket'
import * as Messages from '$lib/platform_shared/message'
import * as FSMessages from '$lib/platform_shared/filesystem'
import type {
FSDeleteRequest,
FSMkdirRequest,
@@ -12,7 +12,7 @@ import type {
FSUploadData,
FSUploadComplete,
FSCancelTransfer
} from '$lib/platform_shared/message'
} from '$lib/platform_shared/filesystem'
const MAX_CHUNK_SIZE = 2 ** 14 // ~= 16 kb
@@ -93,7 +93,7 @@ export class FileSystemClient {
private setupListeners() {
// Listen for download metadata (sent first with file size)
this.metadataListenerCleanup = socket.on(
Messages.FSDownloadMetadata,
FSMessages.FSDownloadMetadata,
(metadata: FSDownloadMetadata) => {
this.handleDownloadMetadata(metadata)
}
@@ -101,7 +101,7 @@ export class FileSystemClient {
// Listen for download data chunks
this.downloadListenerCleanup = socket.on(
Messages.FSDownloadData,
FSMessages.FSDownloadData,
(data: FSDownloadData) => {
this.handleDownloadData(data)
}
@@ -109,7 +109,7 @@ export class FileSystemClient {
// Listen for download completion
this.completeListenerCleanup = socket.on(
Messages.FSDownloadComplete,
FSMessages.FSDownloadComplete,
(complete: FSDownloadComplete) => {
this.handleDownloadComplete(complete)
}
@@ -117,7 +117,7 @@ export class FileSystemClient {
// Listen for upload completion
this.uploadCompleteListenerCleanup = socket.on(
Messages.FSUploadComplete,
FSMessages.FSUploadComplete,
(complete: FSUploadComplete) => {
this.handleUploadComplete(complete)
}
@@ -408,7 +408,7 @@ export class FileSystemClient {
}
// Send chunk as fire-and-forget message
socket.emit(Messages.FSUploadData, uploadData)
socket.emit(FSMessages.FSUploadData, uploadData)
upload.chunksSent++
app/src/lib/stores/socket.ts
+8 -1
View File
@@ -7,6 +7,7 @@ import {
type MessageFns
} from '$lib/platform_shared/message'
import * as Messages from '$lib/platform_shared/message'
import { protoMetadata as filesystemProtoMetadata } from '$lib/platform_shared/filesystem'
export const MESSAGE_TYPE_TO_KEY = new Map<MessageFns<unknown>, string>()
export const MESSAGE_TYPE_TO_TAG = new Map<MessageFns<unknown>, number>()
@@ -20,6 +21,12 @@ type PendingRequest = {
timeoutId: ReturnType<typeof setTimeout>
}
// Combine references from both message.proto and filesystem.proto
const combinedReferences: Record<string, MessageFns<unknown>> = {
...protoMetadata.references,
...filesystemProtoMetadata.references
}
const MessageType = protoMetadata.fileDescriptor.messageType?.find(
(msg: { name: string }) => msg.name === 'Message'
)
@@ -27,7 +34,7 @@ const MessageType = protoMetadata.fileDescriptor.messageType?.find(
if (MessageType?.field) {
for (const field of MessageType.field) {
if (field.typeName) {
const messageFns = protoMetadata.references[field.typeName]
const messageFns = combinedReferences[field.typeName]
if (messageFns && field.jsonName && field.number) {
MESSAGE_TYPE_TO_KEY.set(messageFns, field.jsonName)
MESSAGE_TYPE_TO_TAG.set(messageFns, field.number)
esp32/scripts/compile_protos.py
+1 -1
View File
@@ -34,7 +34,7 @@ def compile_nanopb():
output_dir.mkdir(parents=True, exist_ok=True)
proto_files = [proto_dir / "message.proto"]
proto_files = [proto_dir / "filesystem.proto", proto_dir / "message.proto"]
print(f"Compiling protobuf files with nanopb...")
print(f" Proto dir: {proto_dir}")
platform_shared/ADDING_PROTO_FILES.md
+140
View File
@@ -0,0 +1,140 @@
# Adding New Proto Files
## Step-by-Step Guide
### 1. Create the new .proto file
Create `platform_shared/myfeature.proto`:
```protobuf
syntax = "proto3";
package socket_message;
message MyFeatureRequest {
string name = 1;
}
message MyFeatureResponse {
bool success = 1;
string error = 2;
}
```
### 2. Create the .options file (for nanopb size constraints)
Create `platform_shared/myfeature.options`:
```
socket_message.MyFeatureRequest.name max_size:64
socket_message.MyFeatureResponse.error max_size:128
```
### 3. Import in message.proto
Add the import at the top of `platform_shared/message.proto`:
```protobuf
import "myfeature.proto";
```
### 4. Add to Message oneof (if needed for streaming/pub-sub)
If your message needs to be sent directly (not via correlation request/response), add it to the `Message` oneof in `message.proto`:
```protobuf
message Message {
oneof message {
// ... existing fields ...
MyFeatureRequest my_feature_request = 300; // Pick unused tag number
MyFeatureResponse my_feature_response = 301;
}
}
```
### 5. Add to CorrelationRequest/Response (if using request/response pattern)
For request/response messages, add to the correlation oneofs in `message.proto`:
```protobuf
message CorrelationRequest {
oneof request {
// ... existing fields ...
MyFeatureRequest my_feature_request = 200; // Pick unused tag number
}
}
message CorrelationResponse {
oneof response {
// ... existing fields ...
MyFeatureResponse my_feature_response = 200;
}
}
```
### 6. Update compile scripts
**ESP32 (esp32/scripts/compile_protos.py):**
```python
proto_files = [proto_dir / "filesystem.proto", proto_dir / "myfeature.proto", proto_dir / "message.proto"]
```
**TypeScript (app/scripts/compile_protos.js):**
```javascript
const protoFiles = ['filesystem.proto', 'myfeature.proto', 'message.proto']
```
### 7. Update socket.ts (for TypeScript - only if added to Message oneof)
If you added messages to the `Message` oneof, import the protoMetadata in `app/src/lib/stores/socket.ts`:
```typescript
import { protoMetadata as myfeatureProtoMetadata } from '$lib/platform_shared/myfeature'
// Add to combinedReferences
const combinedReferences: Record<string, MessageFns<unknown>> = {
...protoMetadata.references,
...filesystemProtoMetadata.references,
...myfeatureProtoMetadata.references // Add this
}
```
### 8. Add MessageTraits (for ESP32 - only if added to Message oneof)
If you added messages to the `Message` oneof, add traits in `esp32/include/communication/proto_helpers.h`:
```cpp
// Before #undef DEFINE_MESSAGE_TRAITS
DEFINE_MESSAGE_TRAITS(MyFeatureRequest, my_feature_request)
DEFINE_MESSAGE_TRAITS(MyFeatureResponse, my_feature_response)
```
### 9. Build and test
```bash
# ESP32
pio run
# TypeScript
cd app && pnpm proto
```
## Quick Reference
| File | Purpose |
|------|---------|
| `platform_shared/*.proto` | Protocol definitions |
| `platform_shared/*.options` | Nanopb size constraints |
| `esp32/scripts/compile_protos.py` | ESP32 proto compilation |
| `app/scripts/compile_protos.js` | TypeScript proto compilation |
| `app/src/lib/stores/socket.ts` | Tag mapping for socket.on/emit |
| `esp32/include/communication/proto_helpers.h` | MessageTraits for ESP32 emit |
## Notes
- Messages in `CorrelationRequest/Response` don't need MessageTraits or socket.ts updates
- Messages in `Message` oneof (for streaming/pub-sub) need both
- Always use the same `package socket_message;` in all proto files
- Tag numbers in oneofs must be unique across all fields
platform_shared/filesystem.options
+36
View File
@@ -0,0 +1,36 @@
# Filesystem message options
socket_message.File.name max_size:256
socket_message.Directory.name max_size:256
socket_message.FSDeleteRequest.path max_size:256
socket_message.FSDeleteResponse.error max_size:128
socket_message.FSMkdirRequest.path max_size:256
socket_message.FSMkdirResponse.error max_size:128
socket_message.FSListRequest.path max_size:256
socket_message.FSListResponse.error max_size:128
socket_message.FSListResponse.files max_count:20
socket_message.FSListResponse.directories max_count:20
# Streaming download messages
socket_message.FSDownloadRequest.path max_size:256
socket_message.FSDownloadMetadata.transfer_id max_size:64
socket_message.FSDownloadMetadata.error max_size:128
socket_message.FSDownloadData.transfer_id max_size:64
socket_message.FSDownloadData.data max_size:16384
socket_message.FSDownloadComplete.transfer_id max_size:64
socket_message.FSDownloadComplete.error max_size:128
# Streaming upload messages
socket_message.FSUploadStart.path max_size:256
socket_message.FSUploadStartResponse.error max_size:128
socket_message.FSUploadStartResponse.transfer_id max_size:64
socket_message.FSUploadData.transfer_id max_size:64
socket_message.FSUploadData.data max_size:16384
socket_message.FSUploadComplete.transfer_id max_size:64
socket_message.FSUploadComplete.error max_size:128
# Transfer control
socket_message.FSCancelTransfer.transfer_id max_size:64
socket_message.FSCancelTransferResponse.transfer_id max_size:64
platform_shared/filesystem.proto
+115
View File
@@ -0,0 +1,115 @@
syntax = "proto3";
package socket_message;
// ----- FILESYSTEM -----
message File {
string name = 10;
uint32 size = 20;
}
// Represents a single directory entry (metadata only)
message Directory {
string name = 1;
}
// Delete a file or directory
message FSDeleteRequest {
string path = 1;
}
message FSDeleteResponse {
bool success = 1;
string error = 2;
}
// Create directory
message FSMkdirRequest {
string path = 1;
}
message FSMkdirResponse {
bool success = 1;
string error = 2;
}
// List files/directories
message FSListRequest {
string path = 1;
}
message FSListResponse {
bool success = 1;
string error = 2;
repeated File files = 3;
repeated Directory directories = 4;
}
// ===== STREAMING DOWNLOAD (ESP -> Client) =====
// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete
message FSDownloadRequest {
string path = 1; // File path on ESP to download
}
message FSDownloadMetadata {
string transfer_id = 1; // Transfer identifier
bool success = 2; // True if file exists and is readable
string error = 3; // Error message if failed
uint32 file_size = 4; // Total file size in bytes
uint32 total_chunks = 5; // Total number of chunks to expect
}
message FSDownloadData {
string transfer_id = 1; // Transfer identifier
uint32 chunk_index = 2; // Which chunk this is (0-based)
bytes data = 3; // Chunk data (up to 16KB)
}
message FSDownloadComplete {
string transfer_id = 1;
bool success = 2;
string error = 3; // Error message if failed
uint32 total_chunks = 4; // Total chunks that were sent
uint32 file_size = 5; // Total file size in bytes
}
// ===== STREAMING UPLOAD (Client -> ESP) =====
// Flow: Client sends FSUploadStart -> Server responds with transfer_id -> Client streams FSUploadData chunks -> Server sends FSUploadComplete
message FSUploadStart {
string path = 1; // Destination path on ESP
uint32 file_size = 2; // Total file size in bytes
uint32 total_chunks = 3; // Total number of chunks to expect
}
message FSUploadStartResponse {
bool success = 1;
string error = 2;
string transfer_id = 3; // Unique ID for this transfer
}
message FSUploadData {
string transfer_id = 1; // Transfer identifier
uint32 chunk_index = 2; // Which chunk this is (0-based)
bytes data = 3; // Chunk data (up to 16KB)
}
message FSUploadComplete {
string transfer_id = 1;
bool success = 2;
string error = 3; // Error message if failed
uint32 chunks_received = 4; // Number of chunks actually received
}
// ===== TRANSFER CONTROL =====
message FSCancelTransfer {
string transfer_id = 1;
}
message FSCancelTransferResponse {
string transfer_id = 1;
bool success = 2;
}
platform_shared/message.options
+1 -38
View File
@@ -37,41 +37,4 @@ socket_message.SonarData.dummy_field max_size:16
socket_message.FeaturesDataResponse.variant type:FT_POINTER
socket_message.FeaturesDataResponse.firmware_built_target type:FT_POINTER
socket_message.FeaturesDataResponse.firmware_name type:FT_POINTER
socket_message.FeaturesDataResponse.firmware_version type:FT_POINTER
# Filesystem message options
socket_message.File.name max_size:256
socket_message.Directory.name max_size:256
socket_message.FSDeleteRequest.path max_size:256
socket_message.FSDeleteResponse.error max_size:128
socket_message.FSMkdirRequest.path max_size:256
socket_message.FSMkdirResponse.error max_size:128
socket_message.FSListRequest.path max_size:256
socket_message.FSListResponse.error max_size:128
socket_message.FSListResponse.files max_count:20
socket_message.FSListResponse.directories max_count:20
# Streaming download messages
socket_message.FSDownloadRequest.path max_size:256
socket_message.FSDownloadMetadata.transfer_id max_size:64
socket_message.FSDownloadMetadata.error max_size:128
socket_message.FSDownloadData.transfer_id max_size:64
socket_message.FSDownloadData.data max_size:16384
socket_message.FSDownloadComplete.transfer_id max_size:64
socket_message.FSDownloadComplete.error max_size:128
# Streaming upload messages
socket_message.FSUploadStart.path max_size:256
socket_message.FSUploadStartResponse.error max_size:128
socket_message.FSUploadStartResponse.transfer_id max_size:64
socket_message.FSUploadData.transfer_id max_size:64
socket_message.FSUploadData.data max_size:16384
socket_message.FSUploadComplete.transfer_id max_size:64
socket_message.FSUploadComplete.error max_size:128
# Transfer control
socket_message.FSCancelTransfer.transfer_id max_size:64
socket_message.FSCancelTransferResponse.transfer_id max_size:64
socket_message.FeaturesDataResponse.firmware_version type:FT_POINTER
platform_shared/message.proto
+2 -115
View File
@@ -2,127 +2,14 @@ syntax = "proto3";
package socket_message;
import "filesystem.proto";
// Partial data types
message Vector { float x = 1; float y = 2; }
message I2CDevice { int32 address = 1; string part_number = 2; string name = 3; }
message PinConfig { int32 pin = 1; string mode = 2; string type = 3; string role = 4; }
message KnownNetworkItem { string ssid = 1; string password = 2; bool static_ip = 3; uint32 local_ip = 4; uint32 subnet_mask = 5; uint32 gateway_ip = 6; uint32 dns_ip_1 = 7; uint32 dns_ip_2 = 8; }
// ----- FILESYSTEM -----
message File {
string name = 10;
uint32 size = 20;
}
// Represents a single directory entry (metadata only)
message Directory {
string name = 1;
}
// Delete a file or directory
message FSDeleteRequest {
string path = 1;
}
message FSDeleteResponse {
bool success = 1;
string error = 2;
}
// Create directory
message FSMkdirRequest {
string path = 1;
}
message FSMkdirResponse {
bool success = 1;
string error = 2;
}
// List files/directories
message FSListRequest {
string path = 1;
}
message FSListResponse {
bool success = 1;
string error = 2;
repeated File files = 3;
repeated Directory directories = 4;
}
// ===== STREAMING DOWNLOAD (ESP -> Client) =====
// Flow: Client sends FSDownloadRequest -> Server sends FSDownloadMetadata -> Server streams FSDownloadData chunks -> Server sends FSDownloadComplete
message FSDownloadRequest {
string path = 1; // File path on ESP to download
}
message FSDownloadMetadata {
string transfer_id = 1; // Transfer identifier
bool success = 2; // True if file exists and is readable
string error = 3; // Error message if failed
uint32 file_size = 4; // Total file size in bytes
uint32 total_chunks = 5; // Total number of chunks to expect
}
message FSDownloadData {
string transfer_id = 1; // Transfer identifier
uint32 chunk_index = 2; // Which chunk this is (0-based)
bytes data = 3; // Chunk data (up to 16KB)
}
message FSDownloadComplete {
string transfer_id = 1;
bool success = 2;
string error = 3; // Error message if failed
uint32 total_chunks = 4; // Total chunks that were sent
uint32 file_size = 5; // Total file size in bytes
}
// ===== STREAMING UPLOAD (Client -> ESP) =====
// Flow: Client sends FSUploadStart -> Server responds with transfer_id -> Client streams FSUploadData chunks -> Server sends FSUploadComplete
message FSUploadStart {
string path = 1; // Destination path on ESP
uint32 file_size = 2; // Total file size in bytes
uint32 total_chunks = 3; // Total number of chunks to expect
}
message FSUploadStartResponse {
bool success = 1;
string error = 2;
string transfer_id = 3; // Unique ID for this transfer
}
message FSUploadData {
string transfer_id = 1; // Transfer identifier
uint32 chunk_index = 2; // Which chunk this is (0-based)
bytes data = 3; // Chunk data (up to 16KB)
}
message FSUploadComplete {
string transfer_id = 1;
bool success = 2;
string error = 3; // Error message if failed
uint32 chunks_received = 4; // Number of chunks actually received
}
// ===== TRANSFER CONTROL =====
message FSCancelTransfer {
string transfer_id = 1;
}
message FSCancelTransferResponse {
string transfer_id = 1;
bool success = 2;
}
// ----- FILESYSTEM -----
// Individual message data types
message IMUData {
float x = 1;