From 94cfc9ca9d953160dbe2541a068f93dd4eb087e0 Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Tue, 15 Oct 2024 16:23:44 +0200 Subject: [PATCH 1/9] Add protobufs for media events --- fishjam/media_events/client.proto | 136 ++++++++++++++++++++ fishjam/media_events/server.proto | 203 ++++++++++++++++++++++++++++++ fishjam/media_events/shared.proto | 24 ++++ 3 files changed, 363 insertions(+) create mode 100644 fishjam/media_events/client.proto create mode 100644 fishjam/media_events/server.proto create mode 100644 fishjam/media_events/shared.proto diff --git a/fishjam/media_events/client.proto b/fishjam/media_events/client.proto new file mode 100644 index 0000000..8d2d2e7 --- /dev/null +++ b/fishjam/media_events/client.proto @@ -0,0 +1,136 @@ +syntax = "proto3"; + +package media_events.client; + +import "fishjam/media_events/shared.proto"; + +// Defines any type of message sent between FJ and a peer +message MediaEvent { + message VariantBitrates { + media_events.shared.Encoding rid = 1; + int32 bitrate = 2; + } + + // Client -> WebRTC Endpoint + + // Sent when a client wants to join WebRTC Endpoint. It contains only endpoint's metadata + message Connect { + string metadata = 1; + } + + // Sent when client disconnects from WebRTC Endpoint. It contains no data + message Disconnect {}; + + // Contains new metadata of client's endpoint + message UpdateEndpointMetadata { + string metadata = 1; + } + + // Sent when client disables one of the track variants + message DisableTrackEncoding { + string track_id = 1; + media_events.shared.Encoding encoding = 2; + } + + // Sent when client enables one of the track variants + message EnableTrackEncoding { + string track_id = 1; + media_events.shared.Encoding encoding = 2; + } + + // Contains new metadata of some track and an id of this track + message UpdateTrackMetadata { + string track_id = 1; + string track_metadata = 2; + } + + // Sent when a track is muted + message MuteTrack { + string track_id = 1; + } + + + // Sent when a track is unmuted + message UnmuteTrack { + string track_id = 1; + } + + // Client -> WebRTC Enpoint + + // Informs that a client wants to renegotiate connection due to adding a track or removing a track + message RenegotiateTracks {} + + // Contains an id of the track that the client wants to prioritize + message PrioritizeTrack { + string track_id = 1; + } + + // Contains an id of the track that the client want to unprioritize. + message UnprioritizeTrack { + string track_id = 1; + } + + // Contains information of how many videos in different quality (high, medium, small) the client would like to receive. + // `allSameSize` flag which indicates that all videos should be in the same quality. + message PreferedVideoSizes { + int32 big_screens = 1; + int32 medium_screens = 2; + int32 small_screens = 3; + bool all_same_size = 4; + } + + // Contains an SDP offer, a mapping between `track_id` and `track_metadata`, mapping between `track_id` + // and this track's bitrates (or all of its variants bitrates) and a mapping between `mid` and `track_id`. + // Maps contain only information about current client's `sendonly` tracks. + message SdpOffer { + message TrackIdToMetadata { + string id = 1; + media_events.shared.Metadata metadata = 2; + } + + message TrackIdToTrackBitrates { + string id = 1; + repeated VariantBitrates bitrates = 2; + } + + string sdp = 1; + repeated TrackIdToMetadata trackIdToTrackMetadata = 2; + repeated TrackIdToTrackBitrates trackIdToTrackBitrates = 3; + repeated media_events.shared.MidToTrackId mid_to_track_id = 4; + } + + // Informs that a client wants to receive a specific track variant. + // The track variant will be sent whenever it is available. + // If choosen variant is unavailable, some other variant will be + // sent until choosen variant becomes active again. + message SetTargetTrackVariant { + string track_id = 1; + media_events.shared.Encoding variant = 2; // TODO: should be encoding + } + + // Contains updated bitrates of variants of the track send by the client. + // Needs to contain all of the variants. + message TrackVariantBitrates { + string track_id = 1; + repeated VariantBitrates variantBitrates = 2; + } + + oneof content { + Connect connect = 1; + Disconnect disconnect = 2; + UpdateEndpointMetadata update_endpoint_metadata = 3; + DisableTrackEncoding disable_track_encoding = 4; + EnableTrackEncoding enable_track_encoding = 5; + UpdateTrackMetadata udpate_track_metadtata = 6; + MuteTrack mute_track = 7; + UnmuteTrack unmute_track = 8; + RenegotiateTracks renegotiate_tracks = 9; + PrioritizeTrack prioritize_track = 10; + UnprioritizeTrack unprioritize_track = 11; + PreferedVideoSizes prefered_video_sizes = 12; + media_events.shared.Candidate candidate = 13; + SdpOffer sdp_offer = 14; + SetTargetTrackVariant set_target_track_variant = 15; + TrackVariantBitrates track_variant_bitrates = 16; + } +} \ No newline at end of file diff --git a/fishjam/media_events/server.proto b/fishjam/media_events/server.proto new file mode 100644 index 0000000..0bbe946 --- /dev/null +++ b/fishjam/media_events/server.proto @@ -0,0 +1,203 @@ +syntax = "proto3"; + +package webrtc; + +import "fishjam/media_events/shared.proto"; + +// Defines any type of message sent between FJ and a peer +message MediaEvent { + // WebRTC Endpoint -> Client + + // Contains all tracks that will be forwarded to client until the next `tracks_priority` message. + // This type of messages is sent only if DisplayManager is spawned + message TracksPriority { + repeated string tracks = 1; + } + + // Contains information about updated metadata of one of the endpoints + message EndpointUpdated { + string endpoint_id = 1; + string metadata = 2; + } + + // Contains information about new metadata of one of tracks + message TrackUpdated { + string endpoint_id = 1; + string track_id = 2; + string metadata = 3; + } + + + // Informs that one of the clients has added one or more tracks. + // It contains: + // - an id of endpoint associated with that client, + // - a map of all tracks with `track_id`'s as keys and objects with `track_metadata` and simulcast conifg as a value + // - (Depracated field use tracks) a map of all tracks with `track_id`s as keys and `track_metadata` as value. + message TracksAdded { + message Track { + message SimulcastConfig { + bool enabled = 1; + repeated string active_encodings = 2; + repeated string disabled_encodings = 3; + } + + string track_id = 1; + string metadata = 2; + string simulcast_config = 3; + } + + string endpoint_id = 1; + repeated Track tracks = 2; + } + + // Contains a list of tracks which have been removed by some client and id of that client's + message TracksRemoved { + string endpoint_id = 1; + repeated string track_ids = 2; + } + + // Sent when some track variant were disabled by an endpoint + message TrackEncodingDisabled { + string track_id = 1; + media_events.shared.Encoding encoding = 2; + } + + // Sent when some track variant were enabled by an endpoint + message TrackEncodingEnabled { + string track_id = 1; + media_events.shared.Encoding encoding = 2; + } + + // Message sent to all clients in the room after a new endpoint was added. + // It contains id and metadata of the new endpoint. + message EndpointAdded { + string id = 1; + string metadata = 2; + } + + // Message sent to the client after connecting to the WebRTC Endpoint. + // It contains the id of that client's endpoint and a list of information about endpoints in the Engine + // (id, metadata, a `trackIdToMetadata` and tracks like seen in `tracksAdded`) + message Connected { + message SimulcastConfig { + bool enabled = 1; + repeated media_events.shared.Encoding active_encodings = 2; + repeated media_events.shared.Encoding disabled_encodings = 3; + } + + message Track { + string id = 1; + media_events.shared.Metadata metadata = 2; + SimulcastConfig SimulcastConfig = 3; + } + + message Endpoint { + string id = 1; + string type = 2; + media_events.shared.Metadata metadata = 3; + // deprecated field + // track_id_to_metadata + repeated Track tracks = 4; + } + string id = 1; + repeated Endpoint endpoints = 2; + } + + // Sent to all remaining clients in the room after some endpoint was removed. It contains an id of the removed endpoint. + message EndpointRemoved { + string id = 1; + } + + // Informs that an error occurred on the server providing a message to show + message Error { + string message = 1; + } + + // WebRTC Enpoint -> Client + + // Contains information about the number of audio and video tracks that will be sent from the engine + // to the client and information regarding the integrated TURN server. + message OfferData { + message TrackTypes { + int32 audio = 1; + int32 video = 2; + } + + message IntegratedTurnServers { + string server_addr = 1; + int32 server_port = 2; + string transport = 3; + string password = 4; + string username = 5; + } + + TrackTypes tracks_types = 1; + repeated IntegratedTurnServers integrated_turn_servers = 2; + } + + + // Contains an SDP answer and mapping between `mid` and `track_id` for all tracks (active, inactive, inbound and outbound) + message SdpAnswer { + string sdp_answer = 1; + repeated media_events.shared.MidToTrackId midToTrackId = 2; + } + + // Informs that track with id `trackId` belonging to endpoint with id `endpointId` will be sent in encoding `encoding` now. + // The meaning is as follows: + // "low_bandwidth" - we no longer have enough bandwidth to maintain current track quality + // "encoding_inactive" - encoding became inactive + // "other" - it was hard to determine exact reason of encoding switch + message EncodingSwitched { + enum Reason { + LOW_BANDWIDTH = 0; + ENCODING_INACTIVE = 1; + OTHER = 2; + } + + string endpoint_id = 1; + string track_id = 2; + media_events.shared.Encoding encoding = 3; + Reason reason = 4; + } + + // Informs that the track denoted by `trackId` has changed their voice actiivty + // For this notification to work, the server must be configured to use VAD extension + // and the sender must support it. + message VadNotification { + enum Status { + SILENCE = 0; + SPEECH = 1; + } + + string track_id = 1; + Status status = 2; + } + + // Informs about client's available incoming bitrate (a.k.a. download) estimated by the server. + // It is measured in bits per second. + // For this notification to work, the server must be configured to use TWCC extension + // and the sender must support it. + message BandwidthEstimation { + int32 estimation = 1; + } + + oneof content { + TracksPriority tracks_priority = 1; + EndpointUpdated endpoint_updated = 2; + TrackUpdated track_updated = 3; + TracksAdded tracks_added = 4; + TracksRemoved tracks_removed = 5; + TrackEncodingDisabled track_encoding_disabled = 6; + TrackEncodingEnabled track_encoding_enabled = 7; + EndpointAdded endpoint_added = 8; + EndpointRemoved endpoint_removed = 9; + Connected connected = 10; + Error error = 11; + OfferData offer_data = 12; + media_events.shared.Candidate candidate = 13; + SdpAnswer sdp_answer = 14; + EncodingSwitched encoding_switched = 15; + VadNotification vad_notification = 16; + BandwidthEstimation bandwidth_estimation = 17; + } +} \ No newline at end of file diff --git a/fishjam/media_events/shared.proto b/fishjam/media_events/shared.proto new file mode 100644 index 0000000..164a162 --- /dev/null +++ b/fishjam/media_events/shared.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package media_events.shared; + +enum Encoding { + LOW = 0; + MEDIUM = 1; + HIGH = 2; +} + +message Metadata { + string json = 1; +} + +message MidToTrackId { + string mid = 1; + string track_id = 2; +} + +// Contains information about an ICE candidate which will be sent to the client/server +message Candidate { + string candidate = 1; + int32 sdp_m_line_index = 2; +} \ No newline at end of file From 424d693915b62ab1e609f25919ec3c3e5606f3a2 Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Tue, 15 Oct 2024 16:29:23 +0200 Subject: [PATCH 2/9] Format media events --- fishjam/media_events/client.proto | 13 ++++++------- fishjam/media_events/server.proto | 20 +++++++++----------- fishjam/media_events/shared.proto | 6 +++--- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/fishjam/media_events/client.proto b/fishjam/media_events/client.proto index 8d2d2e7..f5ef82d 100644 --- a/fishjam/media_events/client.proto +++ b/fishjam/media_events/client.proto @@ -10,7 +10,7 @@ message MediaEvent { media_events.shared.Encoding rid = 1; int32 bitrate = 2; } - + // Client -> WebRTC Endpoint // Sent when a client wants to join WebRTC Endpoint. It contains only endpoint's metadata @@ -19,13 +19,13 @@ message MediaEvent { } // Sent when client disconnects from WebRTC Endpoint. It contains no data - message Disconnect {}; + message Disconnect {} // Contains new metadata of client's endpoint message UpdateEndpointMetadata { string metadata = 1; } - + // Sent when client disables one of the track variants message DisableTrackEncoding { string track_id = 1; @@ -49,7 +49,6 @@ message MediaEvent { string track_id = 1; } - // Sent when a track is unmuted message UnmuteTrack { string track_id = 1; @@ -98,7 +97,7 @@ message MediaEvent { repeated TrackIdToTrackBitrates trackIdToTrackBitrates = 3; repeated media_events.shared.MidToTrackId mid_to_track_id = 4; } - + // Informs that a client wants to receive a specific track variant. // The track variant will be sent whenever it is available. // If choosen variant is unavailable, some other variant will be @@ -107,7 +106,7 @@ message MediaEvent { string track_id = 1; media_events.shared.Encoding variant = 2; // TODO: should be encoding } - + // Contains updated bitrates of variants of the track send by the client. // Needs to contain all of the variants. message TrackVariantBitrates { @@ -133,4 +132,4 @@ message MediaEvent { SetTargetTrackVariant set_target_track_variant = 15; TrackVariantBitrates track_variant_bitrates = 16; } -} \ No newline at end of file +} diff --git a/fishjam/media_events/server.proto b/fishjam/media_events/server.proto index 0bbe946..c1ead79 100644 --- a/fishjam/media_events/server.proto +++ b/fishjam/media_events/server.proto @@ -26,19 +26,18 @@ message MediaEvent { string track_id = 2; string metadata = 3; } - // Informs that one of the clients has added one or more tracks. // It contains: - // - an id of endpoint associated with that client, - // - a map of all tracks with `track_id`'s as keys and objects with `track_metadata` and simulcast conifg as a value + // - an id of endpoint associated with that client, + // - a map of all tracks with `track_id`'s as keys and objects with `track_metadata` and simulcast conifg as a value // - (Depracated field use tracks) a map of all tracks with `track_id`s as keys and `track_metadata` as value. message TracksAdded { message Track { message SimulcastConfig { bool enabled = 1; repeated string active_encodings = 2; - repeated string disabled_encodings = 3; + repeated string disabled_encodings = 3; } string track_id = 1; @@ -50,7 +49,7 @@ message MediaEvent { repeated Track tracks = 2; } - // Contains a list of tracks which have been removed by some client and id of that client's + // Contains a list of tracks which have been removed by some client and id of that client's message TracksRemoved { string endpoint_id = 1; repeated string track_ids = 2; @@ -75,13 +74,13 @@ message MediaEvent { string metadata = 2; } - // Message sent to the client after connecting to the WebRTC Endpoint. + // Message sent to the client after connecting to the WebRTC Endpoint. // It contains the id of that client's endpoint and a list of information about endpoints in the Engine // (id, metadata, a `trackIdToMetadata` and tracks like seen in `tracksAdded`) message Connected { message SimulcastConfig { bool enabled = 1; - repeated media_events.shared.Encoding active_encodings = 2; + repeated media_events.shared.Encoding active_encodings = 2; repeated media_events.shared.Encoding disabled_encodings = 3; } @@ -135,13 +134,12 @@ message MediaEvent { repeated IntegratedTurnServers integrated_turn_servers = 2; } - // Contains an SDP answer and mapping between `mid` and `track_id` for all tracks (active, inactive, inbound and outbound) message SdpAnswer { string sdp_answer = 1; repeated media_events.shared.MidToTrackId midToTrackId = 2; } - + // Informs that track with id `trackId` belonging to endpoint with id `endpointId` will be sent in encoding `encoding` now. // The meaning is as follows: // "low_bandwidth" - we no longer have enough bandwidth to maintain current track quality @@ -177,7 +175,7 @@ message MediaEvent { // It is measured in bits per second. // For this notification to work, the server must be configured to use TWCC extension // and the sender must support it. - message BandwidthEstimation { + message BandwidthEstimation { int32 estimation = 1; } @@ -200,4 +198,4 @@ message MediaEvent { VadNotification vad_notification = 16; BandwidthEstimation bandwidth_estimation = 17; } -} \ No newline at end of file +} diff --git a/fishjam/media_events/shared.proto b/fishjam/media_events/shared.proto index 164a162..f18aab1 100644 --- a/fishjam/media_events/shared.proto +++ b/fishjam/media_events/shared.proto @@ -19,6 +19,6 @@ message MidToTrackId { // Contains information about an ICE candidate which will be sent to the client/server message Candidate { - string candidate = 1; - int32 sdp_m_line_index = 2; -} \ No newline at end of file + string candidate = 1; + int32 sdp_m_line_index = 2; +} From 973189ba9d4fbbd7e904cda3eecfd309b085e626 Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Wed, 16 Oct 2024 16:40:59 +0200 Subject: [PATCH 3/9] Add ADR document open for discussion --- fishjam/media_events/ADR.md | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 fishjam/media_events/ADR.md diff --git a/fishjam/media_events/ADR.md b/fishjam/media_events/ADR.md new file mode 100644 index 0000000..84d652b --- /dev/null +++ b/fishjam/media_events/ADR.md @@ -0,0 +1,51 @@ +# Media Events defined as protobufs + +## Why ? +Currently, all communication between the server/client and Fishjam is clearly defined using protobufs or openapi (REST API), with the exception of media events. +Media events are currently sent using JSON encoding, which doesn't enforce or document the structure of specific events, leading to unnecessary errors in the implementation of client SDKs. +It would be beneficial to have a protobufs documentation that includes comprehensive details about media events and their specific structure and provide event parsers for SDKs and Fishjam to streamline the process. + +## How to Ensure Backward Compatibility + +### 1. Send Both Old and New Media Events +* Client SDKs would have to ignore media events that are not compatible with their version. +* The engine would have to handle/send both types of media events. + + +### 2. Differentiating Between SDKs +Fishjam can differentiate between SDKs by creating a WebRTC endpoint with a configuration that specifies what kind of media events should be handled/sent based on the client SDK version. + +* There is no need to change the client SDK. +* Fishjam has to recognize the client version and create adequate endpoint configuration. +* The engine must allow the choice of media event types. + +## Required Changes +The migration of media events to protobufs requires changes in: + +* Fishjam + * Parsing protobufs - as it sometimes changes something in events (e.g., peer metadata). + * Changes related to backward compatibility. + +* Engine + * Definition of protobufs - the engine is open source, so protobufs configuration has to be available within its repository. + * Changes should be made only to the ExWebRTC endpoint. + * Changes related to backward compatibility. + +* Web SDKs + * Parsing protobufs instead of JSON. + * Optional changes related to backward compatibility. + +* Mobile SDKs + * Parsing protobufs instead of JSON. + * Optional changes related to backward compatibility. + +Backward compatibility for SDKs is only needed with the first option.​ + +## Implementation (In Order) + +1. Implementing protobufs in Engine +2. Implementing protbufs in Fishjam +3. Implementing backward compatibility in Fishjam +4. Ensuring that Fishjam works with current SDKs +5. Support for protobufs in Web Client SDKs +6. Support for protobufs in Mobile Client SDKs \ No newline at end of file From 8eaa618b995f47e9775f7524334c3a8ed34e3b0d Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Mon, 28 Oct 2024 16:55:54 +0100 Subject: [PATCH 4/9] Add protobufs for media events (no simulcast) --- fishjam/media_events/ADR.md | 51 --------- fishjam/media_events/client.proto | 135 ------------------------ fishjam/media_events/peer.proto | 83 +++++++++++++++ fishjam/media_events/server.proto | 159 ++++++++--------------------- fishjam/media_events/shared.proto | 4 +- fishjam/peer_notifications.proto | 13 ++- fishjam/server_notifications.proto | 8 +- 7 files changed, 140 insertions(+), 313 deletions(-) delete mode 100644 fishjam/media_events/ADR.md delete mode 100644 fishjam/media_events/client.proto create mode 100644 fishjam/media_events/peer.proto diff --git a/fishjam/media_events/ADR.md b/fishjam/media_events/ADR.md deleted file mode 100644 index 84d652b..0000000 --- a/fishjam/media_events/ADR.md +++ /dev/null @@ -1,51 +0,0 @@ -# Media Events defined as protobufs - -## Why ? -Currently, all communication between the server/client and Fishjam is clearly defined using protobufs or openapi (REST API), with the exception of media events. -Media events are currently sent using JSON encoding, which doesn't enforce or document the structure of specific events, leading to unnecessary errors in the implementation of client SDKs. -It would be beneficial to have a protobufs documentation that includes comprehensive details about media events and their specific structure and provide event parsers for SDKs and Fishjam to streamline the process. - -## How to Ensure Backward Compatibility - -### 1. Send Both Old and New Media Events -* Client SDKs would have to ignore media events that are not compatible with their version. -* The engine would have to handle/send both types of media events. - - -### 2. Differentiating Between SDKs -Fishjam can differentiate between SDKs by creating a WebRTC endpoint with a configuration that specifies what kind of media events should be handled/sent based on the client SDK version. - -* There is no need to change the client SDK. -* Fishjam has to recognize the client version and create adequate endpoint configuration. -* The engine must allow the choice of media event types. - -## Required Changes -The migration of media events to protobufs requires changes in: - -* Fishjam - * Parsing protobufs - as it sometimes changes something in events (e.g., peer metadata). - * Changes related to backward compatibility. - -* Engine - * Definition of protobufs - the engine is open source, so protobufs configuration has to be available within its repository. - * Changes should be made only to the ExWebRTC endpoint. - * Changes related to backward compatibility. - -* Web SDKs - * Parsing protobufs instead of JSON. - * Optional changes related to backward compatibility. - -* Mobile SDKs - * Parsing protobufs instead of JSON. - * Optional changes related to backward compatibility. - -Backward compatibility for SDKs is only needed with the first option.​ - -## Implementation (In Order) - -1. Implementing protobufs in Engine -2. Implementing protbufs in Fishjam -3. Implementing backward compatibility in Fishjam -4. Ensuring that Fishjam works with current SDKs -5. Support for protobufs in Web Client SDKs -6. Support for protobufs in Mobile Client SDKs \ No newline at end of file diff --git a/fishjam/media_events/client.proto b/fishjam/media_events/client.proto deleted file mode 100644 index f5ef82d..0000000 --- a/fishjam/media_events/client.proto +++ /dev/null @@ -1,135 +0,0 @@ -syntax = "proto3"; - -package media_events.client; - -import "fishjam/media_events/shared.proto"; - -// Defines any type of message sent between FJ and a peer -message MediaEvent { - message VariantBitrates { - media_events.shared.Encoding rid = 1; - int32 bitrate = 2; - } - - // Client -> WebRTC Endpoint - - // Sent when a client wants to join WebRTC Endpoint. It contains only endpoint's metadata - message Connect { - string metadata = 1; - } - - // Sent when client disconnects from WebRTC Endpoint. It contains no data - message Disconnect {} - - // Contains new metadata of client's endpoint - message UpdateEndpointMetadata { - string metadata = 1; - } - - // Sent when client disables one of the track variants - message DisableTrackEncoding { - string track_id = 1; - media_events.shared.Encoding encoding = 2; - } - - // Sent when client enables one of the track variants - message EnableTrackEncoding { - string track_id = 1; - media_events.shared.Encoding encoding = 2; - } - - // Contains new metadata of some track and an id of this track - message UpdateTrackMetadata { - string track_id = 1; - string track_metadata = 2; - } - - // Sent when a track is muted - message MuteTrack { - string track_id = 1; - } - - // Sent when a track is unmuted - message UnmuteTrack { - string track_id = 1; - } - - // Client -> WebRTC Enpoint - - // Informs that a client wants to renegotiate connection due to adding a track or removing a track - message RenegotiateTracks {} - - // Contains an id of the track that the client wants to prioritize - message PrioritizeTrack { - string track_id = 1; - } - - // Contains an id of the track that the client want to unprioritize. - message UnprioritizeTrack { - string track_id = 1; - } - - // Contains information of how many videos in different quality (high, medium, small) the client would like to receive. - // `allSameSize` flag which indicates that all videos should be in the same quality. - message PreferedVideoSizes { - int32 big_screens = 1; - int32 medium_screens = 2; - int32 small_screens = 3; - bool all_same_size = 4; - } - - // Contains an SDP offer, a mapping between `track_id` and `track_metadata`, mapping between `track_id` - // and this track's bitrates (or all of its variants bitrates) and a mapping between `mid` and `track_id`. - // Maps contain only information about current client's `sendonly` tracks. - message SdpOffer { - message TrackIdToMetadata { - string id = 1; - media_events.shared.Metadata metadata = 2; - } - - message TrackIdToTrackBitrates { - string id = 1; - repeated VariantBitrates bitrates = 2; - } - - string sdp = 1; - repeated TrackIdToMetadata trackIdToTrackMetadata = 2; - repeated TrackIdToTrackBitrates trackIdToTrackBitrates = 3; - repeated media_events.shared.MidToTrackId mid_to_track_id = 4; - } - - // Informs that a client wants to receive a specific track variant. - // The track variant will be sent whenever it is available. - // If choosen variant is unavailable, some other variant will be - // sent until choosen variant becomes active again. - message SetTargetTrackVariant { - string track_id = 1; - media_events.shared.Encoding variant = 2; // TODO: should be encoding - } - - // Contains updated bitrates of variants of the track send by the client. - // Needs to contain all of the variants. - message TrackVariantBitrates { - string track_id = 1; - repeated VariantBitrates variantBitrates = 2; - } - - oneof content { - Connect connect = 1; - Disconnect disconnect = 2; - UpdateEndpointMetadata update_endpoint_metadata = 3; - DisableTrackEncoding disable_track_encoding = 4; - EnableTrackEncoding enable_track_encoding = 5; - UpdateTrackMetadata udpate_track_metadtata = 6; - MuteTrack mute_track = 7; - UnmuteTrack unmute_track = 8; - RenegotiateTracks renegotiate_tracks = 9; - PrioritizeTrack prioritize_track = 10; - UnprioritizeTrack unprioritize_track = 11; - PreferedVideoSizes prefered_video_sizes = 12; - media_events.shared.Candidate candidate = 13; - SdpOffer sdp_offer = 14; - SetTargetTrackVariant set_target_track_variant = 15; - TrackVariantBitrates track_variant_bitrates = 16; - } -} diff --git a/fishjam/media_events/peer.proto b/fishjam/media_events/peer.proto new file mode 100644 index 0000000..17b42ba --- /dev/null +++ b/fishjam/media_events/peer.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package media_events.peer; + +import "fishjam/media_events/shared.proto"; + +// Defines any type of message sent between FJ and a peer +message MediaEvent { + message VariantBitrates { + media_events.shared.Encoding rid = 1; + int32 bitrate = 2; + } + + // Peer -> WebRTC Endpoint + + // Sent when a peer wants to join WebRTC Endpoint. It contains only endpoint's metadata + message Connect { + media_events.shared.Metadata metadata = 1; + } + + // Sent when peer disconnects from WebRTC Endpoint. It contains no data + message Disconnect {} + + // Contains new metadata of peer's endpoint + message UpdateEndpointMetadata { + media_events.shared.Metadata metadata = 1; + } + + // Contains new metadata of some track and an id of this track + message UpdateTrackMetadata { + string track_id = 1; + media_events.shared.Metadata metadata = 2; + } + + // Informs that a peer wants to renegotiate connection due to adding a track or removing a track + message RenegotiateTracks {} + + // Contains an SDP offer, a mapping between `track_id` and `track_metadata`, mapping between `track_id` + // and this track's bitrates (or all of its variants bitrates) and a mapping between `mid` and `track_id`. + // Maps contain only information about current peer's `sendonly` tracks. + message SdpOffer { + message TrackIdToMetadata { + string track_id = 1; + media_events.shared.Metadata metadata = 2; + } + + message TrackIdToTrackBitrates { + oneof tracks { + TrackBitrate track_bitrate = 1; + TrackVariantBitrates TrackVariantBitrates = 2; + } + } + + string sdp_offer = 1; + repeated TrackIdToMetadata trackIdToTrackMetadata = 2; + repeated TrackIdToTrackBitrates trackIdToTrackBitrates = 3; + repeated media_events.shared.MidToTrackId mid_to_track_id = 4; + } + + // Contains updated bitrate of the track send by the peer (no simulcast). + message TrackBitrate { + string track_id = 1; + int32 bitrate = 2; + } + + // Contains updated bitrates of variants of the track send by the peer. + // Needs to contain all of the variants. + message TrackVariantBitrates { + string track_id = 1; + repeated VariantBitrates variant_bitrates = 2; + } + + oneof content { + Connect connect = 1; + Disconnect disconnect = 2; + UpdateEndpointMetadata update_endpoint_metadata = 3; + UpdateTrackMetadata udpate_track_metadtata = 4; + RenegotiateTracks renegotiate_tracks = 5; + media_events.shared.Candidate candidate = 6; + SdpOffer sdp_offer = 7; + TrackBitrate track_bitrate = 8; + } +} diff --git a/fishjam/media_events/server.proto b/fishjam/media_events/server.proto index c1ead79..7fd29ac 100644 --- a/fishjam/media_events/server.proto +++ b/fishjam/media_events/server.proto @@ -1,110 +1,79 @@ syntax = "proto3"; -package webrtc; +package media_events.server; import "fishjam/media_events/shared.proto"; -// Defines any type of message sent between FJ and a peer +// Defines any type of media event message sent between Fishjam and a peer message MediaEvent { - // WebRTC Endpoint -> Client + message SimulcastConfig { + bool enabled = 1; + repeated media_events.shared.Encoding active_encodings = 2; + repeated media_events.shared.Encoding disabled_encodings = 3; + } - // Contains all tracks that will be forwarded to client until the next `tracks_priority` message. - // This type of messages is sent only if DisplayManager is spawned - message TracksPriority { - repeated string tracks = 1; + message Track { + string track_id = 1; + media_events.shared.Metadata metadata = 2; + string simulcast_config = 3; } + // WebRTC Endpoint -> Peer + // Contains information about updated metadata of one of the endpoints message EndpointUpdated { string endpoint_id = 1; - string metadata = 2; + media_events.shared.Metadata metadata = 2; } // Contains information about new metadata of one of tracks message TrackUpdated { string endpoint_id = 1; string track_id = 2; - string metadata = 3; + media_events.shared.Metadata metadata = 3; } - // Informs that one of the clients has added one or more tracks. + // Informs that one of the peers has added one or more tracks. // It contains: - // - an id of endpoint associated with that client, + // - an id of endpoint associated with that peer, // - a map of all tracks with `track_id`'s as keys and objects with `track_metadata` and simulcast conifg as a value - // - (Depracated field use tracks) a map of all tracks with `track_id`s as keys and `track_metadata` as value. message TracksAdded { - message Track { - message SimulcastConfig { - bool enabled = 1; - repeated string active_encodings = 2; - repeated string disabled_encodings = 3; - } - - string track_id = 1; - string metadata = 2; - string simulcast_config = 3; - } - string endpoint_id = 1; repeated Track tracks = 2; } - // Contains a list of tracks which have been removed by some client and id of that client's + // Contains a list of tracks which have been removed by some peer and id of that peer's message TracksRemoved { string endpoint_id = 1; repeated string track_ids = 2; } - // Sent when some track variant were disabled by an endpoint - message TrackEncodingDisabled { - string track_id = 1; - media_events.shared.Encoding encoding = 2; - } - - // Sent when some track variant were enabled by an endpoint - message TrackEncodingEnabled { - string track_id = 1; - media_events.shared.Encoding encoding = 2; - } - - // Message sent to all clients in the room after a new endpoint was added. + // Message sent to all peers in the room after a new endpoint was added. // It contains id and metadata of the new endpoint. message EndpointAdded { - string id = 1; - string metadata = 2; + string endpoint_id = 1; + media_events.shared.Metadata metadata = 2; } - // Message sent to the client after connecting to the WebRTC Endpoint. - // It contains the id of that client's endpoint and a list of information about endpoints in the Engine + // Message sent to the peer after connecting to the WebRTC Endpoint. + // It contains the id of that peer's endpoint and a list of information about endpoints in the Engine // (id, metadata, a `trackIdToMetadata` and tracks like seen in `tracksAdded`) + // TODO: SIMULCAST message Connected { - message SimulcastConfig { - bool enabled = 1; - repeated media_events.shared.Encoding active_encodings = 2; - repeated media_events.shared.Encoding disabled_encodings = 3; - } - - message Track { - string id = 1; - media_events.shared.Metadata metadata = 2; - SimulcastConfig SimulcastConfig = 3; - } - message Endpoint { - string id = 1; - string type = 2; + string endpoint_id = 1; + string endpoint_type = 2; media_events.shared.Metadata metadata = 3; - // deprecated field - // track_id_to_metadata repeated Track tracks = 4; } - string id = 1; + + string endpoint_id = 1; repeated Endpoint endpoints = 2; } - // Sent to all remaining clients in the room after some endpoint was removed. It contains an id of the removed endpoint. + // Sent to all remaining peers in the room after some endpoint was removed. It contains an id of the removed endpoint. message EndpointRemoved { - string id = 1; + string endpoint_id = 1; } // Informs that an error occurred on the server providing a message to show @@ -112,26 +81,17 @@ message MediaEvent { string message = 1; } - // WebRTC Enpoint -> Client + // WebRTC Enpoint -> Peer // Contains information about the number of audio and video tracks that will be sent from the engine - // to the client and information regarding the integrated TURN server. + // to the peer and information regarding the integrated TURN server. message OfferData { message TrackTypes { int32 audio = 1; int32 video = 2; } - message IntegratedTurnServers { - string server_addr = 1; - int32 server_port = 2; - string transport = 3; - string password = 4; - string username = 5; - } - TrackTypes tracks_types = 1; - repeated IntegratedTurnServers integrated_turn_servers = 2; } // Contains an SDP answer and mapping between `mid` and `track_id` for all tracks (active, inactive, inbound and outbound) @@ -140,24 +100,6 @@ message MediaEvent { repeated media_events.shared.MidToTrackId midToTrackId = 2; } - // Informs that track with id `trackId` belonging to endpoint with id `endpointId` will be sent in encoding `encoding` now. - // The meaning is as follows: - // "low_bandwidth" - we no longer have enough bandwidth to maintain current track quality - // "encoding_inactive" - encoding became inactive - // "other" - it was hard to determine exact reason of encoding switch - message EncodingSwitched { - enum Reason { - LOW_BANDWIDTH = 0; - ENCODING_INACTIVE = 1; - OTHER = 2; - } - - string endpoint_id = 1; - string track_id = 2; - media_events.shared.Encoding encoding = 3; - Reason reason = 4; - } - // Informs that the track denoted by `trackId` has changed their voice actiivty // For this notification to work, the server must be configured to use VAD extension // and the sender must support it. @@ -171,31 +113,18 @@ message MediaEvent { Status status = 2; } - // Informs about client's available incoming bitrate (a.k.a. download) estimated by the server. - // It is measured in bits per second. - // For this notification to work, the server must be configured to use TWCC extension - // and the sender must support it. - message BandwidthEstimation { - int32 estimation = 1; - } - oneof content { - TracksPriority tracks_priority = 1; - EndpointUpdated endpoint_updated = 2; - TrackUpdated track_updated = 3; - TracksAdded tracks_added = 4; - TracksRemoved tracks_removed = 5; - TrackEncodingDisabled track_encoding_disabled = 6; - TrackEncodingEnabled track_encoding_enabled = 7; - EndpointAdded endpoint_added = 8; - EndpointRemoved endpoint_removed = 9; - Connected connected = 10; - Error error = 11; - OfferData offer_data = 12; - media_events.shared.Candidate candidate = 13; - SdpAnswer sdp_answer = 14; - EncodingSwitched encoding_switched = 15; - VadNotification vad_notification = 16; - BandwidthEstimation bandwidth_estimation = 17; + EndpointUpdated endpoint_updated = 1; + TrackUpdated track_updated = 2; + TracksAdded tracks_added = 3; + TracksRemoved tracks_removed = 4; + EndpointAdded endpoint_added = 5; + EndpointRemoved endpoint_removed = 6; + Connected connected = 7; + Error error = 8; + OfferData offer_data = 9; + media_events.shared.Candidate candidate = 10; + SdpAnswer sdp_answer = 11; + VadNotification vad_notification = 12; } } diff --git a/fishjam/media_events/shared.proto b/fishjam/media_events/shared.proto index f18aab1..e5ac82d 100644 --- a/fishjam/media_events/shared.proto +++ b/fishjam/media_events/shared.proto @@ -17,8 +17,10 @@ message MidToTrackId { string track_id = 2; } -// Contains information about an ICE candidate which will be sent to the client/server +// Contains information about an ICE candidate which will be sent to the peer/server message Candidate { string candidate = 1; int32 sdp_m_line_index = 2; + int32 sdp_mid = 3; + string username_fragment = 4; } diff --git a/fishjam/peer_notifications.proto b/fishjam/peer_notifications.proto index aff60c2..bd00217 100644 --- a/fishjam/peer_notifications.proto +++ b/fishjam/peer_notifications.proto @@ -2,6 +2,9 @@ syntax = "proto3"; package fishjam; +import "fishjam/media_events/peer.proto"; +import "fishjam/media_events/server.proto"; + // Defines any type of message sent between FJ and a peer message PeerMessage { // Response sent by FJ, confirming successfull authentication @@ -12,11 +15,6 @@ message PeerMessage { string token = 1; } - // Any type of WebRTC messages passed betweend FJ and peer - message MediaEvent { - string data = 1; - } - // PeerConnection stats sent by peer // https://developer.mozilla.org/en-US/docs/Web/API/RTCStatsReport#the_statistic_types message RTCStatsReport { @@ -26,7 +24,8 @@ message PeerMessage { oneof content { Authenticated authenticated = 1; AuthRequest auth_request = 2; - MediaEvent media_event = 3; - RTCStatsReport rtc_stats_report = 4; + media_events.peer.MediaEvent peer_media_event = 3; + media_events.server.MediaEvent server_media_event = 4; + RTCStatsReport rtc_stats_report = 5; } } diff --git a/fishjam/server_notifications.proto b/fishjam/server_notifications.proto index c3814c6..a137bce 100644 --- a/fishjam/server_notifications.proto +++ b/fishjam/server_notifications.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package fishjam; -// Defines any type of message passed between FJ and server client +// Defines any type of message passed between FJ and server peer message ServerMessage { // Notification sent when a room crashes message RoomCrashed { @@ -49,19 +49,19 @@ message ServerMessage { // Response sent by FJ, confirming successfull authentication message Authenticated {} - // Request sent by client, to authenticate to FJ server + // Request sent by peer, to authenticate to FJ server message AuthRequest { string token = 1; } - // Defines message groups for which client can subscribe + // Defines message groups for which peer can subscribe enum EventType { EVENT_TYPE_UNSPECIFIED = 0; EVENT_TYPE_SERVER_NOTIFICATION = 1; EVENT_TYPE_METRICS = 2; } - // Request sent by client to subsribe for certain message type + // Request sent by peer to subsribe for certain message type message SubscribeRequest { EventType event_type = 1; } From 0f7d4ce0e5015cf1e7b7aa797996ed5d45041dbf Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Mon, 28 Oct 2024 17:20:20 +0100 Subject: [PATCH 5/9] Fix lint warnings --- fishjam/media_events/{ => peer}/peer.proto | 22 ++++++++-------- .../media_events/{ => server}/server.proto | 25 ++++++++++--------- fishjam/media_events/shared.proto | 9 ++++--- fishjam/peer_notifications.proto | 4 +-- 4 files changed, 31 insertions(+), 29 deletions(-) rename fishjam/media_events/{ => peer}/peer.proto (78%) rename fishjam/media_events/{ => server}/server.proto (86%) diff --git a/fishjam/media_events/peer.proto b/fishjam/media_events/peer/peer.proto similarity index 78% rename from fishjam/media_events/peer.proto rename to fishjam/media_events/peer/peer.proto index 17b42ba..36ccd7a 100644 --- a/fishjam/media_events/peer.proto +++ b/fishjam/media_events/peer/peer.proto @@ -1,13 +1,13 @@ syntax = "proto3"; -package media_events.peer; +package fishjam.media_events.peer; import "fishjam/media_events/shared.proto"; // Defines any type of message sent between FJ and a peer message MediaEvent { message VariantBitrates { - media_events.shared.Encoding rid = 1; + media_events.Encoding rid = 1; int32 bitrate = 2; } @@ -15,7 +15,7 @@ message MediaEvent { // Sent when a peer wants to join WebRTC Endpoint. It contains only endpoint's metadata message Connect { - media_events.shared.Metadata metadata = 1; + media_events.Metadata metadata = 1; } // Sent when peer disconnects from WebRTC Endpoint. It contains no data @@ -23,13 +23,13 @@ message MediaEvent { // Contains new metadata of peer's endpoint message UpdateEndpointMetadata { - media_events.shared.Metadata metadata = 1; + media_events.Metadata metadata = 1; } // Contains new metadata of some track and an id of this track message UpdateTrackMetadata { string track_id = 1; - media_events.shared.Metadata metadata = 2; + media_events.Metadata metadata = 2; } // Informs that a peer wants to renegotiate connection due to adding a track or removing a track @@ -41,20 +41,20 @@ message MediaEvent { message SdpOffer { message TrackIdToMetadata { string track_id = 1; - media_events.shared.Metadata metadata = 2; + media_events.Metadata metadata = 2; } message TrackIdToTrackBitrates { oneof tracks { TrackBitrate track_bitrate = 1; - TrackVariantBitrates TrackVariantBitrates = 2; + TrackVariantBitrates track_variant_bitrates = 2; } } string sdp_offer = 1; - repeated TrackIdToMetadata trackIdToTrackMetadata = 2; - repeated TrackIdToTrackBitrates trackIdToTrackBitrates = 3; - repeated media_events.shared.MidToTrackId mid_to_track_id = 4; + repeated TrackIdToMetadata track_id_to_track_metadata = 2; + repeated TrackIdToTrackBitrates track_id_to_track_bitrates = 3; + repeated media_events.MidToTrackId mid_to_track_id = 4; } // Contains updated bitrate of the track send by the peer (no simulcast). @@ -76,7 +76,7 @@ message MediaEvent { UpdateEndpointMetadata update_endpoint_metadata = 3; UpdateTrackMetadata udpate_track_metadtata = 4; RenegotiateTracks renegotiate_tracks = 5; - media_events.shared.Candidate candidate = 6; + media_events.Candidate candidate = 6; SdpOffer sdp_offer = 7; TrackBitrate track_bitrate = 8; } diff --git a/fishjam/media_events/server.proto b/fishjam/media_events/server/server.proto similarity index 86% rename from fishjam/media_events/server.proto rename to fishjam/media_events/server/server.proto index 7fd29ac..6c9fb22 100644 --- a/fishjam/media_events/server.proto +++ b/fishjam/media_events/server/server.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -package media_events.server; +package fishjam.media_events.server; import "fishjam/media_events/shared.proto"; @@ -8,13 +8,13 @@ import "fishjam/media_events/shared.proto"; message MediaEvent { message SimulcastConfig { bool enabled = 1; - repeated media_events.shared.Encoding active_encodings = 2; - repeated media_events.shared.Encoding disabled_encodings = 3; + repeated media_events.Encoding active_encodings = 2; + repeated media_events.Encoding disabled_encodings = 3; } message Track { string track_id = 1; - media_events.shared.Metadata metadata = 2; + media_events.Metadata metadata = 2; string simulcast_config = 3; } @@ -23,14 +23,14 @@ message MediaEvent { // Contains information about updated metadata of one of the endpoints message EndpointUpdated { string endpoint_id = 1; - media_events.shared.Metadata metadata = 2; + media_events.Metadata metadata = 2; } // Contains information about new metadata of one of tracks message TrackUpdated { string endpoint_id = 1; string track_id = 2; - media_events.shared.Metadata metadata = 3; + media_events.Metadata metadata = 3; } // Informs that one of the peers has added one or more tracks. @@ -52,7 +52,7 @@ message MediaEvent { // It contains id and metadata of the new endpoint. message EndpointAdded { string endpoint_id = 1; - media_events.shared.Metadata metadata = 2; + media_events.Metadata metadata = 2; } // Message sent to the peer after connecting to the WebRTC Endpoint. @@ -63,7 +63,7 @@ message MediaEvent { message Endpoint { string endpoint_id = 1; string endpoint_type = 2; - media_events.shared.Metadata metadata = 3; + media_events.Metadata metadata = 3; repeated Track tracks = 4; } @@ -97,7 +97,7 @@ message MediaEvent { // Contains an SDP answer and mapping between `mid` and `track_id` for all tracks (active, inactive, inbound and outbound) message SdpAnswer { string sdp_answer = 1; - repeated media_events.shared.MidToTrackId midToTrackId = 2; + repeated media_events.MidToTrackId mid_to_track_id = 2; } // Informs that the track denoted by `trackId` has changed their voice actiivty @@ -105,8 +105,9 @@ message MediaEvent { // and the sender must support it. message VadNotification { enum Status { - SILENCE = 0; - SPEECH = 1; + STATUS_UNSPECIFIED = 0; + STATUS_SILENCE = 1; + STATUS_SPEECH = 2; } string track_id = 1; @@ -123,7 +124,7 @@ message MediaEvent { Connected connected = 7; Error error = 8; OfferData offer_data = 9; - media_events.shared.Candidate candidate = 10; + media_events.Candidate candidate = 10; SdpAnswer sdp_answer = 11; VadNotification vad_notification = 12; } diff --git a/fishjam/media_events/shared.proto b/fishjam/media_events/shared.proto index e5ac82d..8b6c97d 100644 --- a/fishjam/media_events/shared.proto +++ b/fishjam/media_events/shared.proto @@ -1,11 +1,12 @@ syntax = "proto3"; -package media_events.shared; +package fishjam.media_events; enum Encoding { - LOW = 0; - MEDIUM = 1; - HIGH = 2; + ENCODING_UNSPECIFIED = 0; + ENCODING_LOW = 1; + ENCODING_MEDIUM = 2; + ENCODING_HIGH = 3; } message Metadata { diff --git a/fishjam/peer_notifications.proto b/fishjam/peer_notifications.proto index bd00217..4efbbb0 100644 --- a/fishjam/peer_notifications.proto +++ b/fishjam/peer_notifications.proto @@ -2,8 +2,8 @@ syntax = "proto3"; package fishjam; -import "fishjam/media_events/peer.proto"; -import "fishjam/media_events/server.proto"; +import "fishjam/media_events/peer/peer.proto"; +import "fishjam/media_events/server/server.proto"; // Defines any type of message sent between FJ and a peer message PeerMessage { From 0addea9a3850814f79eb1f215dae4756a297a013 Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Wed, 30 Oct 2024 08:51:40 +0100 Subject: [PATCH 6/9] Requested changes --- fishjam/media_events/peer/peer.proto | 66 ++++++++++++------------ fishjam/media_events/server/server.proto | 65 ++++++++++------------- 2 files changed, 61 insertions(+), 70 deletions(-) diff --git a/fishjam/media_events/peer/peer.proto b/fishjam/media_events/peer/peer.proto index 36ccd7a..18ea46a 100644 --- a/fishjam/media_events/peer/peer.proto +++ b/fishjam/media_events/peer/peer.proto @@ -4,77 +4,77 @@ package fishjam.media_events.peer; import "fishjam/media_events/shared.proto"; -// Defines any type of message sent between FJ and a peer +// Defines any type of message sent from Peer to Membrane RTC Engine message MediaEvent { + // SCHEMAS message VariantBitrates { media_events.Encoding rid = 1; int32 bitrate = 2; } - // Peer -> WebRTC Endpoint + // Sent when peer wants to update the bitrates of a simulcast track. + // Needs to contain all of the variants. + message TrackVariantBitrates { + string track_id = 1; + repeated VariantBitrates variant_bitrates = 2; + } + + message TrackIdToMetadata { + string track_id = 1; + media_events.Metadata metadata = 2; + } + + message TrackIdToBitrates { + oneof tracks { + TrackBitrate track_bitrate = 1; + TrackVariantBitrates track_variant_bitrates = 2; + } + } - // Sent when a peer wants to join WebRTC Endpoint. It contains only endpoint's metadata + // MEDIA EVENTS + + // Sent when a peer wants to join WebRTC Endpoint. message Connect { media_events.Metadata metadata = 1; } - // Sent when peer disconnects from WebRTC Endpoint. It contains no data + // Sent when a peer disconnects from WebRTC Endpoint. message Disconnect {} - // Contains new metadata of peer's endpoint + // Sent when a peer wants to update its metadata message UpdateEndpointMetadata { media_events.Metadata metadata = 1; } - // Contains new metadata of some track and an id of this track + // Sent when a peer wants to update its track's metadata message UpdateTrackMetadata { string track_id = 1; media_events.Metadata metadata = 2; } - // Informs that a peer wants to renegotiate connection due to adding a track or removing a track + // Sent when peer wants to renegatiate connection due to adding a track or removing a track message RenegotiateTracks {} - // Contains an SDP offer, a mapping between `track_id` and `track_metadata`, mapping between `track_id` - // and this track's bitrates (or all of its variants bitrates) and a mapping between `mid` and `track_id`. - // Maps contain only information about current peer's `sendonly` tracks. + // Sent as a response to `offerData` media event during renegotiation + // Maps contain only information about current peer's `sendonly` tracks. message SdpOffer { - message TrackIdToMetadata { - string track_id = 1; - media_events.Metadata metadata = 2; - } - - message TrackIdToTrackBitrates { - oneof tracks { - TrackBitrate track_bitrate = 1; - TrackVariantBitrates track_variant_bitrates = 2; - } - } - string sdp_offer = 1; - repeated TrackIdToMetadata track_id_to_track_metadata = 2; - repeated TrackIdToTrackBitrates track_id_to_track_bitrates = 3; + repeated TrackIdToMetadata track_id_to_metadata = 2; + repeated TrackIdToBitrates track_id_to_bitrates = 3; repeated media_events.MidToTrackId mid_to_track_id = 4; } - // Contains updated bitrate of the track send by the peer (no simulcast). + // Sent when Peer wants to update its track's bitrate message TrackBitrate { string track_id = 1; int32 bitrate = 2; } - // Contains updated bitrates of variants of the track send by the peer. - // Needs to contain all of the variants. - message TrackVariantBitrates { - string track_id = 1; - repeated VariantBitrates variant_bitrates = 2; - } - oneof content { Connect connect = 1; Disconnect disconnect = 2; UpdateEndpointMetadata update_endpoint_metadata = 3; - UpdateTrackMetadata udpate_track_metadtata = 4; + UpdateTrackMetadata update_track_metadata = 4; RenegotiateTracks renegotiate_tracks = 5; media_events.Candidate candidate = 6; SdpOffer sdp_offer = 7; diff --git a/fishjam/media_events/server/server.proto b/fishjam/media_events/server/server.proto index 6c9fb22..d7b83d9 100644 --- a/fishjam/media_events/server/server.proto +++ b/fishjam/media_events/server/server.proto @@ -4,8 +4,9 @@ package fishjam.media_events.server; import "fishjam/media_events/shared.proto"; -// Defines any type of media event message sent between Fishjam and a peer +// Defines any type of message sent from Membrane RTC Engine to Peer message MediaEvent { + // SCHEMAS message SimulcastConfig { bool enabled = 1; repeated media_events.Encoding active_encodings = 2; @@ -15,76 +16,72 @@ message MediaEvent { message Track { string track_id = 1; media_events.Metadata metadata = 2; - string simulcast_config = 3; + SimulcastConfig simulcast_config = 3; } - // WebRTC Endpoint -> Peer + message Endpoint { + string endpoint_id = 1; + string endpoint_type = 2; + media_events.Metadata metadata = 3; + repeated Track tracks = 4; + } + + enum Status { + STATUS_UNSPECIFIED = 0; + STATUS_SILENCE = 1; + STATUS_SPEECH = 2; + } + + // MEDIA EVENTS - // Contains information about updated metadata of one of the endpoints + // Sent when metadata of one of the endpoints was updated message EndpointUpdated { string endpoint_id = 1; media_events.Metadata metadata = 2; } - // Contains information about new metadata of one of tracks + // Sent when metadata of one of the tracks was updated message TrackUpdated { string endpoint_id = 1; string track_id = 2; media_events.Metadata metadata = 3; } - // Informs that one of the peers has added one or more tracks. - // It contains: - // - an id of endpoint associated with that peer, - // - a map of all tracks with `track_id`'s as keys and objects with `track_metadata` and simulcast conifg as a value + // Sent to informs that one of the peers has added one or more tracks. message TracksAdded { string endpoint_id = 1; repeated Track tracks = 2; } - // Contains a list of tracks which have been removed by some peer and id of that peer's + // Sent to informs that one of the peers has removed one or more tracks. message TracksRemoved { string endpoint_id = 1; repeated string track_ids = 2; } - // Message sent to all peers in the room after a new endpoint was added. - // It contains id and metadata of the new endpoint. + // Sent to all peers in the room after a new endpoint was added. message EndpointAdded { string endpoint_id = 1; media_events.Metadata metadata = 2; } - // Message sent to the peer after connecting to the WebRTC Endpoint. - // It contains the id of that peer's endpoint and a list of information about endpoints in the Engine - // (id, metadata, a `trackIdToMetadata` and tracks like seen in `tracksAdded`) - // TODO: SIMULCAST + // Sent to the peer after connecting to the WebRTC Endpoint. message Connected { - message Endpoint { - string endpoint_id = 1; - string endpoint_type = 2; - media_events.Metadata metadata = 3; - repeated Track tracks = 4; - } - string endpoint_id = 1; repeated Endpoint endpoints = 2; } - // Sent to all remaining peers in the room after some endpoint was removed. It contains an id of the removed endpoint. + // Sent to all remaining peers in the room after some endpoint was removed. message EndpointRemoved { string endpoint_id = 1; } - // Informs that an error occurred on the server providing a message to show + // Sent to inform that an error occurred on the server providing a message to show message Error { string message = 1; } - // WebRTC Enpoint -> Peer - - // Contains information about the number of audio and video tracks that will be sent from the engine - // to the peer and information regarding the integrated TURN server. + // Sent to inform about the number of audio and video tracks that will be sent from the engine to the peer message OfferData { message TrackTypes { int32 audio = 1; @@ -94,22 +91,16 @@ message MediaEvent { TrackTypes tracks_types = 1; } - // Contains an SDP answer and mapping between `mid` and `track_id` for all tracks (active, inactive, inbound and outbound) + // Sent after receiving `SdpOffer` from Peer message SdpAnswer { string sdp_answer = 1; repeated media_events.MidToTrackId mid_to_track_id = 2; } - // Informs that the track denoted by `trackId` has changed their voice actiivty + // Sent to inform that the track denoted by `trackId` has changed their voice actiivty // For this notification to work, the server must be configured to use VAD extension // and the sender must support it. message VadNotification { - enum Status { - STATUS_UNSPECIFIED = 0; - STATUS_SILENCE = 1; - STATUS_SPEECH = 2; - } - string track_id = 1; Status status = 2; } From 964527e16069d4a32149a2eeb96f191dee7da3d8 Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Wed, 30 Oct 2024 13:27:05 +0100 Subject: [PATCH 7/9] Requested changes --- fishjam/media_events/peer/peer.proto | 7 ++++--- fishjam/media_events/server/server.proto | 16 ++++++++-------- fishjam/media_events/shared.proto | 12 ++++++------ fishjam/peer_notifications.proto | 1 + 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/fishjam/media_events/peer/peer.proto b/fishjam/media_events/peer/peer.proto index 18ea46a..3b54608 100644 --- a/fishjam/media_events/peer/peer.proto +++ b/fishjam/media_events/peer/peer.proto @@ -7,8 +7,8 @@ import "fishjam/media_events/shared.proto"; // Defines any type of message sent from Peer to Membrane RTC Engine message MediaEvent { // SCHEMAS - message VariantBitrates { - media_events.Encoding rid = 1; + message VariantBitrate { + media_events.Variant variant = 1; int32 bitrate = 2; } @@ -16,7 +16,7 @@ message MediaEvent { // Needs to contain all of the variants. message TrackVariantBitrates { string track_id = 1; - repeated VariantBitrates variant_bitrates = 2; + repeated VariantBitrate variant_bitrates = 2; } message TrackIdToMetadata { @@ -57,6 +57,7 @@ message MediaEvent { // Sent as a response to `offerData` media event during renegotiation // Maps contain only information about current peer's `sendonly` tracks. + // The "mid" is an identifier used to associate an RTP packet with an MLine from the SDP offer/answer. message SdpOffer { string sdp_offer = 1; repeated TrackIdToMetadata track_id_to_metadata = 2; diff --git a/fishjam/media_events/server/server.proto b/fishjam/media_events/server/server.proto index d7b83d9..1195d18 100644 --- a/fishjam/media_events/server/server.proto +++ b/fishjam/media_events/server/server.proto @@ -9,8 +9,8 @@ message MediaEvent { // SCHEMAS message SimulcastConfig { bool enabled = 1; - repeated media_events.Encoding active_encodings = 2; - repeated media_events.Encoding disabled_encodings = 3; + repeated media_events.Variant active_variants = 2; + repeated media_events.Variant disabled_variants = 3; } message Track { @@ -26,12 +26,6 @@ message MediaEvent { repeated Track tracks = 4; } - enum Status { - STATUS_UNSPECIFIED = 0; - STATUS_SILENCE = 1; - STATUS_SPEECH = 2; - } - // MEDIA EVENTS // Sent when metadata of one of the endpoints was updated @@ -101,6 +95,12 @@ message MediaEvent { // For this notification to work, the server must be configured to use VAD extension // and the sender must support it. message VadNotification { + enum Status { + STATUS_UNSPECIFIED = 0; + STATUS_SILENCE = 1; + STATUS_SPEECH = 2; + } + string track_id = 1; Status status = 2; } diff --git a/fishjam/media_events/shared.proto b/fishjam/media_events/shared.proto index 8b6c97d..b66496f 100644 --- a/fishjam/media_events/shared.proto +++ b/fishjam/media_events/shared.proto @@ -2,11 +2,11 @@ syntax = "proto3"; package fishjam.media_events; -enum Encoding { - ENCODING_UNSPECIFIED = 0; - ENCODING_LOW = 1; - ENCODING_MEDIUM = 2; - ENCODING_HIGH = 3; +enum Variant { + VARIANT_UNSPECIFIED = 0; + VARIANT_LOW = 1; + VARIANT_MEDIUM = 2; + VARIANT_HIGH = 3; } message Metadata { @@ -22,6 +22,6 @@ message MidToTrackId { message Candidate { string candidate = 1; int32 sdp_m_line_index = 2; - int32 sdp_mid = 3; + string sdp_mid = 3; string username_fragment = 4; } diff --git a/fishjam/peer_notifications.proto b/fishjam/peer_notifications.proto index 4efbbb0..d75bc5e 100644 --- a/fishjam/peer_notifications.proto +++ b/fishjam/peer_notifications.proto @@ -13,6 +13,7 @@ message PeerMessage { // Request sent by peer, to authenticate to FJ server message AuthRequest { string token = 1; + string sdk_version = 2; } // PeerConnection stats sent by peer From 8c4edabb20df4d798382bb7a62c0e044e3a6418e Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Wed, 30 Oct 2024 15:44:42 +0100 Subject: [PATCH 8/9] Requested changes --- fishjam/media_events/server/server.proto | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fishjam/media_events/server/server.proto b/fishjam/media_events/server/server.proto index 1195d18..448d13c 100644 --- a/fishjam/media_events/server/server.proto +++ b/fishjam/media_events/server/server.proto @@ -7,16 +7,9 @@ import "fishjam/media_events/shared.proto"; // Defines any type of message sent from Membrane RTC Engine to Peer message MediaEvent { // SCHEMAS - message SimulcastConfig { - bool enabled = 1; - repeated media_events.Variant active_variants = 2; - repeated media_events.Variant disabled_variants = 3; - } - message Track { string track_id = 1; media_events.Metadata metadata = 2; - SimulcastConfig simulcast_config = 3; } message Endpoint { From 2190cd95db41592cc5dcacd97b5114fcd548a437 Mon Sep 17 00:00:00 2001 From: Karol Konkol Date: Thu, 31 Oct 2024 12:28:52 +0100 Subject: [PATCH 9/9] Remove simulcast track variants --- fishjam/media_events/peer/peer.proto | 8 -------- 1 file changed, 8 deletions(-) diff --git a/fishjam/media_events/peer/peer.proto b/fishjam/media_events/peer/peer.proto index 3b54608..e577f5d 100644 --- a/fishjam/media_events/peer/peer.proto +++ b/fishjam/media_events/peer/peer.proto @@ -12,13 +12,6 @@ message MediaEvent { int32 bitrate = 2; } - // Sent when peer wants to update the bitrates of a simulcast track. - // Needs to contain all of the variants. - message TrackVariantBitrates { - string track_id = 1; - repeated VariantBitrate variant_bitrates = 2; - } - message TrackIdToMetadata { string track_id = 1; media_events.Metadata metadata = 2; @@ -27,7 +20,6 @@ message MediaEvent { message TrackIdToBitrates { oneof tracks { TrackBitrate track_bitrate = 1; - TrackVariantBitrates track_variant_bitrates = 2; } }