diff --git a/crates/matchmaker-proto/src/lib.rs b/crates/matchmaker-proto/src/lib.rs index f268302b31..a9ce6fac12 100644 --- a/crates/matchmaker-proto/src/lib.rs +++ b/crates/matchmaker-proto/src/lib.rs @@ -68,9 +68,12 @@ pub struct SendProxyMessage { pub message: Vec, } +/// The client to send a network message to. #[derive(Serialize, Deserialize, Debug, Clone)] pub enum TargetClient { + /// Send the message to all connected clients. All, + /// Send the message to the client with the specified index. One(u8), } diff --git a/src/main.rs b/src/main.rs index 210b184bda..93677cb39a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -34,7 +34,6 @@ pub mod loading; pub mod localization; pub mod map; pub mod metadata; -pub mod name; pub mod networking; pub mod physics; pub mod platform; @@ -62,7 +61,6 @@ use crate::{ localization::LocalizationPlugin, map::MapPlugin, metadata::{GameMeta, MetadataPlugin}, - name::NamePlugin, networking::NetworkingPlugin, physics::PhysicsPlugin, platform::PlatformPlugin, @@ -216,7 +214,6 @@ pub fn main() { .add_plugin(LoadingPlugin) .add_plugin(AssetPlugin) .add_plugin(LocalizationPlugin) - .add_plugin(NamePlugin) .add_plugin(AnimationPlugin) .add_plugin(PlayerPlugin) .add_plugin(ItemPlugin) diff --git a/src/map.rs b/src/map.rs index 06e0977848..9786118e68 100644 --- a/src/map.rs +++ b/src/map.rs @@ -7,7 +7,6 @@ use bevy_prototype_lyon::{prelude::*, shapes::Rectangle}; use crate::{ camera::GameRenderLayers, metadata::{MapElementMeta, MapLayerKind, MapLayerMeta, MapMeta}, - name::EntityName, physics::collisions::{CollisionLayerTag, TileCollision}, player::{PlayerIdx, PlayerKillCommand}, prelude::*, @@ -220,7 +219,7 @@ pub fn hydrate_map( let entity = commands .spawn() - .insert(EntityName(format!( + .insert(Name::new(format!( "Map Element ( {layer_id} ): {element_name}" ))) .insert(Visibility::default()) diff --git a/src/map/elements.rs b/src/map/elements.rs index 53b5f98d3f..71f54a688f 100644 --- a/src/map/elements.rs +++ b/src/map/elements.rs @@ -5,7 +5,6 @@ use crate::{ lifetime::Lifetime, map::{MapElementHydrated, MapRespawnPoint}, metadata::{BuiltinElementKind, MapElementMeta}, - name::EntityName, physics::{collisions::CollisionWorld, KinematicBody}, player::{input::PlayerInputs, PlayerIdx, MAX_PLAYERS}, prelude::*, diff --git a/src/map/elements/crate_item.rs b/src/map/elements/crate_item.rs index 210fe40061..847ae38450 100644 --- a/src/map/elements/crate_item.rs +++ b/src/map/elements/crate_item.rs @@ -87,7 +87,7 @@ fn pre_update_in_game( script: "core:crate".into(), }) .insert(IdleCrateItem { spawner: entity }) - .insert(EntityName("Item: Crate".into())) + .insert(Name::new("Item: Crate")) .insert(AnimatedSprite { start: 0, end: 0, diff --git a/src/map/elements/grenade.rs b/src/map/elements/grenade.rs index aab9828d0b..78d19a5357 100644 --- a/src/map/elements/grenade.rs +++ b/src/map/elements/grenade.rs @@ -82,7 +82,7 @@ fn pre_update_in_game( script: "core:grenade".into(), }) .insert(IdleGrenade { spawner: entity }) - .insert(EntityName("Item: Grenade".into())) + .insert(Name::new("Item: Grenade")) .insert(AnimatedSprite { start: 0, end: 0, diff --git a/src/map/elements/mine.rs b/src/map/elements/mine.rs index bdc6da2861..877188fc91 100644 --- a/src/map/elements/mine.rs +++ b/src/map/elements/mine.rs @@ -84,7 +84,7 @@ fn pre_update_in_game( script: "core:mine".into(), }) .insert(IdleMine { spawner: entity }) - .insert(EntityName("Item: Mine".into())) + .insert(Name::new("Item: Mine")) .insert(AnimatedSprite { start: 0, end: 0, diff --git a/src/map/elements/stomp_boots.rs b/src/map/elements/stomp_boots.rs index 6759e499e1..9b285eac5a 100644 --- a/src/map/elements/stomp_boots.rs +++ b/src/map/elements/stomp_boots.rs @@ -58,7 +58,7 @@ fn pre_update_in_game( .insert(Item { script: "core:stomp_boots".into(), }) - .insert(EntityName("Item: Stomp Boots".into())) + .insert(Name::new("Item: Stomp Boots")) .insert(IdleStompBoots { spawner: entity }) .insert(AnimatedSprite { start: 0, diff --git a/src/map/elements/sword.rs b/src/map/elements/sword.rs index ff837faedf..c375ad6efd 100644 --- a/src/map/elements/sword.rs +++ b/src/map/elements/sword.rs @@ -49,7 +49,7 @@ fn pre_update_in_game( .insert(Item { script: "core:sword".into(), }) - .insert(EntityName("Item: Sword".into())) + .insert(Name::new("Item: Sword")) .insert(SwordState::default()) .insert(AnimatedSprite { start: 0, diff --git a/src/name.rs b/src/name.rs deleted file mode 100644 index a229ea9acf..0000000000 --- a/src/name.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::prelude::*; - -pub struct NamePlugin; - -impl Plugin for NamePlugin { - fn build(&self, app: &mut App) { - app.register_type::() - .add_system_to_stage(CoreStage::Last, update_entity_names) - .extend_rollback_plugin(|plugin| plugin.register_rollback_type::()); - } -} - -/// Conceptually identical to the [`Name`] component, but structured so that it can be added and -/// modified from scripts. Adding an [`EntityName`] component will cause a [`Name`] component to be -/// added and synced automatically. -#[derive(Reflect, Component, Default)] -#[reflect(Component, Default)] -pub struct EntityName(pub String); - -fn update_entity_names( - mut commands: Commands, - names: Query<(Entity, &EntityName), Changed>, -) { - for (entity, name) in &names { - commands.entity(entity).insert(Name::new(name.0.clone())); - } -} diff --git a/src/networking.md b/src/networking.md index 5ae8fa746a..8d75ec5962 100644 --- a/src/networking.md +++ b/src/networking.md @@ -1,3 +1,220 @@ Networked multi-player plugin. -TODO: describe network architecture. +Jumpy uses a Peer-to-Peer, rollback networking model built on [GGRS] and the [`bevy_ggrs`] plugin. + +We use a centralized matchmaking server to connect peers to each-other and to forward the peers' +network traffic. All connections utilize UDP and the QUIC protocol. + +Messages are serialized/deserialized to a binary representation using [`serde`] and the [`postcard`] +crate. + +The major facets of our networking are: + +- [Matchmaking](#matchmaking): How we connect clients to each-other and start an online match. +- [Synchronization](#synchronization): How we synchronize a network game between multiple players. + +You may also want to see: + +- [Future Changes](#future-changes) for some thoughts on changes we + might make to the current design. +- [Development & Debuggin](#development--debugging) for tips on testing networking during + development. + +[ggrs]: https://github.com/gschup/ggrs +[`bevy_ggrs`]: https://github.com/gschup/bevy_ggrs +[`serde`]: https://docs.rs/serde +[`postcard`]: https://docs.rs/postcard + +## Matchmaking + +In order to establish the peer connections we use a matchmaking server implemented in the +[`jumpy_matchmaker`] crate. This server binds one UDP port and listens for client connections. +Because QUIC supports mutliplexing connections, we are able to handle any number of clients on a +single UDP port. + +All client traffic is proxied to other peers through the matchmaking server. In this way it is not +true peer-to-peer networking, but logically, once the match starts, clients are sending messages to +each-other, and the server doesn't take part in the match protocol. + +Having the matchmaker proxy client messages has the following pros and cons: + +**Cons:** + +- It uses up more of the matchmaking server's bandwidth +- It adds an extra network hop between peers, increasing latency. + +**Pros:** + +- It reduces the number of connections each peer needs to make. Each peer only holds one + connection to the matchmaking server and nothing else. +- It hides the IP addresses of clients from each-other. This is an important privacy feature. +- It avoids a number of difficulties that you may run into while trying to establish true + peer-to-peer connections, and makes it much easier to bypass firewalls, NATs, etc. + +This doesn't prevent us from supporting true peer-to-peer connections in the future, though. +Similarly, another scenario we will support in the future is LAN games that you can join without +needing a matchmaking server. + +[`jumpy_matchmaker`]: https://fishfolk.github.io/jumpy/developers/rustdoc/jumpy_matchmaker/index.html + +### Matchmaking Protocol + +> **ℹ️ Note:** This is meant as an overview and is not an exact specification of the matchmaking +> protocol. + +#### Initial Connection + +When a client connects to the matchmaking server, the very first thing it will do is send a +[`RequestMatch`][jumpy_matchmaker_proto::MatchmakerRequest::RequestMatch] message to the server over +a reliable channel. + +This message contains the [`MatchInfo`][`jumpy_matchmaker_proto::MatchInfo`] that tells the server +how many players the client wants to connect to for the match, along with an arbitrary byte sequence +for the `match_data`. + +In order for players to end up in the same match as each-other, they must specify the _exact_ same +`MatchInfo`, including the `match_data`. We use the `match_data` as a way to specify which game mode +and parameters, etc. that the player wants to connect to, so that all the players that are connected +to each-other are playing the same mode. + +The `match_data` also contains the game name and version. Because the matchmaker does not take part +in the match protocol itself, just the matchmaking protocol, **this makes the matchmaking server +game agnostic**. Different games can connect to the same matchmaking server, and they can make sure +they are only connected to players playing the same game, by specifying a unique `match_data`. + +> **Note:** To be clear, the game implementation sets the `match_data` for players. Players are +> never exposed directly to the concept of the `match_data`. + +#### Waiting For Players + +After the initial connection and match request, the server will send the client an +[`Accepted`][`jumpy_matchmaker_proto::MatchmakerResponse::Accepted`] message. + +If the waiting room for that match already has the desired number of players in it, the server will +then respond immediately with a [`Success`][jumpy_matchmaker_proto::MatchmakerResponse::Success] +message. This message comes with: + +- a `random_seed` that can be used by all clients to generate deterministic random numbers, and +- a `player_idx` that tells the client _which_ player in the match it is. This is used throughout + the game to keep track of the players, and is between `0` and `player_count - 1`. + +#### In the Match + +Immediately after the desired number of clients have joined and the `Success` message has been sent +to all players, the matchmaker goes into proxy mode for all clients in the match. + +Once in proxy mode, the server listens for +[`SendProxyMessage`][`jumpy_matchmaker_proto::SendProxyMessage`]s from clients. Each message simply +specifies a [`TargetClient`][jumpy_matchmaker_proto::TargetClient] ( either a specific client or all +of them ), and a binary message data. + +Once it a `SendProxyMessage` it will send it to the target client, which will receive it in the form +of a [`RecvProxyMessage`][jumpy_matchmaker_proto::RecvProxyMessage], containing the message data, +and the index of the client that sent the message. + +The matchmaking server supports forwarding both reliable and unreliable message in this way, +allowing the game to chose any kind of protocol it sees fit to synchronize the match data. + +## Synchronization + +Match synchronization, as mentioned above, is accomplished with [GGRS]. GGRS re-imagines [GGPO] +network SDK. + +All of the Bevy systems that need to be synchronized during a match are added to their own Bevy +[Schedule][bevy::ecs::schedule::Schedule]. We use an [extension +trait][crate::schedule::RollbackScheduleAppExt] on the Bevy [`App`][bevy::app::App] to make it +easier to add systems to the rollback schedule in our plugins throughout Jumpy. + +The key requirement for rollback networking is: + +- The synchronized game loop must be **deterministic**. +- We must have the ability to **snapshot** and **restore** the game state. +- We must be able to run up to 8 game simulation frames in 16ms ( to achieve a 60 FPS frame rate ). + +### Determinism + +Luckily, Jumpy's physics and game logic is simple and we don't face any major non-determinism +issues. The primary source of potential non-determinism is Bevy's query iteration order and entity +allocation. + +#### Sorting Queries + +Because Bevy doesn't guarantee any specific order for entity iteration, we have to manually collect +and sort queries when a different order could produce a different in-game result. + +To aid in this we have a simple [`Sort`][crate::utils::Sort] component that we add to entities and +use to sort query results where it matters. + +It's easy to accidentally forget to sort entities when querying, and you may not notice issues until +you try to run a network game, and the clients end up playing a "different version" of the same +game. We hope we can improve this: see [Future Changes](#future-changes). + +[ggpo]: https://www.ggpo.net/ + +### Spawning Entities + +When spawning entities, we need to attach [`Rollback`][bevy_ggrs::Rollback] components to them, that +contain a unique index identifying the entity across rollbacks and restores, which may modify the +Entity's entity ID. + +We must be careful every time we spawn an item, that we deterministically assign the same `Rid` to +the entity on all clients. This mostly boils down to making sure we spawn them in the same order. + +### Snapshot & Restore + +All of the components that need to be synchronized during rollback must be registered with the +[`bevy_ggrs`] plugin. This is usually done in the Bevy plugin that adds the component, by calling +[`extend_rollback_plugin()`][crate::schedule::RollbackScheduleAppExt::extend_rollback_plugin] using +the extension trait on the Bevy `App` type. + +The [`bevy_ggrs`] plugin will then make sure that that component is snapshot and restored during +rollback and restore. + +Currently [`bevy_ggrs`] requires a [`Reflect`][bevy::reflect::Reflect] implementation on components +that will be synchronized, and it uses the `Reflect` implementation to clone the objects. We have +noticed that snapshot and restore using this technique can take up to 1ms. There are already plans, +once Bevy lands it's "Stageless" implementation, to re-implement [`bevy_ggrs`] and remove the +`Reflect` requirement, which should improve snapshot performance. + +This is important because it is hard to fit 8 frames into a 16ms time period, and taking a whole 1ms +to snapshot cuts down on how many frames we can run in that period of time. + +## Future Changes + +These are some ideas for future changes to the networking strategy. + +### Encapsulate Core Match Logic in an Isolated Micro ECS + +In order to improve our determinism and snapshot/restore story, we are +discussing ( see [#489] and [#510] ) an alternative architecture for handling the synchronization of +the match state. + +The idea is to move the core match game loop into it's own, tiny ECS that doesn't have the +non-deterministic iteration order problem, and that can also be snapshot and restored simply by +copying the entire ECS world. + +This creates a healthy isolation between Bevy and it's various resources and entities, and our core +game loop. Additionally, we may put this isolated ECS in a WASM module to allow for hot reloading +core game logic, and enabling mods in the future. + +[#489]: https://github.com/fishfolk/jumpy/discussions/489 +[#510]: https://github.com/fishfolk/jumpy/discussions/510 + +## Development & Debugging + +Here are some tips for debugging networking features while developing. + +### Local Sync Test + +It can be cumbersome to start a new networked match every time you need to troubleshoot some part of +the game that may not be rolling back or restoring properly. To help with this, you can run the game +with the `--sync-test-check-distance 7` to make the game test rolling back and forward 8 times every +frame as a test when starting a local game. + +This allows you to test the rollback without having to connect to a server. If things start popping +around the map or having other weird behaviors that they don't have without the sync-test mode, then +you know you have a rollback issue. + +> **ℹ️ Note:** Just because you **don't** have an issue in sync test mode, doesn't mean that there +> is no determinism issues. You still have to test network games with multiple game instances. There +> are some non-determinism issues that only exhibit themselves when restarting the game. diff --git a/src/networking.rs b/src/networking.rs index e3861fb136..2009409e72 100644 --- a/src/networking.rs +++ b/src/networking.rs @@ -22,7 +22,7 @@ impl Plugin for NetworkingPlugin { } } -// TODO: Map changes aren't working on network games for now. +/// TODO: Map changes aren't working on network games for now, so this isn't properly used/working. fn listen_for_map_changes( mut commands: Commands, client: Res, diff --git a/src/networking/client.rs b/src/networking/client.rs index ca5547b5f0..e37d12c63c 100644 --- a/src/networking/client.rs +++ b/src/networking/client.rs @@ -1,3 +1,11 @@ +//! The Bevy network client implementation. +//! +//! This plugin provides bevy the [`NetClient`] resource which is used to send and receive messages +//! over the the network. +//! +//! Note that, because we use a P2P rollback networking model, Bevy only ever acts as a client and +//! is never a "server". Messages are sent to other peers by means of the matchmaking server. + use std::{net::SocketAddr, sync::Arc}; use async_channel::{Receiver, RecvError, Sender}; diff --git a/src/networking/proto.rs b/src/networking/proto.rs index ddca46e916..2c8c7e32ca 100644 --- a/src/networking/proto.rs +++ b/src/networking/proto.rs @@ -1,3 +1,5 @@ +//! Serializable data types for network messages used by the game. + use crate::prelude::*; pub mod match_setup;