From 3a7c4865578ae66ef45ef587332c1573fcdad7e3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 21 Jul 2022 18:53:29 +0100 Subject: [PATCH 1/7] Regression test for infinite-loop-in-resync A rather commplicated test which demonstrated that the infinite loop was possible. --- ...federation_room_join_partial_state_test.go | 188 ++++++++++++++++++ 1 file changed, 188 insertions(+) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index 3fa1092a..01017c3c 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -9,6 +9,8 @@ package tests import ( "encoding/json" "fmt" + "github.com/gorilla/mux" + "io/ioutil" "net/http" "net/url" "strconv" @@ -511,6 +513,192 @@ func TestPartialStateJoin(t *testing.T) { t.Errorf("Did not find derek's m.room.member event in gappy /sync response: %s", err) } }) + + // regression test for https://github.com/matrix-org/synapse/issues/13001 + // + // There wass an edge case where, if we initially receive lots of events as outliers, + // and they then get de-outliered as partial state events, + t.Run("Resync works with many prev_events with partial state", func(t *testing.T) { + deployment := Deploy(t, b.BlueprintAlice) + defer deployment.Destroy(t) + alice := deployment.Client(t, "hs1", "@alice:hs1") + + psjResult := beginPartialStateJoin(t, deployment, alice) + defer psjResult.Destroy() + + // Alice has now joined the room, and the server is syncing the state in the background. + + // utility function to build a regular event in the test room + makeTimelineEvent := func(body string) *gomatrixserverlib.Event { + event := psjResult.Server.MustCreateEvent(t, psjResult.ServerRoom, b.Event{ + Type: "m.room.message", + Sender: psjResult.Server.UserID("charlie"), + Content: map[string]interface{}{ + "msgtype": "m.text", + "body": body, + }, + }) + psjResult.ServerRoom.AddEvent(event) + t.Logf("Created event %s: %s", body, event.EventID()) + return event + } + + // utility function to wait for a given event to arrive at the remote server. + // This works simply by polling /event until we get a 200. + awaitEventArrival := func(eventID string) { + start := time.Now() + for time.Since(start) < 10*time.Second { + res := alice.DoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", psjResult.ServerRoom.RoomID, "event", eventID}) + eventResBody := client.ParseJSON(t, res) + if res.StatusCode == 200 { + t.Logf("Alice successfully received event %s", eventID) + return + } + if res.StatusCode == 404 && gjson.GetBytes(eventResBody, "errcode").String() == "M_NOT_FOUND" { + t.Logf("Fetching event %s failed with M_NOT_FOUND; will retry", eventID) + time.Sleep(1000 * time.Millisecond) + continue + } + t.Fatalf("GET /event failed with %d: %s", res.StatusCode, string(eventResBody)) + } + t.Fatalf("timeout waiting for event %s to be received", eventID) + } + + // here's the first event which we *ought* to un-partial-state, but won't + lateEvent := makeTimelineEvent("late event") + + // next, we want to create 100 outliers. So, charlie creates 100 state events, and + // then persuades the SUT to create a backwards extremity using those events as + // part of the room state. + outliers := make([]*gomatrixserverlib.Event, 100) + outlierEventIDs := make([]string, len(outliers)) + for i := range outliers { + body := fmt.Sprintf("outlier event %d", i) + outliers[i] = psjResult.Server.MustCreateEvent(t, psjResult.ServerRoom, b.Event{ + Type: "outlier_state", + Sender: psjResult.Server.UserID("charlie"), + StateKey: b.Ptr(fmt.Sprintf("state_%d", i)), + Content: map[string]interface{}{"body": body}, + }) + psjResult.ServerRoom.AddEvent(outliers[i]) + outlierEventIDs[i] = outliers[i].EventID() + } + t.Logf("Created outliers: %s ... %s", outliers[0].EventID(), outliers[len(outliers)-1].EventID()) + + // a couple of regular timeline events to pull in the outliers... Note that these are persisted with *full* + // state rather than becoming partial state events. + timelineEvent1 := makeTimelineEvent("timeline event 1") + timelineEvent2 := makeTimelineEvent("timeline event 2") + + // dedicated get_missing_event handler for timelineEvent2. + // we grudgingly return a single event. + psjResult.Server.Mux().Handle("/_matrix/federation/v1/get_missing_events/{roomID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + vars := mux.Vars(req) + roomID := vars["roomID"] + + // Unmarshal the request body into an object + content, _ := ioutil.ReadAll(req.Body) + var request gomatrixserverlib.MissingEvents + if err := json.Unmarshal(content, &request); err != nil { + t.Fatalf("get_missing_events: Unable to unmarshal request body: %s", err.Error()) + } + + t.Logf("Got get_missing_events request in %s: %#v", roomID, request) + if roomID != psjResult.ServerRoom.RoomID { + t.Fatalf("get_missing_events for wrong room: got %s, want %s", roomID, psjResult.ServerRoom.RoomID) + } + + if request.LatestEvents[0] != timelineEvent2.EventID() { + t.Fatalf("get_missing_events for wrong event: got %v, want %s", request.LatestEvents, timelineEvent2.EventID()) + } + + // return timelineEvent1 + jsonb, _ := json.Marshal(gomatrixserverlib.RespMissingEvents{ + Events: gomatrixserverlib.EventJSONs{timelineEvent1.JSON()}, + }) + w.WriteHeader(200) + w.Write(jsonb) + t.Logf("Processed get_missing_events request: returned event %s", timelineEvent1.EventID()) + })) + + // dedicated state_ids and state handlers for timelineEvent1's prev event (ie, the last outlier event) + psjResult.Server.Mux().NewRoute().Methods("GET").Path( + fmt.Sprintf("/_matrix/federation/v1/state_ids/%s", psjResult.ServerRoom.RoomID), + ).Queries("event_id", outliers[len(outliers)-1].EventID()).Handler( + http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + queryParams := req.URL.Query() + t.Logf("Incoming state_ids request for last outlier event %s", queryParams["event_id"]) + jsonb, _ := json.Marshal(gomatrixserverlib.RespStateIDs{ + AuthEventIDs: eventIDsFromEvents(psjResult.ServerRoom.AuthChain()), + StateEventIDs: eventIDsFromEvents(psjResult.ServerRoom.AllCurrentState()), + }) + w.WriteHeader(200) + w.Write(jsonb) + }), + ) + psjResult.Server.Mux().NewRoute().Methods("GET").Path( + fmt.Sprintf("/_matrix/federation/v1/state/%s", psjResult.ServerRoom.RoomID), + ).Queries("event_id", outliers[len(outliers)-1].EventID()).Handler( + http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + queryParams := req.URL.Query() + t.Logf("Incoming state request for last outlier event %s", queryParams["event_id"]) + jsonb, _ := json.Marshal(gomatrixserverlib.RespState{ + AuthEvents: gomatrixserverlib.NewEventJSONsFromEvents(psjResult.ServerRoom.AuthChain()), + StateEvents: gomatrixserverlib.NewEventJSONsFromEvents(psjResult.ServerRoom.AllCurrentState()), + }) + w.WriteHeader(200) + w.Write(jsonb) + }), + ) + + // now, send over the most recent event, which will make the server get_missing_events + // (we will send timelineEvent1), and then request state (we will send all the outliers). + psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{timelineEvent2.JSON()}, nil) + + t.Logf("Charlie sent timeline event 2") + // wait for it to become visible, which implies that all the outliers have been pulled in. + awaitEventArrival(timelineEvent2.EventID()) + + // now we send over all the other events in the gap. + psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{lateEvent.JSON()}, nil) + t.Logf("Charlie sent late event") + + for i := 0; i < len(outliers); { + var transactionEvents []json.RawMessage + // a transaction can contain max 50 events + for j := i; j < i+50 && j < len(outliers); j++ { + transactionEvents = append(transactionEvents, outliers[j].JSON()) + } + psjResult.Server.MustSendTransaction(t, deployment, "hs1", transactionEvents, nil) + t.Logf("Charlie sent %d ex-outliers", len(transactionEvents)) + i += len(transactionEvents) + } + + // wait for the last outlier to arrive + awaitEventArrival(outliers[len(outliers)-1].EventID()) + + // release the federation /state response + psjResult.FinishStateRequest() + + // alice should be able to sync the room. We can't use SyncJoinedTo here because that looks for the + // membership event in the response (which we won't see, because all of the outlier events). + // instead let's just check for the presence of the room in the timeline + alice.MustSyncUntil(t, + client.SyncReq{}, + func(clientUserID string, topLevelSyncJSON gjson.Result) error { + key := "rooms.join." + client.GjsonEscape(psjResult.ServerRoom.RoomID) + ".timeline.events" + array := topLevelSyncJSON.Get(key) + if !array.Exists() { + return fmt.Errorf("Key %s does not exist", key) + } + if !array.IsArray() { + return fmt.Errorf("Key %s exists but it isn't an array", key) + } + return nil + }, + ) + t.Logf("Alice successfully synced") + }) } // buildLazyLoadingSyncFilter constructs a json-marshalled filter suitable the 'Filter' field of a client.SyncReq From 6c97ee90c5a434cfe57a1838aa6609d6f60ba1a4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 22 Jul 2022 10:53:57 +0100 Subject: [PATCH 2/7] Update tests/federation_room_join_partial_state_test.go Co-authored-by: kegsay --- tests/federation_room_join_partial_state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index 01017c3c..cd27e6ee 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -516,7 +516,7 @@ func TestPartialStateJoin(t *testing.T) { // regression test for https://github.com/matrix-org/synapse/issues/13001 // - // There wass an edge case where, if we initially receive lots of events as outliers, + // There was an edge case where, if we initially receive lots of events as outliers, // and they then get de-outliered as partial state events, t.Run("Resync works with many prev_events with partial state", func(t *testing.T) { deployment := Deploy(t, b.BlueprintAlice) From 4621b400c66288e24768060d52eac698cc82352e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 22 Jul 2022 11:11:31 +0100 Subject: [PATCH 3/7] expand SUT acronym --- tests/federation_room_join_partial_state_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index cd27e6ee..c2dc6b2d 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -568,7 +568,7 @@ func TestPartialStateJoin(t *testing.T) { lateEvent := makeTimelineEvent("late event") // next, we want to create 100 outliers. So, charlie creates 100 state events, and - // then persuades the SUT to create a backwards extremity using those events as + // then persuades the system under test to create a backwards extremity using those events as // part of the room state. outliers := make([]*gomatrixserverlib.Event, 100) outlierEventIDs := make([]string, len(outliers)) From 09c064404b72abbf1fcb2aeb4fa73b160eac85a4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 26 Jul 2022 13:33:21 +0100 Subject: [PATCH 4/7] use CreateMessageEvent instead of makeTimelineEvent --- ...federation_room_join_partial_state_test.go | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index c56f5023..d8880f3a 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -580,21 +580,6 @@ func TestPartialStateJoin(t *testing.T) { // Alice has now joined the room, and the server is syncing the state in the background. - // utility function to build a regular event in the test room - makeTimelineEvent := func(body string) *gomatrixserverlib.Event { - event := psjResult.Server.MustCreateEvent(t, psjResult.ServerRoom, b.Event{ - Type: "m.room.message", - Sender: psjResult.Server.UserID("charlie"), - Content: map[string]interface{}{ - "msgtype": "m.text", - "body": body, - }, - }) - psjResult.ServerRoom.AddEvent(event) - t.Logf("Created event %s: %s", body, event.EventID()) - return event - } - // utility function to wait for a given event to arrive at the remote server. // This works simply by polling /event until we get a 200. awaitEventArrival := func(eventID string) { @@ -617,7 +602,7 @@ func TestPartialStateJoin(t *testing.T) { } // here's the first event which we *ought* to un-partial-state, but won't - lateEvent := makeTimelineEvent("late event") + lateEvent := psjResult.CreateMessageEvent(t, "charlie", nil) // next, we want to create 100 outliers. So, charlie creates 100 state events, and // then persuades the system under test to create a backwards extremity using those events as @@ -639,8 +624,8 @@ func TestPartialStateJoin(t *testing.T) { // a couple of regular timeline events to pull in the outliers... Note that these are persisted with *full* // state rather than becoming partial state events. - timelineEvent1 := makeTimelineEvent("timeline event 1") - timelineEvent2 := makeTimelineEvent("timeline event 2") + timelineEvent1 := psjResult.CreateMessageEvent(t, "charlie", nil) + timelineEvent2 := psjResult.CreateMessageEvent(t, "charlie", nil) // dedicated get_missing_event handler for timelineEvent2. // we grudgingly return a single event. From ac705ffd44398fa75f19eaf08b0f76bb01b76ca0 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 26 Jul 2022 14:10:31 +0100 Subject: [PATCH 5/7] use `handleGetMissingEventsRequests` --- ...federation_room_join_partial_state_test.go | 31 ++----------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index d8880f3a..0a301149 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -629,34 +629,9 @@ func TestPartialStateJoin(t *testing.T) { // dedicated get_missing_event handler for timelineEvent2. // we grudgingly return a single event. - psjResult.Server.Mux().Handle("/_matrix/federation/v1/get_missing_events/{roomID}", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - roomID := vars["roomID"] - - // Unmarshal the request body into an object - content, _ := ioutil.ReadAll(req.Body) - var request gomatrixserverlib.MissingEvents - if err := json.Unmarshal(content, &request); err != nil { - t.Fatalf("get_missing_events: Unable to unmarshal request body: %s", err.Error()) - } - - t.Logf("Got get_missing_events request in %s: %#v", roomID, request) - if roomID != psjResult.ServerRoom.RoomID { - t.Fatalf("get_missing_events for wrong room: got %s, want %s", roomID, psjResult.ServerRoom.RoomID) - } - - if request.LatestEvents[0] != timelineEvent2.EventID() { - t.Fatalf("get_missing_events for wrong event: got %v, want %s", request.LatestEvents, timelineEvent2.EventID()) - } - - // return timelineEvent1 - jsonb, _ := json.Marshal(gomatrixserverlib.RespMissingEvents{ - Events: gomatrixserverlib.EventJSONs{timelineEvent1.JSON()}, - }) - w.WriteHeader(200) - w.Write(jsonb) - t.Logf("Processed get_missing_events request: returned event %s", timelineEvent1.EventID()) - })) + handleGetMissingEventsRequests(t, psjResult.Server, psjResult.ServerRoom, + []string{timelineEvent2.EventID()}, []*gomatrixserverlib.Event{timelineEvent1}, + ) // dedicated state_ids and state handlers for timelineEvent1's prev event (ie, the last outlier event) psjResult.Server.Mux().NewRoute().Methods("GET").Path( From 48f31be356a746534ff042a536059e8fcc95c2df Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 26 Jul 2022 14:18:42 +0100 Subject: [PATCH 6/7] use state request handlers --- ...federation_room_join_partial_state_test.go | 32 +++---------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index 0a301149..fadf52d6 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -634,34 +634,10 @@ func TestPartialStateJoin(t *testing.T) { ) // dedicated state_ids and state handlers for timelineEvent1's prev event (ie, the last outlier event) - psjResult.Server.Mux().NewRoute().Methods("GET").Path( - fmt.Sprintf("/_matrix/federation/v1/state_ids/%s", psjResult.ServerRoom.RoomID), - ).Queries("event_id", outliers[len(outliers)-1].EventID()).Handler( - http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - queryParams := req.URL.Query() - t.Logf("Incoming state_ids request for last outlier event %s", queryParams["event_id"]) - jsonb, _ := json.Marshal(gomatrixserverlib.RespStateIDs{ - AuthEventIDs: eventIDsFromEvents(psjResult.ServerRoom.AuthChain()), - StateEventIDs: eventIDsFromEvents(psjResult.ServerRoom.AllCurrentState()), - }) - w.WriteHeader(200) - w.Write(jsonb) - }), - ) - psjResult.Server.Mux().NewRoute().Methods("GET").Path( - fmt.Sprintf("/_matrix/federation/v1/state/%s", psjResult.ServerRoom.RoomID), - ).Queries("event_id", outliers[len(outliers)-1].EventID()).Handler( - http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - queryParams := req.URL.Query() - t.Logf("Incoming state request for last outlier event %s", queryParams["event_id"]) - jsonb, _ := json.Marshal(gomatrixserverlib.RespState{ - AuthEvents: gomatrixserverlib.NewEventJSONsFromEvents(psjResult.ServerRoom.AuthChain()), - StateEvents: gomatrixserverlib.NewEventJSONsFromEvents(psjResult.ServerRoom.AllCurrentState()), - }) - w.WriteHeader(200) - w.Write(jsonb) - }), - ) + handleStateIdsRequests(t, psjResult.Server, psjResult.ServerRoom, outliers[len(outliers)-1].EventID(), + psjResult.ServerRoom.AllCurrentState(), nil, nil) + handleStateRequests(t, psjResult.Server, psjResult.ServerRoom, outliers[len(outliers)-1].EventID(), + psjResult.ServerRoom.AllCurrentState(), nil, nil) // now, send over the most recent event, which will make the server get_missing_events // (we will send timelineEvent1), and then request state (we will send all the outliers). From dc2666b02cf186b0d98ad3cb2b0d96021c7e79ff Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 27 Jul 2022 12:27:52 +0100 Subject: [PATCH 7/7] address review comments --- ...federation_room_join_partial_state_test.go | 82 ++++++++----------- 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/tests/federation_room_join_partial_state_test.go b/tests/federation_room_join_partial_state_test.go index fadf52d6..b22432e7 100644 --- a/tests/federation_room_join_partial_state_test.go +++ b/tests/federation_room_join_partial_state_test.go @@ -569,8 +569,9 @@ func TestPartialStateJoin(t *testing.T) { // regression test for https://github.com/matrix-org/synapse/issues/13001 // // There was an edge case where, if we initially receive lots of events as outliers, - // and they then get de-outliered as partial state events, - t.Run("Resync works with many prev_events with partial state", func(t *testing.T) { + // and they then get de-outliered as partial state events, we would get stuck in + // an infinite loop of de-partial-stating. + t.Run("Resync completes even when events arrive before their prev_events", func(t *testing.T) { deployment := Deploy(t, b.BlueprintAlice) defer deployment.Destroy(t) alice := deployment.Client(t, "hs1", "@alice:hs1") @@ -582,24 +583,6 @@ func TestPartialStateJoin(t *testing.T) { // utility function to wait for a given event to arrive at the remote server. // This works simply by polling /event until we get a 200. - awaitEventArrival := func(eventID string) { - start := time.Now() - for time.Since(start) < 10*time.Second { - res := alice.DoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", psjResult.ServerRoom.RoomID, "event", eventID}) - eventResBody := client.ParseJSON(t, res) - if res.StatusCode == 200 { - t.Logf("Alice successfully received event %s", eventID) - return - } - if res.StatusCode == 404 && gjson.GetBytes(eventResBody, "errcode").String() == "M_NOT_FOUND" { - t.Logf("Fetching event %s failed with M_NOT_FOUND; will retry", eventID) - time.Sleep(1000 * time.Millisecond) - continue - } - t.Fatalf("GET /event failed with %d: %s", res.StatusCode, string(eventResBody)) - } - t.Fatalf("timeout waiting for event %s to be received", eventID) - } // here's the first event which we *ought* to un-partial-state, but won't lateEvent := psjResult.CreateMessageEvent(t, "charlie", nil) @@ -645,7 +628,7 @@ func TestPartialStateJoin(t *testing.T) { t.Logf("Charlie sent timeline event 2") // wait for it to become visible, which implies that all the outliers have been pulled in. - awaitEventArrival(timelineEvent2.EventID()) + awaitEventArrival(t, time.Second, alice, psjResult.ServerRoom.RoomID, timelineEvent2.EventID()) // now we send over all the other events in the gap. psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{lateEvent.JSON()}, nil) @@ -663,7 +646,7 @@ func TestPartialStateJoin(t *testing.T) { } // wait for the last outlier to arrive - awaitEventArrival(outliers[len(outliers)-1].EventID()) + awaitEventArrival(t, 10*time.Second, alice, psjResult.ServerRoom.RoomID, outliers[len(outliers)-1].EventID()) // release the federation /state response psjResult.FinishStateRequest() @@ -698,31 +681,7 @@ func testReceiveEventDuringPartialStateJoin( // send the event to the homeserver psjResult.Server.MustSendTransaction(t, deployment, "hs1", []json.RawMessage{event.JSON()}, nil) - /* TODO: check that a lazy-loading sync can see the event. Currently this doesn't work, because /sync blocks. - * https://github.com/matrix-org/synapse/issues/13146 - alice.MustSyncUntil(t, - client.SyncReq{ - Filter: buildLazyLoadingSyncFilter(nil), - }, - client.SyncTimelineHasEventID(psjResult.ServerRoom.RoomID, event.EventID()), - ) - */ - - // still, Alice should be able to see the event with an /event request. We might have to try it a few times. - alice.DoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", psjResult.ServerRoom.RoomID, "event", event.EventID()}, - client.WithRetryUntil(time.Second, func(res *http.Response) bool { - if res.StatusCode == 200 { - return true - } - eventResBody := client.ParseJSON(t, res) - if res.StatusCode == 404 && gjson.GetBytes(eventResBody, "errcode").String() == "M_NOT_FOUND" { - return false - } - t.Fatalf("GET /event failed with %d: %s", res.StatusCode, string(eventResBody)) - return false - }), - ) - t.Logf("Successfully fetched received event %s", event.EventID()) + awaitEventArrival(t, time.Second, alice, psjResult.ServerRoom.RoomID, event.EventID()) // fire off a /state_ids request for the last event. // it must either: @@ -787,6 +746,35 @@ func testReceiveEventDuringPartialStateJoin( must.CheckOffAll(t, gotState, expectedState) } +// awaitEventArrival waits for alice to be able to see a given event +func awaitEventArrival(t *testing.T, timeout time.Duration, alice *client.CSAPI, roomID string, eventID string) { + /* TODO: check that a lazy-loading sync can see the event. Currently this doesn't work, because /sync blocks. + * https://github.com/matrix-org/synapse/issues/13146 + alice.MustSyncUntil(t, + client.SyncReq{ + Filter: buildLazyLoadingSyncFilter(nil), + }, + client.SyncTimelineHasEventID(roomID, eventID), + ) + */ + + // still, Alice should be able to see the event with an /event request. We might have to try it a few times. + alice.DoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "event", eventID}, + client.WithRetryUntil(timeout, func(res *http.Response) bool { + if res.StatusCode == 200 { + return true + } + eventResBody := client.ParseJSON(t, res) + if res.StatusCode == 404 && gjson.GetBytes(eventResBody, "errcode").String() == "M_NOT_FOUND" { + return false + } + t.Fatalf("GET /event failed with %d: %s", res.StatusCode, string(eventResBody)) + return false + }), + ) + t.Logf("Alice successfully received event %s", eventID) +} + // buildLazyLoadingSyncFilter constructs a json-marshalled filter suitable the 'Filter' field of a client.SyncReq func buildLazyLoadingSyncFilter(timelineOptions map[string]interface{}) string { timelineFilter := map[string]interface{}{