From 048887b765d889ef2a80afcd33e667a4c418ef74 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 23 Jul 2021 00:18:53 -0500 Subject: [PATCH 01/19] Update MSC2716 federation test cases --- tests/msc2716_test.go | 55 +++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 7f3777b7..f5eae2a1 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -76,8 +76,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -190,8 +191,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -240,8 +242,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -290,8 +293,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) batchSendHistoricalMessages( @@ -315,8 +319,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -349,8 +354,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -407,8 +413,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -483,8 +490,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) @@ -521,9 +529,9 @@ func TestBackfillingHistory(t *testing.T) { historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) baseInsertionEventID := historicalEventIDs[len(historicalEventIDs)-1] - // [1 insertion event + 2 historical events + 1 insertion event] - if len(historicalEventIDs) != 4 { - t.Fatalf("Expected eventID list should be length 15 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] + if len(historicalEventIDs) != 5 { + t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -571,12 +579,12 @@ func TestBackfillingHistory(t *testing.T) { return ev.Get("event_id").Str == markerEventID }) - messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, })) - must.MatchResponse(t, messagesRes, match.HTTPResponse{ + must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { return r.Get("event_id").Str @@ -590,8 +598,9 @@ func TestBackfillingHistory(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ - "preset": "public_chat", - "name": "the hangout spot", + "room_version": "org.matrix.msc2716", + "preset": "public_chat", + "name": "the hangout spot", }) alice.JoinRoom(t, roomID, nil) From d385044820e9bfc5c63da6238971f0d50e76fe84 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Mon, 26 Jul 2021 16:35:59 -0500 Subject: [PATCH 02/19] Uncomment some federated test cases --- tests/msc2716_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index f5eae2a1..731538cb 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -350,7 +350,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ @@ -409,7 +409,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, map[string]interface{}{ From faec5cac9db968525146b510d311b9ec650cd50e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 28 Jul 2021 20:58:26 -0500 Subject: [PATCH 03/19] More proper waiting to avoid flakey test --- tests/msc2716_test.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index b9b62b87..39676319 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -337,7 +337,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -392,7 +391,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -465,7 +463,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when already joined on federated server", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -484,7 +482,7 @@ func TestBackfillingHistory(t *testing.T) { // Mimic scrollback just through the latest messages remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, - // Limited so we can only see a few of the latest messages + // Limited so we can only see a portion of the latest messages "limit": []string{"5"}, })) @@ -554,6 +552,16 @@ func TestBackfillingHistory(t *testing.T) { return ev.Get("event_id").Str == markerEventID }) + // Make sure all of the base insertion event has been backfilled + // after the marker was received + fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { + if ev.Get("event_id").Str == baseInsertionEventID { + return true + } + + return false + }) + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, From edade9c932539b7485c26f177225afa275a27f37 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 28 Jul 2021 21:19:17 -0500 Subject: [PATCH 04/19] Get both tests working and de-duplicate marker sending verifying --- tests/msc2716_test.go | 101 +++++++++++++++++++++++++++++------------- 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 39676319..e5961e1e 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -534,39 +534,15 @@ func TestBackfillingHistory(t *testing.T) { }, }) - // Send a marker event to let all of the homeservers know about the - // insertion point where all of the historical messages are at - markerEvent := b.Event{ - Type: markerEventType, - Content: map[string]interface{}{ - markerInsertionContentField: baseInsertionEventID, - }, - } - // We can't use as.SendEventSynced(...) because application services can't use the /sync API - markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, "txn-m123"}, client.WithJSONBody(t, markerEvent.Content)) - markerSendBody := client.ParseJSON(t, markerSendRes) - markerEventID := client.GetJSONFieldStr(t, markerSendBody, "event_id") - - // Make sure the marker event has reached the remote homeserver - remoteCharlie.SyncUntilTimelineHas(t, roomID, func(ev gjson.Result) bool { - return ev.Get("event_id").Str == markerEventID - }) - - // Make sure all of the base insertion event has been backfilled - // after the marker was received - fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { - if ev.Get("event_id").Str == baseInsertionEventID { - return true - } - - return false - }) + // Send the marker event + sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, })) + // Make sure all of the historical messages are visible when we scrollback again must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { @@ -577,7 +553,7 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("When messages have already been scrolled back through, new historical messages are visible in next scroll back on federated server", func(t *testing.T) { - t.Skip("Skipping until federation is implemented") + //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createRoomOpts) @@ -615,15 +591,48 @@ func TestBackfillingHistory(t *testing.T) { ) batchSendResBody := client.ParseJSON(t, batchSendRes) historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + baseInsertionEventID := historicalEventIDs[len(historicalEventIDs)-1] - // TODO: Send marker event + // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] + if len(historicalEventIDs) != 5 { + t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + } - messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, "limit": []string{"100"}, })) + beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) + eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody) + // Since the original body can only be read once, create a new one from the body bytes we just read + beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) + // Make sure the history isn't visible before we expect it to be there. + // This is to avoid some bug in the homeserver using some unknown + // mechanism to distribute the historical messages to other homeservers. + must.MatchResponse(t, beforeMarkerMessagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONArrayEach("chunk", func(r gjson.Result) error { + // Throw if we find one of the historical events in the message response + for _, historicalEventID := range historicalEventIDs { + if r.Get("event_id").Str == historicalEventID { + return fmt.Errorf("Historical event (%s) found on remote homeserver before marker event was sent out\nmessage response (%d): %v\nhistoricalEventIDs (%d): %v", historicalEventID, len(eventDebugStringsFromBeforeMarkerResponse), eventDebugStringsFromBeforeMarkerResponse, len(historicalEventIDs), historicalEventIDs) + } + } + return nil + }), + }, + }) - must.MatchResponse(t, messagesRes, match.HTTPResponse{ + // Send the marker event + sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) + + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + // Make sure all of the historical messages are visible when we scrollback again + must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { return r.Get("event_id").Str @@ -742,6 +751,36 @@ func ensureVirtualUserRegistered(t *testing.T, c *client.CSAPI, virtualUserLocal } } +func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) { + // Send a marker event to let all of the homeservers know about the + // insertion point where all of the historical messages are at + markerEvent := b.Event{ + Type: markerEventType, + Content: map[string]interface{}{ + markerInsertionContentField: insertionEventID, + }, + } + // We can't use as.SendEventSynced(...) because application services can't use the /sync API + markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, "txn-m123"}, client.WithJSONBody(t, markerEvent.Content)) + markerSendBody := client.ParseJSON(t, markerSendRes) + markerEventID := client.GetJSONFieldStr(t, markerSendBody, "event_id") + + // Make sure the marker event has reached the remote homeserver + c.SyncUntilTimelineHas(t, roomID, func(ev gjson.Result) bool { + return ev.Get("event_id").Str == markerEventID + }) + + // Make sure all of the base insertion event has been backfilled + // after the marker was received + fetchUntilMessagesResponseHas(t, c, roomID, func(ev gjson.Result) bool { + if ev.Get("event_id").Str == insertionEventID { + return true + } + + return false + }) +} + func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count int) (eventIDs []string) { eventIDs = make([]string, count) for i := 0; i < len(eventIDs); i++ { From b72cc1feca921ccf40d1380b3ce0d382fca60830 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 28 Jul 2021 22:56:45 -0500 Subject: [PATCH 05/19] Make error comment match helper function name --- tests/msc2716_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e5961e1e..e46bd3f0 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -666,7 +666,7 @@ func fetchUntilMessagesResponseHas(t *testing.T, c *client.CSAPI, roomID string, checkCounter := 0 for { if time.Since(start) > c.SyncUntilTimeout { - t.Fatalf("fetchMessagesUntilResponseHas timed out. Called check function %d times", checkCounter) + t.Fatalf("fetchUntilMessagesResponseHas timed out. Called check function %d times", checkCounter) } messagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ From 6bc06f7aaff53d6dc4ba845de7f33184211aadf7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 4 Aug 2021 01:57:24 -0500 Subject: [PATCH 06/19] Add test for private room backfilling --- tests/msc2716_test.go | 151 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 13 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e46bd3f0..e281e072 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -46,12 +46,18 @@ var ( markerInsertionContentField = "org.matrix.msc2716.marker.insertion" ) -var createRoomOpts = map[string]interface{}{ +var createPublicRoomOpts = map[string]interface{}{ "preset": "public_chat", "name": "the hangout spot", "room_version": "org.matrix.msc2716", } +var createPrivateRoomOpts = map[string]interface{}{ + "preset": "private_chat", + "name": "the hangout spot", + "room_version": "org.matrix.msc2716", +} + func TestBackfillingHistory(t *testing.T) { deployment := Deploy(t, b.BlueprintHSWithApplicationService) defer deployment.Destroy(t) @@ -82,7 +88,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Backfilled historical events resolve with proper state in correct order", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Create some normal messages in the timeline. We're creating them in @@ -193,7 +199,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Backfilled historical events from multiple users in the same chunk", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Create the "live" event we are going to insert our backfilled events next to @@ -237,10 +243,10 @@ func TestBackfillingHistory(t *testing.T) { }) }) - t.Run("Backfilled historical events with m.historical do not come down in an incremental sync", func(t *testing.T) { + t.Run("Backfilled historical events do not come down in an incremental sync", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Create the "live" event we are going to insert our backfilled events next to @@ -287,7 +293,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Unrecognised prev_event ID will throw an error", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) batchSendHistoricalMessages( t, @@ -309,7 +315,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Normal users aren't allowed to backfill messages", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) @@ -331,6 +337,48 @@ func TestBackfillingHistory(t *testing.T) { ) }) + t.Run("Should be able to backfill into private room", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createPrivateRoomOpts) + as.InviteRoom(t, roomID, alice.UserID) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + []string{virtualUserID}, + roomID, + eventIdBefore, + timeAfterEventBefore, + "", + 3, + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + + messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) + }) + t.Run("TODO: Test if historical avatar/display name set back in time are picked up on historical messages", func(t *testing.T) { t.Skip("Skipping until implemented") // TODO: Try adding avatar and displayName and see if historical messages get this info @@ -339,7 +387,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) @@ -393,7 +441,7 @@ func TestBackfillingHistory(t *testing.T) { t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) @@ -466,7 +514,7 @@ func TestBackfillingHistory(t *testing.T) { //t.Skip("Skipping until federation is implemented") t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Join the room from a remote homeserver before any backfilled messages are sent @@ -556,7 +604,7 @@ func TestBackfillingHistory(t *testing.T) { //t.Skip("Skipping until federation is implemented") t.Parallel() - roomID := as.CreateRoom(t, createRoomOpts) + roomID := as.CreateRoom(t, createPublicRoomOpts) alice.JoinRoom(t, roomID, nil) // Join the room from a remote homeserver before any backfilled messages are sent @@ -798,6 +846,72 @@ func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count in return eventIDs } +func asdf( + virtualUserIDs []string, + insertTime time.Time, +) []map[string]interface{} { + // Timestamp in milliseconds + insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) + + state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) + for i, virtualUserID := range virtualUserIDs { + inviteEvent := map[string]interface{}{ + "type": "m.room.member", + "sender": c.UserID, + "origin_server_ts": insertOriginServerTs, + "content": map[string]interface{}{ + "membership": "invite", + }, + "state_key": virtualUserID, + } + state_evs[2*i] = inviteEvent + + joinEvent := map[string]interface{}{ + "type": "m.room.member", + "sender": virtualUserID, + "origin_server_ts": insertOriginServerTs, + "content": map[string]interface{}{ + "membership": "join", + }, + "state_key": virtualUserID, + } + + state_evs[2*i+1] = joinEvent + } + + return state_evs +} + +func createBatchSendMessages( + virtualUserIDs []string, + insertTime time.Time, + count int, +) []map[string]interface{} { + // Timestamp in milliseconds + insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) + timeBetweenMessagesMS := uint64(timeBetweenMessages / time.Millisecond) + + evs := make([]map[string]interface{}, count) + for i := 0; i < len(evs); i++ { + virtualUserID := virtualUserIDs[i%len(virtualUserIDs)] + + newEvent := map[string]interface{}{ + "type": "m.room.message", + "sender": virtualUserID, + "origin_server_ts": insertOriginServerTs + (timeBetweenMessagesMS * uint64(i)), + "content": map[string]interface{}{ + "msgtype": "m.text", + "body": fmt.Sprintf("Historical %d (chunk=%d)", i, chunkCount), + historicalContentField: true, + }, + } + + evs[i] = newEvent + } + + return evs +} + var chunkCount int64 = 0 func batchSendHistoricalMessages( @@ -834,8 +948,19 @@ func batchSendHistoricalMessages( evs[i] = newEvent } - state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { + inviteEvent := map[string]interface{}{ + "type": "m.room.member", + "sender": c.UserID, + "origin_server_ts": insertOriginServerTs, + "content": map[string]interface{}{ + "membership": "invite", + }, + "state_key": virtualUserID, + } + state_evs[2*i] = inviteEvent + joinEvent := map[string]interface{}{ "type": "m.room.member", "sender": virtualUserID, @@ -846,7 +971,7 @@ func batchSendHistoricalMessages( "state_key": virtualUserID, } - state_evs[i] = joinEvent + state_evs[2*i+1] = joinEvent } query := make(url.Values, 2) From 3b2358f99dabdbbc2c89b7c2e36ee86543193455 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 4 Aug 2021 02:26:07 -0500 Subject: [PATCH 07/19] Simplify batchSendHistoricalMessages function signature by using sorta composable functions --- tests/msc2716_test.go | 149 ++++++++++++++++-------------------------- 1 file changed, 56 insertions(+), 93 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e281e072..4189815c 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -112,15 +112,15 @@ func TestBackfillingHistory(t *testing.T) { eventIDsAfter := createMessagesInRoom(t, alice, roomID, 2) // Insert the most recent chunk of backfilled history + insertTime1 := timeAfterEventBefore.Add(timeBetweenMessages * 3) batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore.Add(timeBetweenMessages*3), "", - 3, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), // Status 200, ) @@ -130,15 +130,15 @@ func TestBackfillingHistory(t *testing.T) { // Insert another older chunk of backfilled history from the same user. // Make sure the meta data and joins still work on the subsequent chunk + insertTime2 := timeAfterEventBefore batchSendRes2 := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, nextChunkID, - 3, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime2), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime2, 3), // Status 200, ) @@ -213,16 +213,17 @@ func TestBackfillingHistory(t *testing.T) { virtualUserID3 := "@carol:hs1" ensureVirtualUserRegistered(t, as, "carol") + virtualUserList := []string{virtualUserID, virtualUserID2, virtualUserID3} + // Insert a backfilled event batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID, virtualUserID2, virtualUserID3}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 3, + createJoinStateEventsForBackfillRequest(virtualUserList, timeAfterEventBefore), + createMessageEventsForBackfillRequest(virtualUserList, timeAfterEventBefore, 3), // Status 200, ) @@ -261,12 +262,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 1, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), // Status 200, ) @@ -295,15 +295,15 @@ func TestBackfillingHistory(t *testing.T) { roomID := as.CreateRoom(t, createPublicRoomOpts) + insertTime := time.Now() batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, "$some-non-existant-event-id", - time.Now(), "", - 1, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime, 1), // Status // TODO: Seems like this makes more sense as a 404 // But the current Synapse code around unknown prev events will throw -> @@ -325,12 +325,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendHistoricalMessages( t, alice, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 1, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), // Status // Normal user alice should not be able to backfill messages 403, @@ -349,16 +348,19 @@ func TestBackfillingHistory(t *testing.T) { eventIdBefore := eventIDsBefore[0] timeAfterEventBefore := time.Now() + var stateEvents []map[string]interface{} + stateEvents = append(stateEvents, createInviteStateEventsForBackfillRequest(as.UserID, []string{virtualUserID}, timeAfterEventBefore)...) + stateEvents = append(stateEvents, createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore)...) + // Insert a backfilled event batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 3, + stateEvents, + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 3), // Status 200, ) @@ -400,12 +402,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -472,12 +473,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, chunkId, - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -537,12 +537,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -628,12 +627,11 @@ func TestBackfillingHistory(t *testing.T) { batchSendRes := batchSendHistoricalMessages( t, as, - []string{virtualUserID}, roomID, eventIdBefore, - timeAfterEventBefore, "", - 2, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status 200, ) @@ -846,26 +844,41 @@ func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count in return eventIDs } -func asdf( +func createInviteStateEventsForBackfillRequest( + invitedByUserID string, virtualUserIDs []string, insertTime time.Time, ) []map[string]interface{} { // Timestamp in milliseconds insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) + state_evs := make([]map[string]interface{}, len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { inviteEvent := map[string]interface{}{ "type": "m.room.member", - "sender": c.UserID, + "sender": invitedByUserID, "origin_server_ts": insertOriginServerTs, "content": map[string]interface{}{ "membership": "invite", }, "state_key": virtualUserID, } - state_evs[2*i] = inviteEvent + state_evs[i] = inviteEvent + } + + return state_evs +} + +func createJoinStateEventsForBackfillRequest( + virtualUserIDs []string, + insertTime time.Time, +) []map[string]interface{} { + // Timestamp in milliseconds + insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) + + state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + for i, virtualUserID := range virtualUserIDs { joinEvent := map[string]interface{}{ "type": "m.room.member", "sender": virtualUserID, @@ -876,13 +889,13 @@ func asdf( "state_key": virtualUserID, } - state_evs[2*i+1] = joinEvent + state_evs[i] = joinEvent } return state_evs } -func createBatchSendMessages( +func createMessageEventsForBackfillRequest( virtualUserIDs []string, insertTime time.Time, count int, @@ -917,63 +930,13 @@ var chunkCount int64 = 0 func batchSendHistoricalMessages( t *testing.T, c *client.CSAPI, - virtualUserIDs []string, roomID string, insertAfterEventId string, - insertTime time.Time, chunkID string, - count int, + state_events_at_start []map[string]interface{}, + events []map[string]interface{}, expectedStatus int, ) (res *http.Response) { - // Timestamp in milliseconds - insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - - timeBetweenMessagesMS := uint64(timeBetweenMessages / time.Millisecond) - - evs := make([]map[string]interface{}, count) - for i := 0; i < len(evs); i++ { - virtualUserID := virtualUserIDs[i%len(virtualUserIDs)] - - newEvent := map[string]interface{}{ - "type": "m.room.message", - "sender": virtualUserID, - "origin_server_ts": insertOriginServerTs + (timeBetweenMessagesMS * uint64(i)), - "content": map[string]interface{}{ - "msgtype": "m.text", - "body": fmt.Sprintf("Historical %d (chunk=%d)", i, chunkCount), - historicalContentField: true, - }, - } - - evs[i] = newEvent - } - - state_evs := make([]map[string]interface{}, 2*len(virtualUserIDs)) - for i, virtualUserID := range virtualUserIDs { - inviteEvent := map[string]interface{}{ - "type": "m.room.member", - "sender": c.UserID, - "origin_server_ts": insertOriginServerTs, - "content": map[string]interface{}{ - "membership": "invite", - }, - "state_key": virtualUserID, - } - state_evs[2*i] = inviteEvent - - joinEvent := map[string]interface{}{ - "type": "m.room.member", - "sender": virtualUserID, - "origin_server_ts": insertOriginServerTs, - "content": map[string]interface{}{ - "membership": "join", - }, - "state_key": virtualUserID, - } - - state_evs[2*i+1] = joinEvent - } - query := make(url.Values, 2) query.Add("prev_event", insertAfterEventId) // If provided, connect the chunk to the last insertion point @@ -986,8 +949,8 @@ func batchSendHistoricalMessages( "POST", []string{"_matrix", "client", "unstable", "org.matrix.msc2716", "rooms", roomID, "batch_send"}, client.WithJSONBody(t, map[string]interface{}{ - "events": evs, - "state_events_at_start": state_evs, + "events": events, + "state_events_at_start": state_events_at_start, }), client.WithContentType("application/json"), client.WithQueries(query), From 637ac8f9db653691840b44450c6af1c974a9d701 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 Aug 2021 17:38:31 -0500 Subject: [PATCH 08/19] Remove test skips now that it's supported in Synapse Synapse support was added in https://github.com/matrix-org/synapse/pull/10498 --- tests/msc2716_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 4189815c..1c307330 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -511,7 +511,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("Historical messages are visible when already joined on federated server", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) @@ -600,7 +599,6 @@ func TestBackfillingHistory(t *testing.T) { }) t.Run("When messages have already been scrolled back through, new historical messages are visible in next scroll back on federated server", func(t *testing.T) { - //t.Skip("Skipping until federation is implemented") t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) From b06fc957becb54b31bb98e76b9e81d07a614400c Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 Aug 2021 18:48:11 -0500 Subject: [PATCH 09/19] Verify we only return state events that we pass in Synapse change added in https://github.com/matrix-org/synapse/pull/10552 --- tests/msc2716_test.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 1c307330..b1873809 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -290,6 +290,39 @@ func TestBackfillingHistory(t *testing.T) { }) }) + t.Run("Batch send endpoint only returns state events that we passed in via state_events_at_start", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createPublicRoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + stateEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "state_events") + + // We only expect 1 state event to be returned because we only passed in 1 + // event into `?state_events_at_start` + if len(stateEventIDs) != 1 { + t.Fatalf("Expected only 1 state event to be returned but received %d: %s", len(stateEventIDs), stateEventIDs) + } + }) + t.Run("Unrecognised prev_event ID will throw an error", func(t *testing.T) { t.Parallel() @@ -336,6 +369,11 @@ func TestBackfillingHistory(t *testing.T) { ) }) + t.Run("TODO: Trying to send insertion event with same `next_chunk_id` will reject", func(t *testing.T) { + t.Skip("Skipping until implemented") + // (room_id, next_chunk_id) should be unique + }) + t.Run("Should be able to backfill into private room", func(t *testing.T) { t.Parallel() @@ -386,6 +424,10 @@ func TestBackfillingHistory(t *testing.T) { // TODO: Try adding avatar and displayName and see if historical messages get this info }) + t.Run("TODO: What happens when you point multiple chunks at the same insertion event?", func(t *testing.T) { + t.Skip("Skipping until implemented") + }) + t.Run("Historical messages are visible when joining on federated server - auto-generated base insertion event", func(t *testing.T) { t.Parallel() From 0a215fe88ed045e1ff63635d9ba2c086027ac6bb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 10 Aug 2021 21:04:49 -0500 Subject: [PATCH 10/19] Add tests for MSC2716 being backwards compatible with existing room versions Synapse changes added in https://github.com/matrix-org/synapse/pull/10566 --- tests/msc2716_test.go | 125 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 3 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index b1873809..ee4e9ac1 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -726,9 +726,105 @@ func TestBackfillingHistory(t *testing.T) { }, }) }) + + t.Run("Existing room versions", func(t *testing.T) { + createUnsupportedMSC2716RoomOpts := map[string]interface{}{ + "preset": "public_chat", + "name": "the hangout spot", + // v6 is an existing room version that does not support MSC2716 + "room_version": "6", + } + + t.Run("Room creator can send MSC2716 events", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createUnsupportedMSC2716RoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + + messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) + }) + + t.Run("Not allowed to redact MSC2716 insertion, chunk, marker events", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createUnsupportedMSC2716RoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 200, + ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + insertionEventID := historicalEventIDs[0] + chunkEventID := historicalEventIDs[2] + baseInsertionEventID := historicalEventIDs[3] + + // Send the marker event + markerEventID := sendMarkerAndEnsureBackfilled(t, as, alice, roomID, baseInsertionEventID) + + redactEventID(t, alice, roomID, insertionEventID, 403) + redactEventID(t, alice, roomID, chunkEventID, 403) + redactEventID(t, alice, roomID, markerEventID, 403) + }) + }) }) } +var txnCounter int = 0 + +func getTxnID(prefix string) (txnID string) { + txnId := fmt.Sprintf("%s-%d", prefix, txnCounter) + + txnCounter++ + + return txnId +} + func makeInterfaceSlice(slice []string) []interface{} { interfaceSlice := make([]interface{}, len(slice)) for i := range slice { @@ -837,7 +933,7 @@ func ensureVirtualUserRegistered(t *testing.T, c *client.CSAPI, virtualUserLocal } } -func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) { +func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) (markerEventID string) { // Send a marker event to let all of the homeservers know about the // insertion point where all of the historical messages are at markerEvent := b.Event{ @@ -846,10 +942,11 @@ func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSA markerInsertionContentField: insertionEventID, }, } + txnId := getTxnID("sendMarkerAndEnsureBackfilled-txn") // We can't use as.SendEventSynced(...) because application services can't use the /sync API - markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, "txn-m123"}, client.WithJSONBody(t, markerEvent.Content)) + markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, txnId}, client.WithJSONBody(t, markerEvent.Content)) markerSendBody := client.ParseJSON(t, markerSendRes) - markerEventID := client.GetJSONFieldStr(t, markerSendBody, "event_id") + markerEventID = client.GetJSONFieldStr(t, markerSendBody, "event_id") // Make sure the marker event has reached the remote homeserver c.SyncUntilTimelineHas(t, roomID, func(ev gjson.Result) bool { @@ -865,6 +962,8 @@ func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSA return false }) + + return markerEventID } func createMessagesInRoom(t *testing.T, c *client.CSAPI, roomID string, count int) (eventIDs []string) { @@ -965,6 +1064,26 @@ func createMessageEventsForBackfillRequest( return evs } +func redactEventID(t *testing.T, c *client.CSAPI, roomID, eventID string, expectedStatus int) { + t.Helper() + + txnID := getTxnID("redactEventID-txn") + redactionRes := c.DoFunc( + t, + "PUT", + []string{"_matrix", "client", "r0", "rooms", roomID, "redact", eventID, txnID}, + client.WithJSONBody(t, map[string]interface{}{"reason": "chaos"}), + client.WithContentType("application/json"), + ) + redactionResBody := client.ParseJSON(t, redactionRes) + redactionResErrcode := client.GetJSONFieldStr(t, redactionResBody, "error") + redactionResError := client.GetJSONFieldStr(t, redactionResBody, "errcode") + + if redactionRes.StatusCode != expectedStatus { + t.Fatalf("msc2716.redactEventID: Expected redaction response to be %d but received %d -> %s: %s", expectedStatus, redactionRes.StatusCode, redactionResErrcode, redactionResError) + } +} + var chunkCount int64 = 0 func batchSendHistoricalMessages( From a958b34b99bafe74e27c8c152fe123dabff4ae67 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 18 Aug 2021 17:13:34 -0500 Subject: [PATCH 11/19] Add helper decorations --- tests/msc2716_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index b1873809..069ae000 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -838,6 +838,8 @@ func ensureVirtualUserRegistered(t *testing.T, c *client.CSAPI, virtualUserLocal } func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSAPI, roomID, insertionEventID string) { + t.Helper() + // Send a marker event to let all of the homeservers know about the // insertion point where all of the historical messages are at markerEvent := b.Event{ @@ -977,6 +979,8 @@ func batchSendHistoricalMessages( events []map[string]interface{}, expectedStatus int, ) (res *http.Response) { + t.Helper() + query := make(url.Values, 2) query.Add("prev_event", insertAfterEventId) // If provided, connect the chunk to the last insertion point From 09b4a4b6ab688fafa997abfe38d2064eec1617d7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 7 Sep 2021 15:52:59 -0500 Subject: [PATCH 12/19] Add tests to verify ?chunk_id actually corresponds to an insertion event that exists Add tests for: - https://github.com/matrix-org/synapse/pull/10776 --- tests/msc2716_test.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index ee4e9ac1..96ff9c20 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -345,6 +345,29 @@ func TestBackfillingHistory(t *testing.T) { ) }) + t.Run("Unrecognised chunk_id will throw an error", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createPublicRoomOpts) + alice.JoinRoom(t, roomID, nil) + + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "XXX_DOES_NOT_EXIST_CHUNK_ID", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), + // Status + 400, + ) + }) + t.Run("Normal users aren't allowed to backfill messages", func(t *testing.T) { t.Parallel() From ebe141de3ce8cce0e31dfac8d1bbcd5c35d1e348 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 7 Sep 2021 18:56:23 -0500 Subject: [PATCH 13/19] Update tests to accomodate /batch_send returning separate meta event fields Corresponding Synapse changes in https://github.com/matrix-org/synapse/pull/10777 See https://github.com/matrix-org/matrix-doc/pull/2716#discussion_r684765034 Part of https://github.com/matrix-org/matrix-doc/pull/2716 --- tests/msc2716_test.go | 109 +++++++++++++++++++++++++++--------------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 96ff9c20..93104d72 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -112,43 +112,53 @@ func TestBackfillingHistory(t *testing.T) { eventIDsAfter := createMessagesInRoom(t, alice, roomID, 2) // Insert the most recent chunk of backfilled history - insertTime1 := timeAfterEventBefore.Add(timeBetweenMessages * 3) + insertTime0 := timeAfterEventBefore.Add(timeBetweenMessages * 3) batchSendRes := batchSendHistoricalMessages( t, as, roomID, eventIdBefore, "", - createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), - createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime0), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime0, 3), // Status 200, ) - batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) - nextChunkID := getNextChunkIdFromBatchSendResponseBody(t, batchSendResBody) + batchSendResBody0 := client.ParseJSON(t, batchSendRes) + insertionEventID0 := getInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody0) + historicalEventIDs0 := getEventIDsFromBatchSendResponseBody(t, batchSendResBody0) + chunkEventID0 := getChunkEventIdFromBatchSendResponseBody(t, batchSendResBody0) + baseInsertionEventID0 := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody0) + nextChunkID0 := getNextChunkIDFromBatchSendResponseBody(t, batchSendResBody0) // Insert another older chunk of backfilled history from the same user. // Make sure the meta data and joins still work on the subsequent chunk - insertTime2 := timeAfterEventBefore - batchSendRes2 := batchSendHistoricalMessages( + insertTime1 := timeAfterEventBefore + batchSendRes1 := batchSendHistoricalMessages( t, as, roomID, eventIdBefore, - nextChunkID, - createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime2), - createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime2, 3), + nextChunkID0, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), // Status 200, ) - batchSendResBody2 := client.ParseJSON(t, batchSendRes2) - historicalEventIDs2 := getEventsFromBatchSendResponseBody(t, batchSendResBody2) + batchSendResBody1 := client.ParseJSON(t, batchSendRes1) + insertionEventID1 := getInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody1) + historicalEventIDs1 := getEventIDsFromBatchSendResponseBody(t, batchSendResBody1) + chunkEventID1 := getChunkEventIdFromBatchSendResponseBody(t, batchSendResBody1) var expectedEventIDOrder []string expectedEventIDOrder = append(expectedEventIDOrder, eventIDsBefore...) - expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs2...) - expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs...) + expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID1) + expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs1...) + expectedEventIDOrder = append(expectedEventIDOrder, chunkEventID1) + expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID0) + expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs0...) + expectedEventIDOrder = append(expectedEventIDOrder, chunkEventID0) + expectedEventIDOrder = append(expectedEventIDOrder, baseInsertionEventID0) expectedEventIDOrder = append(expectedEventIDOrder, eventIDsAfter...) // Order events from newest to oldest expectedEventIDOrder = reversed(expectedEventIDOrder) @@ -228,7 +238,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, @@ -271,7 +281,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) backfilledEventId := historicalEventIDs[0] // This is just a dummy event we search for after the backfilledEventId @@ -314,7 +324,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - stateEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "state_events") + stateEventIDs := getStateEventIDsFromBatchSendResponseBody(t, batchSendResBody) // We only expect 1 state event to be returned because we only passed in 1 // event into `?state_events_at_start` @@ -426,7 +436,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, @@ -476,7 +486,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) // Join the room from a remote homeserver after the backfilled messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -547,7 +557,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) // Join the room from a remote homeserver after the backfilled messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -610,12 +620,11 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) - baseInsertionEventID := historicalEventIDs[len(historicalEventIDs)-1] + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) - // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] - if len(historicalEventIDs) != 5 { - t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + if len(historicalEventIDs) != 2 { + t.Fatalf("Expected eventID list should be length 2 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -699,12 +708,11 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) - baseInsertionEventID := historicalEventIDs[len(historicalEventIDs)-1] + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) - // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] - if len(historicalEventIDs) != 5 { - t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + if len(historicalEventIDs) != 2 { + t.Fatalf("Expected eventID list should be length 2 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -782,7 +790,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, @@ -822,10 +830,9 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventsFromBatchSendResponseBody(t, batchSendResBody) - insertionEventID := historicalEventIDs[0] - chunkEventID := historicalEventIDs[2] - baseInsertionEventID := historicalEventIDs[3] + insertionEventID := getInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) + chunkEventID := getChunkEventIdFromBatchSendResponseBody(t, batchSendResBody) + baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) // Send the marker event markerEventID := sendMarkerAndEnsureBackfilled(t, as, alice, roomID, baseInsertionEventID) @@ -1147,14 +1154,38 @@ func batchSendHistoricalMessages( return res } -func getEventsFromBatchSendResponseBody(t *testing.T, body []byte) (eventIDs []string) { - eventIDs = client.GetJSONFieldStringArray(t, body, "events") +func getStateEventIDsFromBatchSendResponseBody(t *testing.T, body []byte) (stateEventIDs []string) { + stateEventIDs = client.GetJSONFieldStringArray(t, body, "state_event_ids") + + return stateEventIDs +} + +func getEventIDsFromBatchSendResponseBody(t *testing.T, body []byte) (eventIDs []string) { + eventIDs = client.GetJSONFieldStringArray(t, body, "event_ids") return eventIDs } -func getNextChunkIdFromBatchSendResponseBody(t *testing.T, body []byte) (nextChunkID string) { +func getNextChunkIDFromBatchSendResponseBody(t *testing.T, body []byte) (nextChunkID string) { nextChunkID = client.GetJSONFieldStr(t, body, "next_chunk_id") return nextChunkID } + +func getInsertionEventIDFromBatchSendResponseBody(t *testing.T, body []byte) (insertionEventID string) { + insertionEventID = client.GetJSONFieldStr(t, body, "insertion_event_id") + + return insertionEventID +} + +func getChunkEventIdFromBatchSendResponseBody(t *testing.T, body []byte) (chunkEventId string) { + chunkEventId = client.GetJSONFieldStr(t, body, "chunk_event_id") + + return chunkEventId +} + +func getBaseInsertionEventIDFromBatchSendResponseBody(t *testing.T, body []byte) (baseInsertionEventID string) { + baseInsertionEventID = client.GetJSONFieldStr(t, body, "base_insertion_event_id") + + return baseInsertionEventID +} From 858aef60f9123aeae58d03374988846b932e8219 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 Sep 2021 11:17:23 -0500 Subject: [PATCH 14/19] More idiomatic Go from review --- tests/msc2716_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 069ae000..4b2ddc77 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -319,7 +319,7 @@ func TestBackfillingHistory(t *testing.T) { // We only expect 1 state event to be returned because we only passed in 1 // event into `?state_events_at_start` if len(stateEventIDs) != 1 { - t.Fatalf("Expected only 1 state event to be returned but received %d: %s", len(stateEventIDs), stateEventIDs) + t.Fatalf("Expected only 1 state event to be returned but received %d: %v", len(stateEventIDs), stateEventIDs) } }) @@ -501,7 +501,7 @@ func TestBackfillingHistory(t *testing.T) { }, } // We can't use as.SendEventSynced(...) because application services can't use the /sync API - insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, "txn-m123"}, client.WithJSONBody(t, insertionEvent.Content)) + insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, "txn-i123"}, client.WithJSONBody(t, insertionEvent.Content)) insertionSendBody := client.ParseJSON(t, insertionSendRes) insertionEventID := client.GetJSONFieldStr(t, insertionSendBody, "event_id") // Make sure the insertion event has reached the homeserver @@ -592,7 +592,7 @@ func TestBackfillingHistory(t *testing.T) { // [1 insertion event + 2 historical events + 1 chunk event + 1 insertion event] if len(historicalEventIDs) != 5 { - t.Fatalf("Expected eventID list should be length 5 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + t.Fatalf("Expected eventID list should be length 5 but saw %d: %v", len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -894,7 +894,7 @@ func createInviteStateEventsForBackfillRequest( // Timestamp in milliseconds insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + stateEvents := make([]map[string]interface{}, len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { inviteEvent := map[string]interface{}{ "type": "m.room.member", @@ -906,10 +906,10 @@ func createInviteStateEventsForBackfillRequest( "state_key": virtualUserID, } - state_evs[i] = inviteEvent + stateEvents[i] = inviteEvent } - return state_evs + return stateEvents } func createJoinStateEventsForBackfillRequest( @@ -919,7 +919,7 @@ func createJoinStateEventsForBackfillRequest( // Timestamp in milliseconds insertOriginServerTs := uint64(insertTime.UnixNano() / int64(time.Millisecond)) - state_evs := make([]map[string]interface{}, len(virtualUserIDs)) + stateEvents := make([]map[string]interface{}, len(virtualUserIDs)) for i, virtualUserID := range virtualUserIDs { joinEvent := map[string]interface{}{ "type": "m.room.member", @@ -931,10 +931,10 @@ func createJoinStateEventsForBackfillRequest( "state_key": virtualUserID, } - state_evs[i] = joinEvent + stateEvents[i] = joinEvent } - return state_evs + return stateEvents } func createMessageEventsForBackfillRequest( @@ -975,7 +975,7 @@ func batchSendHistoricalMessages( roomID string, insertAfterEventId string, chunkID string, - state_events_at_start []map[string]interface{}, + stateEventsAtStart []map[string]interface{}, events []map[string]interface{}, expectedStatus int, ) (res *http.Response) { @@ -994,7 +994,7 @@ func batchSendHistoricalMessages( []string{"_matrix", "client", "unstable", "org.matrix.msc2716", "rooms", roomID, "batch_send"}, client.WithJSONBody(t, map[string]interface{}{ "events": events, - "state_events_at_start": state_events_at_start, + "state_events_at_start": stateEventsAtStart, }), client.WithContentType("application/json"), client.WithQueries(query), From 1de961835cf365089c9614dfff2042a185cd1416 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 8 Sep 2021 11:38:10 -0500 Subject: [PATCH 15/19] Use txn helper for other txns --- tests/msc2716_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index d68f6bfd..b0f881db 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -501,7 +501,8 @@ func TestBackfillingHistory(t *testing.T) { }, } // We can't use as.SendEventSynced(...) because application services can't use the /sync API - insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, "txn-i123"}, client.WithJSONBody(t, insertionEvent.Content)) + txnId := getTxnID("sendInsertionAndEnsureBackfilled-txn") + insertionSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", insertionEvent.Type, txnId}, client.WithJSONBody(t, insertionEvent.Content)) insertionSendBody := client.ParseJSON(t, insertionSendRes) insertionEventID := client.GetJSONFieldStr(t, insertionSendBody, "event_id") // Make sure the insertion event has reached the homeserver @@ -944,8 +945,8 @@ func sendMarkerAndEnsureBackfilled(t *testing.T, as *client.CSAPI, c *client.CSA markerInsertionContentField: insertionEventID, }, } - txnId := getTxnID("sendMarkerAndEnsureBackfilled-txn") // We can't use as.SendEventSynced(...) because application services can't use the /sync API + txnId := getTxnID("sendMarkerAndEnsureBackfilled-txn") markerSendRes := as.MustDoFunc(t, "PUT", []string{"_matrix", "client", "r0", "rooms", roomID, "send", markerEvent.Type, txnId}, client.WithJSONBody(t, markerEvent.Content)) markerSendBody := client.ParseJSON(t, markerSendRes) markerEventID = client.GetJSONFieldStr(t, markerSendBody, "event_id") From 2b3535db03e65dbb732c5988f918ad0bc790d059 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 10 Sep 2021 12:44:05 -0500 Subject: [PATCH 16/19] Better assertion comments to explain why 2 See https://github.com/matrix-org/complement/pull/204#discussion_r706051738 --- tests/msc2716_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index a11dfed5..fecd453d 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -609,6 +609,7 @@ func TestBackfillingHistory(t *testing.T) { "limit": []string{"5"}, })) + numMessagesSent := 2 batchSendRes := batchSendHistoricalMessages( t, as, @@ -616,7 +617,7 @@ func TestBackfillingHistory(t *testing.T) { eventIdBefore, "", createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), - createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, numMessagesSent), // Status 200, ) @@ -624,8 +625,8 @@ func TestBackfillingHistory(t *testing.T) { historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) - if len(historicalEventIDs) != 2 { - t.Fatalf("Expected eventID list should be length 2 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + if len(historicalEventIDs) != numMessagesSent { + t.Fatalf("Expected %d event_ids in the response that correspond to the %d events we sent in the request but saw %d: %s", numMessagesSent, numMessagesSent, len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -697,6 +698,7 @@ func TestBackfillingHistory(t *testing.T) { })) // Historical messages are inserted where we have already scrolled back to + numMessagesSent := 2 batchSendRes := batchSendHistoricalMessages( t, as, @@ -704,7 +706,7 @@ func TestBackfillingHistory(t *testing.T) { eventIdBefore, "", createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), - createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, numMessagesSent), // Status 200, ) @@ -712,8 +714,8 @@ func TestBackfillingHistory(t *testing.T) { historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) - if len(historicalEventIDs) != 2 { - t.Fatalf("Expected eventID list should be length 2 but saw %d: %s", len(historicalEventIDs), historicalEventIDs) + if len(historicalEventIDs) != numMessagesSent { + t.Fatalf("Expected %d event_ids in the response that correspond to the %d events we sent in the request but saw %d: %s", numMessagesSent, numMessagesSent, len(historicalEventIDs), historicalEventIDs) } beforeMarkerMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ From 859aa79357046c0e863e5a4a5fe02736f7c86130 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 10 Sep 2021 13:13:01 -0500 Subject: [PATCH 17/19] Remove useless getter functions --- tests/msc2716_test.go | 80 ++++++++++++------------------------------- 1 file changed, 22 insertions(+), 58 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index fecd453d..df723be6 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -125,11 +125,11 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody0 := client.ParseJSON(t, batchSendRes) - insertionEventID0 := getInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody0) - historicalEventIDs0 := getEventIDsFromBatchSendResponseBody(t, batchSendResBody0) - chunkEventID0 := getChunkEventIdFromBatchSendResponseBody(t, batchSendResBody0) - baseInsertionEventID0 := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody0) - nextChunkID0 := getNextChunkIDFromBatchSendResponseBody(t, batchSendResBody0) + insertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "insertion_event_id") + historicalEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") + chunkEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "chunk_event_id") + baseInsertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "base_insertion_event_id") + nextChunkID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_chunk_id") // Insert another older chunk of backfilled history from the same user. // Make sure the meta data and joins still work on the subsequent chunk @@ -146,9 +146,9 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody1 := client.ParseJSON(t, batchSendRes1) - insertionEventID1 := getInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody1) - historicalEventIDs1 := getEventIDsFromBatchSendResponseBody(t, batchSendResBody1) - chunkEventID1 := getChunkEventIdFromBatchSendResponseBody(t, batchSendResBody1) + insertionEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "insertion_event_id") + historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") + chunkEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "chunk_event_id") var expectedEventIDOrder []string expectedEventIDOrder = append(expectedEventIDOrder, eventIDsBefore...) @@ -238,7 +238,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, @@ -281,7 +281,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") backfilledEventId := historicalEventIDs[0] // This is just a dummy event we search for after the backfilledEventId @@ -324,7 +324,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - stateEventIDs := getStateEventIDsFromBatchSendResponseBody(t, batchSendResBody) + stateEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "state_event_ids") // We only expect 1 state event to be returned because we only passed in 1 // event into `?state_events_at_start` @@ -436,7 +436,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, @@ -486,7 +486,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") // Join the room from a remote homeserver after the backfilled messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -558,7 +558,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") // Join the room from a remote homeserver after the backfilled messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -622,8 +622,8 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) - baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") + baseInsertionEventID := client.GetJSONFieldStr(t, batchSendResBody, "base_insertion_event_id") if len(historicalEventIDs) != numMessagesSent { t.Fatalf("Expected %d event_ids in the response that correspond to the %d events we sent in the request but saw %d: %s", numMessagesSent, numMessagesSent, len(historicalEventIDs), historicalEventIDs) @@ -711,8 +711,8 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) - baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") + baseInsertionEventID := client.GetJSONFieldStr(t, batchSendResBody, "base_insertion_event_id") if len(historicalEventIDs) != numMessagesSent { t.Fatalf("Expected %d event_ids in the response that correspond to the %d events we sent in the request but saw %d: %s", numMessagesSent, numMessagesSent, len(historicalEventIDs), historicalEventIDs) @@ -793,7 +793,7 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := getEventIDsFromBatchSendResponseBody(t, batchSendResBody) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, @@ -833,9 +833,9 @@ func TestBackfillingHistory(t *testing.T) { 200, ) batchSendResBody := client.ParseJSON(t, batchSendRes) - insertionEventID := getInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) - chunkEventID := getChunkEventIdFromBatchSendResponseBody(t, batchSendResBody) - baseInsertionEventID := getBaseInsertionEventIDFromBatchSendResponseBody(t, batchSendResBody) + insertionEventID := client.GetJSONFieldStr(t, batchSendResBody, "insertion_event_id") + chunkEventID := client.GetJSONFieldStr(t, batchSendResBody, "chunk_event_id") + baseInsertionEventID := client.GetJSONFieldStr(t, batchSendResBody, "base_insertion_event_id") // Send the marker event markerEventID := sendMarkerAndEnsureBackfilled(t, as, alice, roomID, baseInsertionEventID) @@ -1160,39 +1160,3 @@ func batchSendHistoricalMessages( return res } - -func getStateEventIDsFromBatchSendResponseBody(t *testing.T, body []byte) (stateEventIDs []string) { - stateEventIDs = client.GetJSONFieldStringArray(t, body, "state_event_ids") - - return stateEventIDs -} - -func getEventIDsFromBatchSendResponseBody(t *testing.T, body []byte) (eventIDs []string) { - eventIDs = client.GetJSONFieldStringArray(t, body, "event_ids") - - return eventIDs -} - -func getNextChunkIDFromBatchSendResponseBody(t *testing.T, body []byte) (nextChunkID string) { - nextChunkID = client.GetJSONFieldStr(t, body, "next_chunk_id") - - return nextChunkID -} - -func getInsertionEventIDFromBatchSendResponseBody(t *testing.T, body []byte) (insertionEventID string) { - insertionEventID = client.GetJSONFieldStr(t, body, "insertion_event_id") - - return insertionEventID -} - -func getChunkEventIdFromBatchSendResponseBody(t *testing.T, body []byte) (chunkEventId string) { - chunkEventId = client.GetJSONFieldStr(t, body, "chunk_event_id") - - return chunkEventId -} - -func getBaseInsertionEventIDFromBatchSendResponseBody(t *testing.T, body []byte) (baseInsertionEventID string) { - baseInsertionEventID = client.GetJSONFieldStr(t, body, "base_insertion_event_id") - - return baseInsertionEventID -} From 98e392c79cdaa2652e57e8c851cb921f45078acb Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 Sep 2021 19:16:51 -0500 Subject: [PATCH 18/19] Rename MSC2716 things from chunk to batch to match `/batch_send` endpoint As described in https://github.com/matrix-org/matrix-doc/pull/2716#discussion_r684574497 Synapse changes in https://github.com/matrix-org/synapse/pull/10838 --- tests/msc2716_test.go | 76 +++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index df723be6..c4659b71 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -38,24 +38,24 @@ const timeBetweenMessages = time.Millisecond var ( insertionEventType = "org.matrix.msc2716.insertion" - chunkEventType = "org.matrix.msc2716.chunk" + batchEventType = "org.matrix.msc2716.batch" markerEventType = "org.matrix.msc2716.marker" historicalContentField = "org.matrix.msc2716.historical" - nextChunkIDContentField = "org.matrix.msc2716.next_chunk_id" + nextBatchIDContentField = "org.matrix.msc2716.next_batch_id" markerInsertionContentField = "org.matrix.msc2716.marker.insertion" ) var createPublicRoomOpts = map[string]interface{}{ "preset": "public_chat", "name": "the hangout spot", - "room_version": "org.matrix.msc2716", + "room_version": "org.matrix.msc2716v3", } var createPrivateRoomOpts = map[string]interface{}{ "preset": "private_chat", "name": "the hangout spot", - "room_version": "org.matrix.msc2716", + "room_version": "org.matrix.msc2716v3", } func TestBackfillingHistory(t *testing.T) { @@ -82,9 +82,9 @@ func TestBackfillingHistory(t *testing.T) { t.Run("parallel", func(t *testing.T) { // Test that the message events we insert between A and B come back in the correct order from /messages // - // Final timeline output: ( [n] = historical chunk ) - // (oldest) A, B, [insertion, c, d, e, chunk] [insertion, f, g, h, chunk, insertion], I, J (newest) - // historical chunk 1 historical chunk 0 + // Final timeline output: ( [n] = historical batch ) + // (oldest) A, B, [insertion, c, d, e, batch] [insertion, f, g, h, batch, insertion], I, J (newest) + // historical batch 1 historical batch 0 t.Run("Backfilled historical events resolve with proper state in correct order", func(t *testing.T) { t.Parallel() @@ -111,7 +111,7 @@ func TestBackfillingHistory(t *testing.T) { // inserted history later. eventIDsAfter := createMessagesInRoom(t, alice, roomID, 2) - // Insert the most recent chunk of backfilled history + // Insert the most recent batch of backfilled history insertTime0 := timeAfterEventBefore.Add(timeBetweenMessages * 3) batchSendRes := batchSendHistoricalMessages( t, @@ -127,19 +127,19 @@ func TestBackfillingHistory(t *testing.T) { batchSendResBody0 := client.ParseJSON(t, batchSendRes) insertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "insertion_event_id") historicalEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") - chunkEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "chunk_event_id") + batchEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "batch_event_id") baseInsertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "base_insertion_event_id") - nextChunkID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_chunk_id") + nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") - // Insert another older chunk of backfilled history from the same user. - // Make sure the meta data and joins still work on the subsequent chunk + // Insert another older batch of backfilled history from the same user. + // Make sure the meta data and joins still work on the subsequent batch insertTime1 := timeAfterEventBefore batchSendRes1 := batchSendHistoricalMessages( t, as, roomID, eventIdBefore, - nextChunkID0, + nextBatchID0, createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), // Status @@ -148,23 +148,23 @@ func TestBackfillingHistory(t *testing.T) { batchSendResBody1 := client.ParseJSON(t, batchSendRes1) insertionEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "insertion_event_id") historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") - chunkEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "chunk_event_id") + batchEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "batch_event_id") var expectedEventIDOrder []string expectedEventIDOrder = append(expectedEventIDOrder, eventIDsBefore...) expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID1) expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs1...) - expectedEventIDOrder = append(expectedEventIDOrder, chunkEventID1) + expectedEventIDOrder = append(expectedEventIDOrder, batchEventID1) expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID0) expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs0...) - expectedEventIDOrder = append(expectedEventIDOrder, chunkEventID0) + expectedEventIDOrder = append(expectedEventIDOrder, batchEventID0) expectedEventIDOrder = append(expectedEventIDOrder, baseInsertionEventID0) expectedEventIDOrder = append(expectedEventIDOrder, eventIDsAfter...) // Order events from newest to oldest expectedEventIDOrder = reversed(expectedEventIDOrder) - // (oldest) A, B, [insertion, c, d, e, chunk] [insertion, f, g, h, chunk, insertion], I, J (newest) - // historical chunk 1 historical chunk 0 + // (oldest) A, B, [insertion, c, d, e, batch] [insertion, f, g, h, batch, insertion], I, J (newest) + // historical batch 1 historical batch 0 if len(expectedEventIDOrder) != 15 { t.Fatalf("Expected eventID list should be length 15 but saw %d: %s", len(expectedEventIDOrder), expectedEventIDOrder) } @@ -206,7 +206,7 @@ func TestBackfillingHistory(t *testing.T) { } }) - t.Run("Backfilled historical events from multiple users in the same chunk", func(t *testing.T) { + t.Run("Backfilled historical events from multiple users in the same batch", func(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) @@ -355,7 +355,7 @@ func TestBackfillingHistory(t *testing.T) { ) }) - t.Run("Unrecognised chunk_id will throw an error", func(t *testing.T) { + t.Run("Unrecognised batch_id will throw an error", func(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) @@ -370,7 +370,7 @@ func TestBackfillingHistory(t *testing.T) { as, roomID, eventIdBefore, - "XXX_DOES_NOT_EXIST_CHUNK_ID", + "XXX_DOES_NOT_EXIST_BATCH_ID", createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 1), // Status @@ -402,9 +402,9 @@ func TestBackfillingHistory(t *testing.T) { ) }) - t.Run("TODO: Trying to send insertion event with same `next_chunk_id` will reject", func(t *testing.T) { + t.Run("TODO: Trying to send insertion event with same `next_batch_id` will reject", func(t *testing.T) { t.Skip("Skipping until implemented") - // (room_id, next_chunk_id) should be unique + // (room_id, next_batch_id) should be unique }) t.Run("Should be able to backfill into private room", func(t *testing.T) { @@ -457,7 +457,7 @@ func TestBackfillingHistory(t *testing.T) { // TODO: Try adding avatar and displayName and see if historical messages get this info }) - t.Run("TODO: What happens when you point multiple chunks at the same insertion event?", func(t *testing.T) { + t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) { t.Skip("Skipping until implemented") }) @@ -525,11 +525,11 @@ func TestBackfillingHistory(t *testing.T) { timeAfterEventBefore := time.Now() // Create insertion event in the normal DAG - chunkId := "mynextchunkid123" + batchID := "mynextBatchID123" insertionEvent := b.Event{ Type: insertionEventType, Content: map[string]interface{}{ - nextChunkIDContentField: chunkId, + nextBatchIDContentField: batchID, historicalContentField: true, }, } @@ -551,7 +551,7 @@ func TestBackfillingHistory(t *testing.T) { as, roomID, eventIdBefore, - chunkId, + batchID, createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 2), // Status @@ -809,7 +809,7 @@ func TestBackfillingHistory(t *testing.T) { }) }) - t.Run("Not allowed to redact MSC2716 insertion, chunk, marker events", func(t *testing.T) { + t.Run("Not allowed to redact MSC2716 insertion, batch, marker events", func(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, createUnsupportedMSC2716RoomOpts) @@ -834,14 +834,14 @@ func TestBackfillingHistory(t *testing.T) { ) batchSendResBody := client.ParseJSON(t, batchSendRes) insertionEventID := client.GetJSONFieldStr(t, batchSendResBody, "insertion_event_id") - chunkEventID := client.GetJSONFieldStr(t, batchSendResBody, "chunk_event_id") + batchEventID := client.GetJSONFieldStr(t, batchSendResBody, "batch_event_id") baseInsertionEventID := client.GetJSONFieldStr(t, batchSendResBody, "base_insertion_event_id") // Send the marker event markerEventID := sendMarkerAndEnsureBackfilled(t, as, alice, roomID, baseInsertionEventID) redactEventID(t, alice, roomID, insertionEventID, 403) - redactEventID(t, alice, roomID, chunkEventID, 403) + redactEventID(t, alice, roomID, batchEventID, 403) redactEventID(t, alice, roomID, markerEventID, 403) }) }) @@ -914,7 +914,7 @@ func fetchUntilMessagesResponseHas(t *testing.T, c *client.CSAPI, roomID string, func isRelevantEvent(r gjson.Result) bool { return len(r.Get("content").Get("body").Str) > 0 || r.Get("type").Str == insertionEventType || - r.Get("type").Str == chunkEventType || + r.Get("type").Str == batchEventType || r.Get("type").Str == markerEventType } @@ -1088,7 +1088,7 @@ func createMessageEventsForBackfillRequest( "origin_server_ts": insertOriginServerTs + (timeBetweenMessagesMS * uint64(i)), "content": map[string]interface{}{ "msgtype": "m.text", - "body": fmt.Sprintf("Historical %d (chunk=%d)", i, chunkCount), + "body": fmt.Sprintf("Historical %d (batch=%d)", i, batchCount), historicalContentField: true, }, } @@ -1119,14 +1119,14 @@ func redactEventID(t *testing.T, c *client.CSAPI, roomID, eventID string, expect } } -var chunkCount int64 = 0 +var batchCount int64 = 0 func batchSendHistoricalMessages( t *testing.T, c *client.CSAPI, roomID string, insertAfterEventId string, - chunkID string, + batchID string, stateEventsAtStart []map[string]interface{}, events []map[string]interface{}, expectedStatus int, @@ -1135,9 +1135,9 @@ func batchSendHistoricalMessages( query := make(url.Values, 2) query.Add("prev_event", insertAfterEventId) - // If provided, connect the chunk to the last insertion point - if chunkID != "" { - query.Add("chunk_id", chunkID) + // If provided, connect the batch to the last insertion point + if batchID != "" { + query.Add("batch_id", batchID) } res = c.DoFunc( @@ -1156,7 +1156,7 @@ func batchSendHistoricalMessages( t.Fatalf("msc2716.batchSendHistoricalMessages got %d HTTP status code from batch send response but want %d", res.StatusCode, expectedStatus) } - chunkCount++ + batchCount++ return res } From 46beb9ba0c4102e2622a4d7a73d0cc383768bc38 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 Sep 2021 15:32:24 -0500 Subject: [PATCH 19/19] Update query parameter to comply with rename Synapse changes in https://github.com/matrix-org/synapse/pull/10839 --- tests/msc2716_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index c4659b71..fc6faf72 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -1134,7 +1134,7 @@ func batchSendHistoricalMessages( t.Helper() query := make(url.Values, 2) - query.Add("prev_event", insertAfterEventId) + query.Add("prev_event_id", insertAfterEventId) // If provided, connect the batch to the last insertion point if batchID != "" { query.Add("batch_id", batchID)