From 7dc7d5f51dcdf104055bcfc6f46c05eb3db9617b Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 21 Sep 2021 15:48:15 -0500 Subject: [PATCH 01/17] Verify that we can resolve member avatar/displayname state for historical messages Part of testing https://gitlab.com/gitterHQ/webapp/-/merge_requests/2229#note_654645018 --- tests/msc2716_test.go | 77 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index fc6faf72..fea5a331 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/matrix-org/complement/internal/b" @@ -457,6 +458,82 @@ func TestBackfillingHistory(t *testing.T) { // TODO: Try adding avatar and displayName and see if historical messages get this info }) + t.Run("should resolve member state events for historical events", func(t *testing.T) { + t.Parallel() + + roomID := as.CreateRoom(t, createPublicRoomOpts) + alice.JoinRoom(t, roomID, nil) + + // Create the "live" event we are going to insert our backfilled events next to + eventIDsBefore := createMessagesInRoom(t, alice, roomID, 1) + eventIdBefore := eventIDsBefore[0] + timeAfterEventBefore := time.Now() + + // eventIDsAfter + createMessagesInRoom(t, alice, roomID, 2) + + // Insert a backfilled event + batchSendRes := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + "", + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 3), + // Status + 200, + ) + batchSendResBody0 := client.ParseJSON(t, batchSendRes) + historicalEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") + nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") + + contextRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs0[1]}, client.WithContentType("application/json")) + contextResBody0 := client.ParseJSON(t, contextRes0) + stateJsonResult0 := gjson.GetBytes(contextResBody0, "state") + stateFromContext0 := stateJsonResult0.Array() + + logrus.WithFields(logrus.Fields{ + "stateFromContext": stateFromContext0, + "len": len(stateFromContext0), + }).Error("gaweegegegwegawagewg 0") + + if len(stateFromContext0) == 0 { + t.Fatalf("Expected some state events in the context response for historical event in first batch but saw %d: %s", len(stateFromContext0), stateFromContext0) + } + + // Insert another older batch of backfilled history from the same user. + // Make sure the meta data and joins still work on the subsequent batch + insertTime1 := timeAfterEventBefore + batchSendRes1 := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + nextBatchID0, + createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), + createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), + // Status + 200, + ) + batchSendResBody1 := client.ParseJSON(t, batchSendRes1) + historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") + + contextRes1 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs1[1]}, client.WithContentType("application/json")) + contextResBody1 := client.ParseJSON(t, contextRes1) + stateJsonResult1 := gjson.GetBytes(contextResBody1, "state") + stateFromContext1 := stateJsonResult1.Array() + + logrus.WithFields(logrus.Fields{ + "stateFromContext": stateFromContext1, + "len": len(stateFromContext1), + }).Error("gaweegegegwegawagewg 1") + + if len(stateFromContext1) == 0 { + t.Fatalf("Expected some state events in the context response for historical event in second batch but saw %d: %s", len(stateFromContext1), stateFromContext1) + } + }) + t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) { t.Skip("Skipping until implemented") }) From aa730451dc05f121af5db9264cfd6625e999614a Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 30 Sep 2021 01:46:10 -0500 Subject: [PATCH 02/17] Scratch changes While working on https://github.com/matrix-org/synapse/pull/10948 --- tests/msc2716_test.go | 133 ++++++++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 45 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index fea5a331..9bb1491f 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -486,52 +486,94 @@ func TestBackfillingHistory(t *testing.T) { ) batchSendResBody0 := client.ParseJSON(t, batchSendRes) historicalEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") - nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") - - contextRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs0[1]}, client.WithContentType("application/json")) - contextResBody0 := client.ParseJSON(t, contextRes0) - stateJsonResult0 := gjson.GetBytes(contextResBody0, "state") - stateFromContext0 := stateJsonResult0.Array() - - logrus.WithFields(logrus.Fields{ - "stateFromContext": stateFromContext0, - "len": len(stateFromContext0), - }).Error("gaweegegegwegawagewg 0") - - if len(stateFromContext0) == 0 { - t.Fatalf("Expected some state events in the context response for historical event in first batch but saw %d: %s", len(stateFromContext0), stateFromContext0) - } - - // Insert another older batch of backfilled history from the same user. - // Make sure the meta data and joins still work on the subsequent batch - insertTime1 := timeAfterEventBefore - batchSendRes1 := batchSendHistoricalMessages( - t, - as, - roomID, - eventIdBefore, - nextBatchID0, - createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), - createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), - // Status - 200, - ) - batchSendResBody1 := client.ParseJSON(t, batchSendRes1) - historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") - - contextRes1 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs1[1]}, client.WithContentType("application/json")) - contextResBody1 := client.ParseJSON(t, contextRes1) - stateJsonResult1 := gjson.GetBytes(contextResBody1, "state") - stateFromContext1 := stateJsonResult1.Array() - + stateEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "state_event_ids") + //nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") logrus.WithFields(logrus.Fields{ - "stateFromContext": stateFromContext1, - "len": len(stateFromContext1), - }).Error("gaweegegegwegawagewg 1") + "historicalEventIDs0": historicalEventIDs0, + "stateEventIDs0": stateEventIDs0, + }).Error("batchResponse0") + + //contextRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs0[1]}, client.WithContentType("application/json")) + // contextResBody0 := client.ParseJSON(t, contextRes0) + // stateJsonResult0 := gjson.GetBytes(contextResBody0, "state") + // stateFromContext0 := stateJsonResult0.Array() + + // logrus.WithFields(logrus.Fields{ + // "stateFromContext": stateFromContext0, + // "len": len(stateFromContext0), + // }).Error("gaweegegegwegawagewg 0") + + // if len(stateFromContext0) == 0 { + // t.Fatalf("Expected some state events in the context response for historical event in first batch but saw %d: %s", len(stateFromContext0), stateFromContext0) + // } + + // Make sure the historical member event we passed in via + // `state_events_at_start` resolves in the state of the historical message + // must.MatchResponse(t, contextRes0, match.HTTPResponse{ + // JSON: []match.JSON{ + // match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { + // return r.Get("event_id").Str + // }, nil), + // }, + // }) + + messagesRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, + })) + must.MatchResponse(t, messagesRes0, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs0), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) - if len(stateFromContext1) == 0 { - t.Fatalf("Expected some state events in the context response for historical event in second batch but saw %d: %s", len(stateFromContext1), stateFromContext1) - } + // // Insert another older batch of backfilled history from the same user. + // // Make sure the meta data and joins still work on the subsequent batch + // insertTime1 := timeAfterEventBefore + // batchSendRes1 := batchSendHistoricalMessages( + // t, + // as, + // roomID, + // eventIdBefore, + // nextBatchID0, + // createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), + // createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), + // // Status + // 200, + // ) + // batchSendResBody1 := client.ParseJSON(t, batchSendRes1) + // historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") + // stateEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "state_event_ids") + + // contextRes1 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs1[1]}, client.WithContentType("application/json")) + // // contextResBody1 := client.ParseJSON(t, contextRes1) + // // stateJsonResult1 := gjson.GetBytes(contextResBody1, "state") + // // stateFromContext1 := stateJsonResult1.Array() + + // // logrus.WithFields(logrus.Fields{ + // // "stateFromContext": stateFromContext1, + // // "len": len(stateFromContext1), + // // }).Error("gaweegegegwegawagewg 1") + + // // if len(stateFromContext1) == 0 { + // // t.Fatalf("Expected some state events in the context response for historical event in second batch but saw %d: %s", len(stateFromContext1), stateFromContext1) + // // } + + // // Make sure the historical member event we passed in via + // // `state_events_at_start` resolves in the state of the historical message + // must.MatchResponse(t, contextRes1, match.HTTPResponse{ + // JSON: []match.JSON{ + // match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs1), func(r gjson.Result) interface{} { + // return r.Get("event_id").Str + // }, nil), + // }, + // }) }) t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) { @@ -1135,7 +1177,8 @@ func createJoinStateEventsForBackfillRequest( "sender": virtualUserID, "origin_server_ts": insertOriginServerTs, "content": map[string]interface{}{ - "membership": "join", + "membership": "join", + "displayname": fmt.Sprintf("some-display-name-for-%s", virtualUserID), }, "state_key": virtualUserID, } From d484e5a809c041543c982b636e9726222416efe7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 21 Oct 2021 04:43:49 -0500 Subject: [PATCH 03/17] Merge backfill refactor changes from master --- tests/msc2716_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 047e4992..72736c88 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -474,15 +474,15 @@ func TestImportHistoricalMessages(t *testing.T) { // eventIDsAfter createMessagesInRoom(t, alice, roomID, 2) - // Insert a backfilled event + // Import a historical event batchSendRes := batchSendHistoricalMessages( t, as, roomID, eventIdBefore, "", - createJoinStateEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore), - createMessageEventsForBackfillRequest([]string{virtualUserID}, timeAfterEventBefore, 3), + createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore, 3), // Status 200, ) @@ -535,7 +535,7 @@ func TestImportHistoricalMessages(t *testing.T) { }, }) - // // Insert another older batch of backfilled history from the same user. + // // Import another older batch of history from the same user. // // Make sure the meta data and joins still work on the subsequent batch // insertTime1 := timeAfterEventBefore // batchSendRes1 := batchSendHistoricalMessages( @@ -544,8 +544,8 @@ func TestImportHistoricalMessages(t *testing.T) { // roomID, // eventIdBefore, // nextBatchID0, - // createJoinStateEventsForBackfillRequest([]string{virtualUserID}, insertTime1), - // createMessageEventsForBackfillRequest([]string{virtualUserID}, insertTime1, 3), + // createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, insertTime1), + // createMessageEventsForBatchSendRequest([]string{virtualUserID}, insertTime1, 3), // // Status // 200, // ) From 293926ebcbcfbed479536a31c2701d28f9b5b2ba Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 26 Oct 2021 19:12:14 -0500 Subject: [PATCH 04/17] Update to use same mechanism as scrollback to verify state resolves (/messages) --- tests/msc2716_test.go | 138 +++++++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 54 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index e5051d5e..2aeb99a9 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -502,10 +502,9 @@ func TestImportHistoricalMessages(t *testing.T) { eventIdBefore := eventIDsBefore[0] timeAfterEventBefore := time.Now() - // eventIDsAfter - createMessagesInRoom(t, alice, roomID, 2) + eventIDsAfter := createMessagesInRoom(t, alice, roomID, 2) - // Import a historical event + // Import a batch of historical events batchSendRes := batchSendHistoricalMessages( t, as, @@ -513,53 +512,60 @@ func TestImportHistoricalMessages(t *testing.T) { eventIdBefore, "", createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore), - createMessageEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore, 3), + createMessageEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore, 4), // Status 200, ) batchSendResBody0 := client.ParseJSON(t, batchSendRes) historicalEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") stateEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "state_event_ids") + baseInsertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "base_insertion_event_id") + batchEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "batch_event_id") + insertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "insertion_event_id") //nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") logrus.WithFields(logrus.Fields{ "historicalEventIDs0": historicalEventIDs0, "stateEventIDs0": stateEventIDs0, }).Error("batchResponse0") - //contextRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs0[1]}, client.WithContentType("application/json")) - // contextResBody0 := client.ParseJSON(t, contextRes0) - // stateJsonResult0 := gjson.GetBytes(contextResBody0, "state") - // stateFromContext0 := stateJsonResult0.Array() + var expectedEventIDOrder0 []string + expectedEventIDOrder0 = append(expectedEventIDOrder0, baseInsertionEventID0) + expectedEventIDOrder0 = append(expectedEventIDOrder0, batchEventID0) + expectedEventIDOrder0 = append(expectedEventIDOrder0, historicalEventIDs0...) + expectedEventIDOrder0 = append(expectedEventIDOrder0, insertionEventID0) - // logrus.WithFields(logrus.Fields{ - // "stateFromContext": stateFromContext0, - // "len": len(stateFromContext0), - // }).Error("gaweegegegwegawagewg 0") - - // if len(stateFromContext0) == 0 { - // t.Fatalf("Expected some state events in the context response for historical event in first batch but saw %d: %s", len(stateFromContext0), stateFromContext0) - // } - - // Make sure the historical member event we passed in via - // `state_events_at_start` resolves in the state of the historical message - // must.MatchResponse(t, contextRes0, match.HTTPResponse{ - // JSON: []match.JSON{ - // match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { - // return r.Get("event_id").Str - // }, nil), - // }, - // }) + messagesScrollbackIntoHistoricalRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + // We are aiming to scrollback to the start of the historical messages + // so we can use the `end` pagination token to paginate from + "limit": []string{fmt.Sprintf("%d", len(eventIDsAfter))}, + })) + messagesScrollbackIntoHistoricalBody := client.ParseJSON(t, messagesScrollbackIntoHistoricalRes) + historicalMessagesBeginningPaginationToken := client.GetJSONFieldStr(t, messagesScrollbackIntoHistoricalBody, "end") messagesRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, + "dir": []string{"b"}, + // From the beginning of the historical messages + "from": []string{historicalMessagesBeginningPaginationToken}, + // We are aiming to scrollback to the start of the existing historical + // messages + "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder0))}, "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, })) + must.MatchResponse(t, messagesRes0, match.HTTPResponse{ JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs0), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), + // Double-check that we're in the right place of scrollback + match.JSONCheckOff("chunk", + makeInterfaceSlice(expectedEventIDOrder0), + func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, + nil, + ), + // Make sure the historical m.room.member join state event resolves + // for the given chunk of messages in scrollback. The member event + // will include the displayname and avatar. match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { return r.Get("event_id").Str }, nil), @@ -568,41 +574,65 @@ func TestImportHistoricalMessages(t *testing.T) { // // Import another older batch of history from the same user. // // Make sure the meta data and joins still work on the subsequent batch - // insertTime1 := timeAfterEventBefore // batchSendRes1 := batchSendHistoricalMessages( // t, // as, // roomID, // eventIdBefore, - // nextBatchID0, - // createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, insertTime1), - // createMessageEventsForBatchSendRequest([]string{virtualUserID}, insertTime1, 3), + // "", + // createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore), + // createMessageEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore, 4), // // Status // 200, // ) // batchSendResBody1 := client.ParseJSON(t, batchSendRes1) // historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") // stateEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "state_event_ids") - - // contextRes1 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", historicalEventIDs1[1]}, client.WithContentType("application/json")) - // // contextResBody1 := client.ParseJSON(t, contextRes1) - // // stateJsonResult1 := gjson.GetBytes(contextResBody1, "state") - // // stateFromContext1 := stateJsonResult1.Array() - - // // logrus.WithFields(logrus.Fields{ - // // "stateFromContext": stateFromContext1, - // // "len": len(stateFromContext1), - // // }).Error("gaweegegegwegawagewg 1") - - // // if len(stateFromContext1) == 0 { - // // t.Fatalf("Expected some state events in the context response for historical event in second batch but saw %d: %s", len(stateFromContext1), stateFromContext1) - // // } - - // // Make sure the historical member event we passed in via - // // `state_events_at_start` resolves in the state of the historical message - // must.MatchResponse(t, contextRes1, match.HTTPResponse{ + // batchEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "batch_event_id") + // insertionEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "insertion_event_id") + // logrus.WithFields(logrus.Fields{ + // "historicalEventIDs1": historicalEventIDs1, + // "stateEventIDs1": stateEventIDs1, + // }).Error("batchResponse1") + + // var expectedEventIDOrder1 []string + // expectedEventIDOrder1 = append(expectedEventIDOrder1, batchEventID1) + // expectedEventIDOrder1 = append(expectedEventIDOrder1, historicalEventIDs1...) + // expectedEventIDOrder1 = append(expectedEventIDOrder1, insertionEventID1) + + // messagesScrollbackIntoHistoricalRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + // "dir": []string{"b"}, + // // We are aiming to scrollback to the start of the historical messages + // // so we can use the `end` pagination token to paginate from + // "limit": []string{fmt.Sprintf("%d", len(eventIDsAfter))}, + // })) + // messagesScrollbackIntoHistoricalBody := client.ParseJSON(t, messagesScrollbackIntoHistoricalRes) + // historicalMessagesBeginningPaginationToken := client.GetJSONFieldStr(t, messagesScrollbackIntoHistoricalBody, "end") + + // messagesRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + // "dir": []string{"b"}, + // // From the beginning of the historical messages + // "from": []string{historicalMessagesBeginningPaginationToken}, + // // We are aiming to scrollback to the start of the existing historical + // // messages + // "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder0))}, + // "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, + // })) + + // must.MatchResponse(t, messagesRes0, match.HTTPResponse{ // JSON: []match.JSON{ - // match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs1), func(r gjson.Result) interface{} { + // // Double-check that we're in the right place of scrollback + // match.JSONCheckOff("chunk", + // makeInterfaceSlice(expectedEventIDOrder0), + // func(r gjson.Result) interface{} { + // return r.Get("event_id").Str + // }, + // nil, + // ), + // // Make sure the historical m.room.member join state event resolves + // // for the given chunk of messages in scrollback. The member event + // // will include the displayname and avatar. + // match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { // return r.Get("event_id").Str // }, nil), // }, From ab24695c5dc9366467af7f9024f0cdb05fd14a74 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 27 Oct 2021 18:40:58 -0500 Subject: [PATCH 05/17] Clean up test and generalize batch send validation --- tests/msc2716_test.go | 207 +++++++++++++++++------------------------- 1 file changed, 84 insertions(+), 123 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 2aeb99a9..c3ce1d39 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -15,7 +15,6 @@ import ( "testing" "time" - "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/matrix-org/complement/internal/b" @@ -502,10 +501,11 @@ func TestImportHistoricalMessages(t *testing.T) { eventIdBefore := eventIDsBefore[0] timeAfterEventBefore := time.Now() - eventIDsAfter := createMessagesInRoom(t, alice, roomID, 2) + // eventIDsAfter + createMessagesInRoom(t, alice, roomID, 2) // Import a batch of historical events - batchSendRes := batchSendHistoricalMessages( + batchSendRes0 := batchSendHistoricalMessages( t, as, roomID, @@ -516,127 +516,24 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - batchSendResBody0 := client.ParseJSON(t, batchSendRes) - historicalEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") - stateEventIDs0 := client.GetJSONFieldStringArray(t, batchSendResBody0, "state_event_ids") - baseInsertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "base_insertion_event_id") - batchEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "batch_event_id") - insertionEventID0 := client.GetJSONFieldStr(t, batchSendResBody0, "insertion_event_id") - //nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") - logrus.WithFields(logrus.Fields{ - "historicalEventIDs0": historicalEventIDs0, - "stateEventIDs0": stateEventIDs0, - }).Error("batchResponse0") - - var expectedEventIDOrder0 []string - expectedEventIDOrder0 = append(expectedEventIDOrder0, baseInsertionEventID0) - expectedEventIDOrder0 = append(expectedEventIDOrder0, batchEventID0) - expectedEventIDOrder0 = append(expectedEventIDOrder0, historicalEventIDs0...) - expectedEventIDOrder0 = append(expectedEventIDOrder0, insertionEventID0) - - messagesScrollbackIntoHistoricalRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - // We are aiming to scrollback to the start of the historical messages - // so we can use the `end` pagination token to paginate from - "limit": []string{fmt.Sprintf("%d", len(eventIDsAfter))}, - })) - messagesScrollbackIntoHistoricalBody := client.ParseJSON(t, messagesScrollbackIntoHistoricalRes) - historicalMessagesBeginningPaginationToken := client.GetJSONFieldStr(t, messagesScrollbackIntoHistoricalBody, "end") - - messagesRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - // From the beginning of the historical messages - "from": []string{historicalMessagesBeginningPaginationToken}, - // We are aiming to scrollback to the start of the existing historical - // messages - "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder0))}, - "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, - })) - - must.MatchResponse(t, messagesRes0, match.HTTPResponse{ - JSON: []match.JSON{ - // Double-check that we're in the right place of scrollback - match.JSONCheckOff("chunk", - makeInterfaceSlice(expectedEventIDOrder0), - func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, - nil, - ), - // Make sure the historical m.room.member join state event resolves - // for the given chunk of messages in scrollback. The member event - // will include the displayname and avatar. - match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + validateBatchSendRes(t, as, roomID, batchSendRes0) + batchSendResBody0 := client.ParseJSON(t, batchSendRes0) + nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") - // // Import another older batch of history from the same user. - // // Make sure the meta data and joins still work on the subsequent batch - // batchSendRes1 := batchSendHistoricalMessages( - // t, - // as, - // roomID, - // eventIdBefore, - // "", - // createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore), - // createMessageEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore, 4), - // // Status - // 200, - // ) - // batchSendResBody1 := client.ParseJSON(t, batchSendRes1) - // historicalEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "event_ids") - // stateEventIDs1 := client.GetJSONFieldStringArray(t, batchSendResBody1, "state_event_ids") - // batchEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "batch_event_id") - // insertionEventID1 := client.GetJSONFieldStr(t, batchSendResBody1, "insertion_event_id") - // logrus.WithFields(logrus.Fields{ - // "historicalEventIDs1": historicalEventIDs1, - // "stateEventIDs1": stateEventIDs1, - // }).Error("batchResponse1") - - // var expectedEventIDOrder1 []string - // expectedEventIDOrder1 = append(expectedEventIDOrder1, batchEventID1) - // expectedEventIDOrder1 = append(expectedEventIDOrder1, historicalEventIDs1...) - // expectedEventIDOrder1 = append(expectedEventIDOrder1, insertionEventID1) - - // messagesScrollbackIntoHistoricalRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - // "dir": []string{"b"}, - // // We are aiming to scrollback to the start of the historical messages - // // so we can use the `end` pagination token to paginate from - // "limit": []string{fmt.Sprintf("%d", len(eventIDsAfter))}, - // })) - // messagesScrollbackIntoHistoricalBody := client.ParseJSON(t, messagesScrollbackIntoHistoricalRes) - // historicalMessagesBeginningPaginationToken := client.GetJSONFieldStr(t, messagesScrollbackIntoHistoricalBody, "end") - - // messagesRes0 := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - // "dir": []string{"b"}, - // // From the beginning of the historical messages - // "from": []string{historicalMessagesBeginningPaginationToken}, - // // We are aiming to scrollback to the start of the existing historical - // // messages - // "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder0))}, - // "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, - // })) - - // must.MatchResponse(t, messagesRes0, match.HTTPResponse{ - // JSON: []match.JSON{ - // // Double-check that we're in the right place of scrollback - // match.JSONCheckOff("chunk", - // makeInterfaceSlice(expectedEventIDOrder0), - // func(r gjson.Result) interface{} { - // return r.Get("event_id").Str - // }, - // nil, - // ), - // // Make sure the historical m.room.member join state event resolves - // // for the given chunk of messages in scrollback. The member event - // // will include the displayname and avatar. - // match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs0), func(r gjson.Result) interface{} { - // return r.Get("event_id").Str - // }, nil), - // }, - // }) + // Import another older batch of history from the same user. + // Make sure the meta data and joins still work on the subsequent batch + batchSendRes1 := batchSendHistoricalMessages( + t, + as, + roomID, + eventIdBefore, + nextBatchID0, + createJoinStateEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore), + createMessageEventsForBatchSendRequest([]string{virtualUserID}, timeAfterEventBefore, 4), + // Status + 200, + ) + validateBatchSendRes(t, as, roomID, batchSendRes1) }) t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) { @@ -1361,3 +1258,67 @@ func batchSendHistoricalMessages( return res } + +// Verify that the batch of historical messages looks correct in the message +// scrollback. We also check that the historical state resolves for that chunk +// of messages. +// +// Note: the historical state will only resolve correctly if the +// first message of `/messages` is one of messages in the historical batch. +func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSendRes *http.Response) { + t.Helper() + + batchSendResBody0 := client.ParseJSON(t, batchSendRes) + // Since the original body can only be read once, create a new one from the + // body bytes we just read + batchSendRes.Body = ioutil.NopCloser(bytes.NewBuffer(batchSendResBody0)) + + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody0, "event_ids") + stateEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody0, "state_event_ids") + batchEventID := client.GetJSONFieldStr(t, batchSendResBody0, "batch_event_id") + insertionEventID := client.GetJSONFieldStr(t, batchSendResBody0, "insertion_event_id") + baseInsertionEventID := gjson.GetBytes(batchSendResBody0, "base_insertion_event_id").Str + + var expectedEventIDOrder []string + if baseInsertionEventID != "" { + expectedEventIDOrder = append(expectedEventIDOrder, baseInsertionEventID) + } + expectedEventIDOrder = append(expectedEventIDOrder, batchEventID) + expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs...) + expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID) + + contextRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", expectedEventIDOrder[0]}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "limit": []string{"0"}, + })) + contextResBody := client.ParseJSON(t, contextRes) + batchStartPaginationToken := client.GetJSONFieldStr(t, contextResBody, "end") + + messagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + // From the beginning of the historical messages + "from": []string{batchStartPaginationToken}, + // We are aiming to scrollback to the start of the existing historical + // messages + "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder))}, + "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + // Double-check that we're in the right place of scrollback + match.JSONCheckOff("chunk", + makeInterfaceSlice(expectedEventIDOrder), + func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, + nil, + ), + // Make sure the historical m.room.member join state event resolves + // for the given chunk of messages in scrollback. The member event + // will include the displayname and avatar. + match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) +} From f1703d2d9139667b319e4dc845fce658f20a3689 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 27 Oct 2021 21:34:02 -0500 Subject: [PATCH 06/17] Validate state and order from /messages separately --- tests/msc2716_test.go | 114 +++++++++++++++++++++++------------------- 1 file changed, 63 insertions(+), 51 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index c3ce1d39..f77cde88 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -468,26 +468,16 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") - - messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) - }) - - t.Run("TODO: Test if historical avatar/display name set back in time are picked up on historical messages", func(t *testing.T) { - t.Skip("Skipping until implemented") - // TODO: Try adding avatar and displayName and see if historical messages get this info + validateBatchSendRes( + t, + as, + roomID, + batchSendRes, + // We can't validate the state in this case because the invite event + // won't be resolved in `/messages` state field (i.e. only the member + // event is needed to auth those events) + false, + ) }) t.Run("should resolve member state events for historical events", func(t *testing.T) { @@ -516,7 +506,7 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - validateBatchSendRes(t, as, roomID, batchSendRes0) + validateBatchSendRes(t, as, roomID, batchSendRes0, true) batchSendResBody0 := client.ParseJSON(t, batchSendRes0) nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") @@ -533,7 +523,7 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - validateBatchSendRes(t, as, roomID, batchSendRes1) + validateBatchSendRes(t, as, roomID, batchSendRes1, true) }) t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) { @@ -1265,7 +1255,7 @@ func batchSendHistoricalMessages( // // Note: the historical state will only resolve correctly if the // first message of `/messages` is one of messages in the historical batch. -func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSendRes *http.Response) { +func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSendRes *http.Response, validateState bool) { t.Helper() batchSendResBody0 := client.ParseJSON(t, batchSendRes) @@ -1287,36 +1277,58 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs...) expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID) - contextRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", expectedEventIDOrder[0]}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "limit": []string{"0"}, - })) - contextResBody := client.ParseJSON(t, contextRes) - batchStartPaginationToken := client.GetJSONFieldStr(t, contextResBody, "end") - - messagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - // From the beginning of the historical messages - "from": []string{batchStartPaginationToken}, - // We are aiming to scrollback to the start of the existing historical - // messages - "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder))}, - "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, - })) + if validateState { + // Get the pagination token for the end of the historical batch itself + contextRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", expectedEventIDOrder[0]}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "limit": []string{"0"}, + })) + contextResBody := client.ParseJSON(t, contextRes) + batchStartPaginationToken := client.GetJSONFieldStr(t, contextResBody, "end") - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - // Double-check that we're in the right place of scrollback - match.JSONCheckOff("chunk", - makeInterfaceSlice(expectedEventIDOrder), - func(r gjson.Result) interface{} { + // Fetch a chunk of `/messages` which only contains the historical batch. We + // want to do this because `/messages` only returns the state for the first + // message in the `chunk` and we want to be able assert that the historical + // state is able to be resolved. + messagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + // From the end of the historical batch (this will be pointing at the ) + "from": []string{batchStartPaginationToken}, + // We are aiming to scrollback to the start of the existing historical + // messages + "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder))}, + // We add these options to the filter so we get member events in the state field + "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + // Double-check that we're in the right place of scrollback + match.JSONCheckOff("chunk", + makeInterfaceSlice(expectedEventIDOrder), + func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, + nil, + ), + // Make sure the historical m.room.member join state event resolves + // for the given chunk of messages in scrollback. The member event + // will include the displayname and avatar. + match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs), func(r gjson.Result) interface{} { return r.Get("event_id").Str - }, - nil, - ), - // Make sure the historical m.room.member join state event resolves - // for the given chunk of messages in scrollback. The member event - // will include the displayname and avatar. - match.JSONCheckOffAllowUnwanted("state", makeInterfaceSlice(stateEventIDs), func(r gjson.Result) interface{} { + }, nil), + }, + }) + } + + // Make sure the historical events appear in scrollback without jumping back + // in time specifically. + fullMessagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + must.MatchResponse(t, fullMessagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { return r.Get("event_id").Str }, nil), }, From 60add3da116331938b59464576cca62c60ed89b2 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 27 Oct 2021 23:13:14 -0500 Subject: [PATCH 07/17] Assert messages in order --- tests/msc2716_test.go | 135 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 111 insertions(+), 24 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index f77cde88..21616e1b 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -12,9 +12,11 @@ import ( "io/ioutil" "net/http" "net/url" + "strings" "testing" "time" + "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/matrix-org/complement/internal/b" @@ -175,7 +177,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) messsageResBody := client.ParseJSON(t, messagesRes) - eventDebugStringsFromResponse := getRelevantEventDebugStringsFromMessagesResponse(t, messsageResBody) + eventDebugStringsFromResponse := getRelevantEventDebugStringsFromMessagesResponse(t, messsageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read messagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(messsageResBody)) @@ -187,7 +189,7 @@ func TestImportHistoricalMessages(t *testing.T) { JSON: []match.JSON{ match.JSONArrayEach("chunk", func(r gjson.Result) error { // Find all events in order - if isRelevantEvent(r) { + if relevantToScrollbackEventFilter(r) { // Pop the next message off the expected list nextEventIdInOrder := workingExpectedEventIDOrder[0] workingExpectedEventIDOrder = workingExpectedEventIDOrder[1:] @@ -703,7 +705,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) - eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody) + eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) @@ -792,7 +794,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) - eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody) + eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) // Make sure the history isn't visible before we expect it to be there. @@ -998,33 +1000,45 @@ func fetchUntilMessagesResponseHas(t *testing.T, c *client.CSAPI, roomID string, } } -func isRelevantEvent(r gjson.Result) bool { - return len(r.Get("content").Get("body").Str) > 0 || - r.Get("type").Str == insertionEventType || - r.Get("type").Str == batchEventType || - r.Get("type").Str == markerEventType +func historicalEventFilter(r gjson.Result) bool { + // This includes messages, insertion, batch, and marker events because we + // include the historical field on all of them. + return r.Get("content").Get(strings.ReplaceAll(historicalContentField, ".", "\\.")).Exists() } -func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, body []byte) (eventIDsFromResponse []string) { +func relevantToScrollbackEventFilter(r gjson.Result) bool { + return r.Get("type").Str == "m.room.message" || historicalEventFilter(r) +} + +func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string) { t.Helper() + debugStrings, err := _getRelevantEventDebugStringsFromMessagesResponse(body, eventFilter) + if err != nil { + t.Fatal(err) + } + + return debugStrings +} + +func _getRelevantEventDebugStringsFromMessagesResponse(body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string, err error) { wantKey := "chunk" res := gjson.GetBytes(body, wantKey) if !res.Exists() { - t.Fatalf("missing key '%s'", wantKey) + return nil, fmt.Errorf("missing key '%s'", wantKey) } if !res.IsArray() { - t.Fatalf("key '%s' is not an array (was %s)", wantKey, res.Type) + return nil, fmt.Errorf("key '%s' is not an array (was %s)", wantKey, res.Type) } res.ForEach(func(key, r gjson.Result) bool { - if isRelevantEvent(r) { + if eventFilter(r) { eventIDsFromResponse = append(eventIDsFromResponse, r.Get("event_id").Str+" ("+r.Get("content").Get("body").Str+")") } return true }) - return eventIDsFromResponse + return eventIDsFromResponse, nil } // ensureVirtualUserRegistered makes sure the user is registered for the homeserver regardless @@ -1274,7 +1288,7 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen expectedEventIDOrder = append(expectedEventIDOrder, baseInsertionEventID) } expectedEventIDOrder = append(expectedEventIDOrder, batchEventID) - expectedEventIDOrder = append(expectedEventIDOrder, historicalEventIDs...) + expectedEventIDOrder = append(expectedEventIDOrder, reversed(historicalEventIDs)...) expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID) if validateState { @@ -1303,12 +1317,9 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen must.MatchResponse(t, messagesRes, match.HTTPResponse{ JSON: []match.JSON{ // Double-check that we're in the right place of scrollback - match.JSONCheckOff("chunk", - makeInterfaceSlice(expectedEventIDOrder), - func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, - nil, + matcherJSONEventIDArrayInOrder("chunk", + expectedEventIDOrder, + historicalEventFilter, ), // Make sure the historical m.room.member join state event resolves // for the given chunk of messages in scrollback. The member event @@ -1328,9 +1339,85 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen })) must.MatchResponse(t, fullMessagesRes, match.HTTPResponse{ JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), + matcherJSONEventIDArrayInOrder("chunk", + expectedEventIDOrder, + historicalEventFilter, + ), }, }) } + +// Looks through a list of events to find the sliding window of expected event +// ID's somewhere in the list in order. The expected list can start anywhere in +// the overall list. +func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []string, eventFilter func(gjson.Result) bool) match.JSON { + return func(body []byte) error { + if len(expectedEventIDOrder) == 0 { + return fmt.Errorf("expectedEventIDOrder can not be an empty list") + } + + // Copy the array by slice so we can modify it as we iterate in the foreach loop. + // We save the full untouched `expectedEventIDOrder` for use in the log messages + workingExpectedEventIDOrder := expectedEventIDOrder + logrus.WithFields(logrus.Fields{ + "workingExpectedEventIDOrder": workingExpectedEventIDOrder, + }).Error("matcherJSONEventIDArrayInOrder") + + var res gjson.Result + if wantKey == "" { + res = gjson.ParseBytes(body) + } else { + res = gjson.GetBytes(body, wantKey) + } + + if !res.Exists() { + return fmt.Errorf("missing key '%s'", wantKey) + } + if !res.IsArray() { + return fmt.Errorf("key '%s' is not an array", wantKey) + } + + eventDebugStringsFromResponse, err := _getRelevantEventDebugStringsFromMessagesResponse(body, eventFilter) + if err != nil { + return err + } + + foundFirstEvent := false + res.ForEach(func(_, r gjson.Result) bool { + eventID := r.Get("event_id").Str + logrus.WithFields(logrus.Fields{ + "workingExpectedEventIDOrder": workingExpectedEventIDOrder, + }).Error("matcherJSONEventIDArrayInOrderf faewafewafew") + nextEventIdInOrder := workingExpectedEventIDOrder[0] + + // We need to find the start of the sliding window inside the overall + // event list + if !foundFirstEvent && eventID == nextEventIdInOrder { + foundFirstEvent = true + } + + // Once we found the first event, find all events in order + if foundFirstEvent && eventFilter(r) { + if r.Get("event_id").Str != nextEventIdInOrder { + err = fmt.Errorf("Next event found was %s but expected %s\nActualEvents (%d): %v\nExpectedEvents (%d): %v", r.Get("event_id").Str, nextEventIdInOrder, len(eventDebugStringsFromResponse), eventDebugStringsFromResponse, len(expectedEventIDOrder), expectedEventIDOrder) + } + + // Now that we found it, pop the message off the expected list + workingExpectedEventIDOrder = workingExpectedEventIDOrder[1:] + } + + // Found all of the expected events, stop iterating + if len(workingExpectedEventIDOrder) == 0 { + return false + } + + return err == nil + }) + + if len(workingExpectedEventIDOrder) != 0 { + return fmt.Errorf("Expected all events to be matched in message response but there were some left-over events: %s", workingExpectedEventIDOrder) + } + + return err + } +} From c8522c0ba8ed0b587be8175c5352aa22cf3a54cd Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 27 Oct 2021 23:19:18 -0500 Subject: [PATCH 08/17] Cleanup debug logs and plumb through wantKey --- tests/msc2716_test.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 21616e1b..a3315317 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -16,7 +16,6 @@ import ( "testing" "time" - "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/matrix-org/complement/internal/b" @@ -177,7 +176,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) messsageResBody := client.ParseJSON(t, messagesRes) - eventDebugStringsFromResponse := getRelevantEventDebugStringsFromMessagesResponse(t, messsageResBody, relevantToScrollbackEventFilter) + eventDebugStringsFromResponse := getRelevantEventDebugStringsFromMessagesResponse(t, "chunk", messsageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read messagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(messsageResBody)) @@ -705,7 +704,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) - eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) + eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, "chunk", beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) @@ -794,7 +793,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) - eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) + eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, "chunk", beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) // Make sure the history isn't visible before we expect it to be there. @@ -1010,10 +1009,10 @@ func relevantToScrollbackEventFilter(r gjson.Result) bool { return r.Get("type").Str == "m.room.message" || historicalEventFilter(r) } -func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string) { +func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, wantKey string, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string) { t.Helper() - debugStrings, err := _getRelevantEventDebugStringsFromMessagesResponse(body, eventFilter) + debugStrings, err := _getRelevantEventDebugStringsFromMessagesResponse(wantKey, body, eventFilter) if err != nil { t.Fatal(err) } @@ -1021,8 +1020,7 @@ func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, body []byte, return debugStrings } -func _getRelevantEventDebugStringsFromMessagesResponse(body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string, err error) { - wantKey := "chunk" +func _getRelevantEventDebugStringsFromMessagesResponse(wantKey string, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string, err error) { res := gjson.GetBytes(body, wantKey) if !res.Exists() { return nil, fmt.Errorf("missing key '%s'", wantKey) @@ -1359,9 +1357,6 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin // Copy the array by slice so we can modify it as we iterate in the foreach loop. // We save the full untouched `expectedEventIDOrder` for use in the log messages workingExpectedEventIDOrder := expectedEventIDOrder - logrus.WithFields(logrus.Fields{ - "workingExpectedEventIDOrder": workingExpectedEventIDOrder, - }).Error("matcherJSONEventIDArrayInOrder") var res gjson.Result if wantKey == "" { @@ -1377,7 +1372,7 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin return fmt.Errorf("key '%s' is not an array", wantKey) } - eventDebugStringsFromResponse, err := _getRelevantEventDebugStringsFromMessagesResponse(body, eventFilter) + eventDebugStringsFromResponse, err := _getRelevantEventDebugStringsFromMessagesResponse("chunk", body, eventFilter) if err != nil { return err } @@ -1385,9 +1380,6 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin foundFirstEvent := false res.ForEach(func(_, r gjson.Result) bool { eventID := r.Get("event_id").Str - logrus.WithFields(logrus.Fields{ - "workingExpectedEventIDOrder": workingExpectedEventIDOrder, - }).Error("matcherJSONEventIDArrayInOrderf faewafewafew") nextEventIdInOrder := workingExpectedEventIDOrder[0] // We need to find the start of the sliding window inside the overall From 2de716b83831e6c01bcb57a6146a78c9e27cd815 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 27 Oct 2021 23:59:53 -0500 Subject: [PATCH 09/17] Use the validation for other tests --- tests/msc2716_test.go | 138 +++++++----------------------------------- 1 file changed, 23 insertions(+), 115 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index a3315317..0d96838b 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -85,9 +85,9 @@ func TestImportHistoricalMessages(t *testing.T) { // come back in the correct order from /messages. // // Final timeline output: ( [n] = historical batch ) - // (oldest) A, B, [insertion, c, d, e, batch] [insertion, f, g, h, batch, insertion], I, J (newest) + // (oldest) A, B, [insertion, c, d, e, batch] [insertion, f, g, h, batch], baseInsertion, I, J (newest) // historical batch 1 historical batch 0 - t.Run("Historical events resolve with proper state in correct order", func(t *testing.T) { + t.Run("Historical events resolve in the correct order", func(t *testing.T) { t.Parallel() roomID := as.CreateRoom(t, createPublicRoomOpts) @@ -175,37 +175,15 @@ func TestImportHistoricalMessages(t *testing.T) { "dir": []string{"b"}, "limit": []string{"100"}, })) - messsageResBody := client.ParseJSON(t, messagesRes) - eventDebugStringsFromResponse := getRelevantEventDebugStringsFromMessagesResponse(t, "chunk", messsageResBody, relevantToScrollbackEventFilter) - // Since the original body can only be read once, create a new one from the body bytes we just read - messagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(messsageResBody)) - - // Copy the array by slice so we can modify it as we iterate in the foreach loop. - // We save the full untouched `expectedEventIDOrder` for use in the log messages - workingExpectedEventIDOrder := expectedEventIDOrder must.MatchResponse(t, messagesRes, match.HTTPResponse{ JSON: []match.JSON{ - match.JSONArrayEach("chunk", func(r gjson.Result) error { - // Find all events in order - if relevantToScrollbackEventFilter(r) { - // Pop the next message off the expected list - nextEventIdInOrder := workingExpectedEventIDOrder[0] - workingExpectedEventIDOrder = workingExpectedEventIDOrder[1:] - - if r.Get("event_id").Str != nextEventIdInOrder { - return fmt.Errorf("Next event found was %s but expected %s\nActualEvents (%d): %v\nExpectedEvents (%d): %v", r.Get("event_id").Str, nextEventIdInOrder, len(eventDebugStringsFromResponse), eventDebugStringsFromResponse, len(expectedEventIDOrder), expectedEventIDOrder) - } - } - - return nil - }), + matcherJSONEventIDArrayInOrder("chunk", + expectedEventIDOrder, + relevantToScrollbackEventFilter, + ), }, }) - - if len(workingExpectedEventIDOrder) != 0 { - t.Fatalf("Expected all events to be matched in message response but there were some left-over events: %s", workingExpectedEventIDOrder) - } }) t.Run("Historical events from multiple users in the same batch", func(t *testing.T) { @@ -239,21 +217,7 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") - - messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + validateBatchSendRes(t, as, roomID, batchSendRes, false) }) t.Run("Historical events from /batch_send do not come down in an incremental sync", func(t *testing.T) { @@ -555,8 +519,6 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") // Join the room from a remote homeserver after the historical messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -570,18 +532,7 @@ func TestImportHistoricalMessages(t *testing.T) { return false }) - messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) }) t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { @@ -627,8 +578,6 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") // Join the room from a remote homeserver after the historical messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -642,18 +591,7 @@ func TestImportHistoricalMessages(t *testing.T) { return false }) - messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) }) t.Run("Historical messages are visible when already joined on federated server", func(t *testing.T) { @@ -729,19 +667,8 @@ func TestImportHistoricalMessages(t *testing.T) { // Send the marker event sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) - remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - // Make sure all of the historical messages are visible when we scrollback again - must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + // Make sure we can see all of the historical messages + validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) }) t.Run("When messages have already been scrolled back through, new historical messages are visible in next scroll back on federated server", func(t *testing.T) { @@ -816,19 +743,8 @@ func TestImportHistoricalMessages(t *testing.T) { // Send the marker event sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) - remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - // Make sure all of the historical messages are visible when we scrollback again - must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + // Make sure we can see all of the historical messages + validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) }) t.Run("Existing room versions", func(t *testing.T) { @@ -865,26 +781,16 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) + // Make sure we can see all of the historical messages + validateBatchSendRes(t, as, roomID, batchSendRes, false) + // Grab the `next_batch_id` for the next batch batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") nextBatchID := client.GetJSONFieldStr(t, batchSendResBody, "next_batch_id") - messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) - - // Now try to do a subsequent batch send. This will make sure - // that insertion events are stored/tracked and can be matched up in the next batch - batchSendHistoricalMessages( + // Now try to do a subsequent batch send. This will make sure that + // insertion events are stored/tracked and can be matched up in the next + // batch + batchSendRes1 := batchSendHistoricalMessages( t, as, roomID, @@ -895,6 +801,8 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) + // Make sure we can see all of the historical messages + validateBatchSendRes(t, as, roomID, batchSendRes1, false) }) t.Run("Not allowed to redact MSC2716 insertion, batch, marker events", func(t *testing.T) { @@ -1407,7 +1315,7 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin }) if len(workingExpectedEventIDOrder) != 0 { - return fmt.Errorf("Expected all events to be matched in message response but there were some left-over events: %s", workingExpectedEventIDOrder) + return fmt.Errorf("Expected all events to be matched in message response but there were some left-over events (%d): %s\nActualEvents (%d): %v\nExpectedEvents (%d): %v", len(workingExpectedEventIDOrder), workingExpectedEventIDOrder, len(eventDebugStringsFromResponse), eventDebugStringsFromResponse, len(expectedEventIDOrder), expectedEventIDOrder) } return err From 36765a8a2b4f21bb8c50368a1239a32cc52b2423 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 28 Oct 2021 00:15:36 -0500 Subject: [PATCH 10/17] More comments --- tests/msc2716_test.go | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 0d96838b..5cfefb9b 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -471,7 +471,14 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - validateBatchSendRes(t, as, roomID, batchSendRes0, true) + validateBatchSendRes( + t, + as, + roomID, + batchSendRes0, + // Validate the state + true, + ) batchSendResBody0 := client.ParseJSON(t, batchSendRes0) nextBatchID0 := client.GetJSONFieldStr(t, batchSendResBody0, "next_batch_id") @@ -488,7 +495,14 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - validateBatchSendRes(t, as, roomID, batchSendRes1, true) + validateBatchSendRes( + t, + as, + roomID, + batchSendRes1, + // Validate the state + true, + ) }) t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) { @@ -1169,12 +1183,12 @@ func batchSendHistoricalMessages( return res } -// Verify that the batch of historical messages looks correct in the message -// scrollback. We also check that the historical state resolves for that chunk -// of messages. +// Verify that the batch of historical messages looks correct and in order in +// the message scrollback. We can also optionally check that the historical +// state resolves for that chunk of messages. // -// Note: the historical state will only resolve correctly if the -// first message of `/messages` is one of messages in the historical batch. +// Note: the historical state will only resolve correctly if the first message +// of `/messages` is one of messages in the historical batch. func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSendRes *http.Response, validateState bool) { t.Helper() @@ -1285,6 +1299,7 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin return err } + // Loop through the overall event list foundFirstEvent := false res.ForEach(func(_, r gjson.Result) bool { eventID := r.Get("event_id").Str @@ -1314,6 +1329,8 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin return err == nil }) + // There was some left-over events in the list but we should have found all + // of them if len(workingExpectedEventIDOrder) != 0 { return fmt.Errorf("Expected all events to be matched in message response but there were some left-over events (%d): %s\nActualEvents (%d): %v\nExpectedEvents (%d): %v", len(workingExpectedEventIDOrder), workingExpectedEventIDOrder, len(eventDebugStringsFromResponse), eventDebugStringsFromResponse, len(expectedEventIDOrder), expectedEventIDOrder) } From d35d7c341b4a72bcdf12cf6a571ea4921b6ca4aa Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 3 Nov 2021 03:09:48 -0500 Subject: [PATCH 11/17] Revert federated tests back to less strict order assertions See https://github.com/matrix-org/complement/pull/206#discussion_r738024318 --- tests/msc2716_test.go | 88 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 5cfefb9b..ed818d53 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -50,13 +50,13 @@ var ( var createPublicRoomOpts = map[string]interface{}{ "preset": "public_chat", "name": "the hangout spot", - "room_version": "org.matrix.msc2716v3", + "room_version": "org.matrix.msc2716v4", } var createPrivateRoomOpts = map[string]interface{}{ "preset": "private_chat", "name": "the hangout spot", - "room_version": "org.matrix.msc2716v3", + "room_version": "org.matrix.msc2716v4", } func TestImportHistoricalMessages(t *testing.T) { @@ -533,6 +533,8 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") // Join the room from a remote homeserver after the historical messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -546,7 +548,24 @@ func TestImportHistoricalMessages(t *testing.T) { return false }) - validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + // FIXME: In the future, we should probably replace the following logic + // with `validateBatchSendRes` to re-use and have some more robust + // assertion logic here. We're currently not using it because the message + // order isn't quite perfect when a remote federated homserver gets + // backfilled. + // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) }) t.Run("Historical messages are visible when joining on federated server - pre-made insertion event", func(t *testing.T) { @@ -592,6 +611,8 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) + batchSendResBody := client.ParseJSON(t, batchSendRes) + historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") // Join the room from a remote homeserver after the historical messages were sent remoteCharlie.JoinRoom(t, roomID, []string{"hs1"}) @@ -605,7 +626,24 @@ func TestImportHistoricalMessages(t *testing.T) { return false }) - validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + // FIXME: In the future, we should probably replace the following logic + // with `validateBatchSendRes` to re-use and have some more robust + // assertion logic here. We're currently not using it because the message + // order isn't quite perfect when a remote federated homserver gets + // backfilled. + // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + must.MatchResponse(t, messagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) }) t.Run("Historical messages are visible when already joined on federated server", func(t *testing.T) { @@ -681,8 +719,25 @@ func TestImportHistoricalMessages(t *testing.T) { // Send the marker event sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) - // Make sure we can see all of the historical messages - validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + // FIXME: In the future, we should probably replace the following logic + // with `validateBatchSendRes` to re-use and have some more robust + // assertion logic here. We're currently not using it because the message + // order isn't quite perfect when a remote federated homserver gets + // backfilled. + // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + // Make sure all of the historical messages are visible when we scrollback again + must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) }) t.Run("When messages have already been scrolled back through, new historical messages are visible in next scroll back on federated server", func(t *testing.T) { @@ -757,8 +812,25 @@ func TestImportHistoricalMessages(t *testing.T) { // Send the marker event sendMarkerAndEnsureBackfilled(t, as, remoteCharlie, roomID, baseInsertionEventID) - // Make sure we can see all of the historical messages - validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + // FIXME: In the future, we should probably replace the following logic + // with `validateBatchSendRes` to re-use and have some more robust + // assertion logic here. We're currently not using it because the message + // order isn't quite perfect when a remote federated homserver gets + // backfilled. + // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) + remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + "dir": []string{"b"}, + "limit": []string{"100"}, + })) + + // Make sure all of the historical messages are visible when we scrollback again + must.MatchResponse(t, remoteMessagesRes, match.HTTPResponse{ + JSON: []match.JSON{ + match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { + return r.Get("event_id").Str + }, nil), + }, + }) }) t.Run("Existing room versions", func(t *testing.T) { From a9eca781922b90676a822bc4bd1f080f4917b27e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Nov 2021 18:09:54 -0600 Subject: [PATCH 12/17] Fix homeserver typo --- tests/msc2716_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 735422d0..75005235 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -593,7 +593,7 @@ func TestImportHistoricalMessages(t *testing.T) { // FIXME: In the future, we should probably replace the following logic // with `validateBatchSendRes` to re-use and have some more robust // assertion logic here. We're currently not using it because the message - // order isn't quite perfect when a remote federated homserver gets + // order isn't quite perfect when a remote federated homeserver gets // backfilled. // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -671,7 +671,7 @@ func TestImportHistoricalMessages(t *testing.T) { // FIXME: In the future, we should probably replace the following logic // with `validateBatchSendRes` to re-use and have some more robust // assertion logic here. We're currently not using it because the message - // order isn't quite perfect when a remote federated homserver gets + // order isn't quite perfect when a remote federated homeserver gets // backfilled. // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -764,7 +764,7 @@ func TestImportHistoricalMessages(t *testing.T) { // FIXME: In the future, we should probably replace the following logic // with `validateBatchSendRes` to re-use and have some more robust // assertion logic here. We're currently not using it because the message - // order isn't quite perfect when a remote federated homserver gets + // order isn't quite perfect when a remote federated homeserver gets // backfilled. // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ @@ -857,7 +857,7 @@ func TestImportHistoricalMessages(t *testing.T) { // FIXME: In the future, we should probably replace the following logic // with `validateBatchSendRes` to re-use and have some more robust // assertion logic here. We're currently not using it because the message - // order isn't quite perfect when a remote federated homserver gets + // order isn't quite perfect when a remote federated homeserver gets // backfilled. // validateBatchSendRes(t, remoteCharlie, roomID, batchSendRes, false) remoteMessagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ From 7c0e41696b831cd2f9d9875c50e19cfc63cee9e0 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Nov 2021 18:16:07 -0600 Subject: [PATCH 13/17] Fix function naming to get relevant event debug strings > Naming: In cases where you have "do a thing" vs "do a thing and fail the test if there's an error" the general keyword to use here is `Must`. I would rename: > - `getRelevantEventDebugStringsFromMessagesResponse` -> `mustGetRelevantEventDebugStringsFromMessagesResponse` > - `_getRelevantEventDebugStringsFromMessagesResponse` -> `getRelevantEventDebugStringsFromMessagesResponse` > > -- https://github.com/matrix-org/complement/pull/206#discussion_r746515541 --- tests/msc2716_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 75005235..c0b9cff4 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -736,7 +736,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) - eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, "chunk", beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) + eventDebugStringsFromBeforeMarkerResponse := mustGetRelevantEventDebugStringsFromMessagesResponse(t, "chunk", beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) @@ -831,7 +831,7 @@ func TestImportHistoricalMessages(t *testing.T) { "limit": []string{"100"}, })) beforeMarkerMesssageResBody := client.ParseJSON(t, beforeMarkerMessagesRes) - eventDebugStringsFromBeforeMarkerResponse := getRelevantEventDebugStringsFromMessagesResponse(t, "chunk", beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) + eventDebugStringsFromBeforeMarkerResponse := mustGetRelevantEventDebugStringsFromMessagesResponse(t, "chunk", beforeMarkerMesssageResBody, relevantToScrollbackEventFilter) // Since the original body can only be read once, create a new one from the body bytes we just read beforeMarkerMessagesRes.Body = ioutil.NopCloser(bytes.NewBuffer(beforeMarkerMesssageResBody)) // Make sure the history isn't visible before we expect it to be there. @@ -1045,10 +1045,10 @@ func relevantToScrollbackEventFilter(r gjson.Result) bool { return r.Get("type").Str == "m.room.message" || historicalEventFilter(r) } -func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, wantKey string, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string) { +func mustGetRelevantEventDebugStringsFromMessagesResponse(t *testing.T, wantKey string, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string) { t.Helper() - debugStrings, err := _getRelevantEventDebugStringsFromMessagesResponse(wantKey, body, eventFilter) + debugStrings, err := getRelevantEventDebugStringsFromMessagesResponse(wantKey, body, eventFilter) if err != nil { t.Fatal(err) } @@ -1056,7 +1056,7 @@ func getRelevantEventDebugStringsFromMessagesResponse(t *testing.T, wantKey stri return debugStrings } -func _getRelevantEventDebugStringsFromMessagesResponse(wantKey string, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string, err error) { +func getRelevantEventDebugStringsFromMessagesResponse(wantKey string, body []byte, eventFilter func(gjson.Result) bool) (eventIDsFromResponse []string, err error) { res := gjson.GetBytes(body, wantKey) if !res.Exists() { return nil, fmt.Errorf("missing key '%s'", wantKey) @@ -1408,7 +1408,7 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin return fmt.Errorf("key '%s' is not an array", wantKey) } - eventDebugStringsFromResponse, err := _getRelevantEventDebugStringsFromMessagesResponse("chunk", body, eventFilter) + eventDebugStringsFromResponse, err := getRelevantEventDebugStringsFromMessagesResponse("chunk", body, eventFilter) if err != nil { return err } From 76f184d93c7891c1c4a3fcb7e7bfed84488329b8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Nov 2021 18:34:04 -0600 Subject: [PATCH 14/17] More clear comments for validateBatchSendRes - https://github.com/matrix-org/complement/pull/206#discussion_r746532259 - https://github.com/matrix-org/complement/pull/206#discussion_r746522162 --- tests/msc2716_test.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index c0b9cff4..8201723e 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -1297,9 +1297,9 @@ func batchSendHistoricalMessages( return res } -// Verify that the batch of historical messages looks correct and in order in -// the message scrollback. We can also optionally check that the historical -// state resolves for that chunk of messages. +// Verify that the batch of historical messages looks correct and in order +// (newest -> oldest) from /messages?dir=b. We can also optionally check that +// the historical state resolves for that chunk of messages. // // Note: the historical state will only resolve correctly if the first message // of `/messages` is one of messages in the historical batch. @@ -1317,6 +1317,7 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen insertionEventID := client.GetJSONFieldStr(t, batchSendResBody0, "insertion_event_id") baseInsertionEventID := gjson.GetBytes(batchSendResBody0, "base_insertion_event_id").Str + // Expected list is ordered from newest -> oldest event time var expectedEventIDOrder []string if baseInsertionEventID != "" { expectedEventIDOrder = append(expectedEventIDOrder, baseInsertionEventID) @@ -1326,7 +1327,7 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen expectedEventIDOrder = append(expectedEventIDOrder, insertionEventID) if validateState { - // Get the pagination token for the end of the historical batch itself + // Get a pagination token for the newest-in-time event in the historical batch itself contextRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "context", expectedEventIDOrder[0]}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "limit": []string{"0"}, })) @@ -1338,11 +1339,12 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen // message in the `chunk` and we want to be able assert that the historical // state is able to be resolved. messagesRes := c.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ + // Go backwards (newest -> oldest) (same direction as if you were using scrollback) "dir": []string{"b"}, - // From the end of the historical batch (this will be pointing at the ) + // From the newest-in-time event in the historical batch "from": []string{batchStartPaginationToken}, - // We are aiming to scrollback to the start of the existing historical - // messages + // We are aiming to scrollback to the oldest-in-time event from the + // historical batch "limit": []string{fmt.Sprintf("%d", len(expectedEventIDOrder))}, // We add these options to the filter so we get member events in the state field "filter": []string{"{\"lazy_load_members\":true,\"include_redundant_members\":true}"}, From 94a379fa0c290104bbae7fdd241807fa4629b8ce Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Nov 2021 18:55:49 -0600 Subject: [PATCH 15/17] Better control flow to return early See https://github.com/matrix-org/complement/pull/206#discussion_r746527505 --- tests/msc2716_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 8201723e..72e65951 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -1431,6 +1431,7 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin if foundFirstEvent && eventFilter(r) { if r.Get("event_id").Str != nextEventIdInOrder { err = fmt.Errorf("Next event found was %s but expected %s\nActualEvents (%d): %v\nExpectedEvents (%d): %v", r.Get("event_id").Str, nextEventIdInOrder, len(eventDebugStringsFromResponse), eventDebugStringsFromResponse, len(expectedEventIDOrder), expectedEventIDOrder) + return false } // Now that we found it, pop the message off the expected list @@ -1442,7 +1443,7 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin return false } - return err == nil + return true }) // There was some left-over events in the list but we should have found all From d14fa07388ef61d260b45afb52dff2fb19b43302 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Nov 2021 19:04:04 -0600 Subject: [PATCH 16/17] Document function inputs and better name See https://github.com/matrix-org/complement/pull/206#discussion_r746527827 --- tests/msc2716_test.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 72e65951..9f190120 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -1383,10 +1383,11 @@ func validateBatchSendRes(t *testing.T, c *client.CSAPI, roomID string, batchSen }) } -// Looks through a list of events to find the sliding window of expected event -// ID's somewhere in the list in order. The expected list can start anywhere in -// the overall list. -func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []string, eventFilter func(gjson.Result) bool) match.JSON { +// matcherJSONEventIDArrayInOrder loops through `jsonArrayKey` in the response +// to find the sliding window of expected event ID's(`expectedEventIDOrder`) +// somewhere in the array in order. The expected list can start anywhere in the +// overall list filtered by `eventFilter`. +func matcherJSONEventIDArrayInOrder(jsonArrayKey string, expectedEventIDOrder []string, eventFilter func(gjson.Result) bool) match.JSON { return func(body []byte) error { if len(expectedEventIDOrder) == 0 { return fmt.Errorf("expectedEventIDOrder can not be an empty list") @@ -1397,17 +1398,17 @@ func matcherJSONEventIDArrayInOrder(wantKey string, expectedEventIDOrder []strin workingExpectedEventIDOrder := expectedEventIDOrder var res gjson.Result - if wantKey == "" { + if jsonArrayKey == "" { res = gjson.ParseBytes(body) } else { - res = gjson.GetBytes(body, wantKey) + res = gjson.GetBytes(body, jsonArrayKey) } if !res.Exists() { - return fmt.Errorf("missing key '%s'", wantKey) + return fmt.Errorf("missing key '%s'", jsonArrayKey) } if !res.IsArray() { - return fmt.Errorf("key '%s' is not an array", wantKey) + return fmt.Errorf("key '%s' is not an array", jsonArrayKey) } eventDebugStringsFromResponse, err := getRelevantEventDebugStringsFromMessagesResponse("chunk", body, eventFilter) From 6df5a5331daea2dc1d40664e2cf51e8246fd48f5 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 10 Nov 2021 19:13:25 -0600 Subject: [PATCH 17/17] Use validateBatchSendRes for test merged in from master See https://github.com/matrix-org/complement/pull/206#discussion_r747068086 --- tests/msc2716_test.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tests/msc2716_test.go b/tests/msc2716_test.go index 9f190120..fd862bb9 100644 --- a/tests/msc2716_test.go +++ b/tests/msc2716_test.go @@ -530,21 +530,14 @@ func TestImportHistoricalMessages(t *testing.T) { // Status 200, ) - batchSendResBody := client.ParseJSON(t, batchSendRes) - historicalEventIDs := client.GetJSONFieldStringArray(t, batchSendResBody, "event_ids") - messagesRes := alice.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ - "dir": []string{"b"}, - "limit": []string{"100"}, - })) - - must.MatchResponse(t, messagesRes, match.HTTPResponse{ - JSON: []match.JSON{ - match.JSONCheckOffAllowUnwanted("chunk", makeInterfaceSlice(historicalEventIDs), func(r gjson.Result) interface{} { - return r.Get("event_id").Str - }, nil), - }, - }) + validateBatchSendRes( + t, + as, + roomID, + batchSendRes, + false, + ) }) t.Run("TODO: What happens when you point multiple batches at the same insertion event?", func(t *testing.T) {