From 1faab51fda46ed6e04335dded08305e241d43b99 Mon Sep 17 00:00:00 2001 From: Ronak Date: Tue, 28 Jun 2022 13:49:58 +0530 Subject: [PATCH 1/4] fix: the integration test with latest pinot image for group.id change --- .../service/htqueries/HTPinotQueriesTest.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java index da0d3148..e9c6d58e 100644 --- a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java +++ b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java @@ -18,16 +18,21 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.avro.file.DataFileReader; import org.apache.avro.specific.SpecificDatumReader; import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.ConsumerGroupListing; import org.apache.kafka.clients.admin.ListConsumerGroupOffsetsResult; +import org.apache.kafka.clients.admin.ListConsumerGroupsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.producer.KafkaProducer; @@ -93,7 +98,7 @@ public static void setup() throws Exception { kafkaZk.start(); pinotServiceManager = - new GenericContainer<>(DockerImageName.parse("hypertrace/pinot-servicemanager:main")) + new GenericContainer<>(DockerImageName.parse("traceableai-docker.jfrog.io/hypertrace/pinot-servicemanager:test")) .withNetwork(network) .withNetworkAliases("pinot-controller", "pinot-server", "pinot-broker") .withExposedPorts(8099, 9000) @@ -262,21 +267,30 @@ private static boolean generateData() throws Exception { "service-call-view-events", 27L, "span-event-view", 50L, "log-event-view", 0L); - int retry = 0; - while (!areMessagesConsumed(endOffSetMap) && retry++ < 5) { - Thread.sleep(2000); + int retry = 0, maxRetries = 20; + while (!areMessagesConsumed(endOffSetMap) && retry++ < maxRetries) { + Thread.sleep(6000); // max 2 min wait time } // stop this service viewGen.stop(); - return retry < 5; + return retry < maxRetries; } private static boolean areMessagesConsumed(Map endOffSetMap) throws Exception { - ListConsumerGroupOffsetsResult consumerGroupOffsetsResult = - adminClient.listConsumerGroupOffsets(""); - Map offsetAndMetadataMap = - consumerGroupOffsetsResult.partitionsToOffsetAndMetadata().get(); + ListConsumerGroupsResult listConsumerGroups = adminClient.listConsumerGroups(); + List groupIds = listConsumerGroups.all().get().stream() + .filter(consumerGroupListing -> consumerGroupListing.isSimpleConsumerGroup()) + .map(consumerGroupListing -> consumerGroupListing.groupId()) + .collect(Collectors.toUnmodifiableList()); + + Map offsetAndMetadataMap = new HashMap<>(); + for(String groupId : groupIds) { + ListConsumerGroupOffsetsResult listConsumerGroupOffsetsResult = adminClient.listConsumerGroupOffsets(groupId); + Map metadataMap = listConsumerGroupOffsetsResult.partitionsToOffsetAndMetadata().get(); + metadataMap.forEach((k, v) -> offsetAndMetadataMap.putIfAbsent(k, v)); + } + if (offsetAndMetadataMap.size() < 6) { return false; } From fc8d0c8c82336ecb89569e6f7a3f9b37bb6a662e Mon Sep 17 00:00:00 2001 From: Ronak Date: Tue, 28 Jun 2022 15:17:04 +0530 Subject: [PATCH 2/4] making it 5 mins for worst case --- .../core/query/service/htqueries/HTPinotQueriesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java index e9c6d58e..a5c499bb 100644 --- a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java +++ b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java @@ -267,7 +267,7 @@ private static boolean generateData() throws Exception { "service-call-view-events", 27L, "span-event-view", 50L, "log-event-view", 0L); - int retry = 0, maxRetries = 20; + int retry = 0, maxRetries = 50; while (!areMessagesConsumed(endOffSetMap) && retry++ < maxRetries) { Thread.sleep(6000); // max 2 min wait time } From 1dd589f59451a404da3ec65d9720de6393728b2d Mon Sep 17 00:00:00 2001 From: Ronak Date: Tue, 28 Jun 2022 15:18:00 +0530 Subject: [PATCH 3/4] updated comments --- .../core/query/service/htqueries/HTPinotQueriesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java index a5c499bb..d36fa19a 100644 --- a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java +++ b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java @@ -269,7 +269,7 @@ private static boolean generateData() throws Exception { "log-event-view", 0L); int retry = 0, maxRetries = 50; while (!areMessagesConsumed(endOffSetMap) && retry++ < maxRetries) { - Thread.sleep(6000); // max 2 min wait time + Thread.sleep(6000); // max 5 min wait time } // stop this service viewGen.stop(); From 55856dfa3f13a24d303fbf9bb8ff1e5e52a2b0ee Mon Sep 17 00:00:00 2001 From: Ronak Date: Tue, 28 Jun 2022 16:13:45 +0530 Subject: [PATCH 4/4] reverted the test image --- .../core/query/service/htqueries/HTPinotQueriesTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java index d36fa19a..7ef25e67 100644 --- a/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java +++ b/query-service/src/integrationTest/java/org/hypertrace/core/query/service/htqueries/HTPinotQueriesTest.java @@ -98,7 +98,7 @@ public static void setup() throws Exception { kafkaZk.start(); pinotServiceManager = - new GenericContainer<>(DockerImageName.parse("traceableai-docker.jfrog.io/hypertrace/pinot-servicemanager:test")) + new GenericContainer<>(DockerImageName.parse("hypertrace/pinot-servicemanager:main")) .withNetwork(network) .withNetworkAliases("pinot-controller", "pinot-server", "pinot-broker") .withExposedPorts(8099, 9000)