From 99d28b721b52eb62e76a5addb0ebcad40d261e49 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Fri, 15 Sep 2023 08:57:30 +0530 Subject: [PATCH 1/8] chore | removing entities edge list formation in api trace graph --- .../trace/util/GraphBuilderUtil.java | 19 +------------------ .../util/StructuredTraceGraphBuilder.java | 16 ++++------------ .../trace/util/GraphBuilderUtilTest.java | 2 -- 3 files changed, 5 insertions(+), 32 deletions(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtil.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtil.java index 188977f10..dd1653078 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtil.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtil.java @@ -23,9 +23,7 @@ static boolean isDifferentTrace(StructuredTrace cachedTrace, StructuredTrace tra } static boolean isStructuredTraceChanged(StructuredTrace cachedTrace, StructuredTrace trace) { - return isDifferentTrace(cachedTrace, trace) - || isTraceEventsChanged(cachedTrace, trace) - || isTraceEntitiesChanged(cachedTrace, trace); + return isDifferentTrace(cachedTrace, trace) || isTraceEventsChanged(cachedTrace, trace); } /** Check if the events or theirs edges has changed */ @@ -41,19 +39,4 @@ static boolean isTraceEventsChanged(StructuredTrace cachedTrace, StructuredTrace } return false; } - - /** Check if the entities or theirs edges has changed */ - static boolean isTraceEntitiesChanged(StructuredTrace cachedTrace, StructuredTrace trace) { - - // trace entities internally changed (full trace comparison is costly, so we are doing only with - // required fields) - if (isDifferentTrace(cachedTrace, trace) - || cachedTrace.getEntityList().size() != trace.getEntityList().size() - || cachedTrace.getEntityEdgeList().size() != trace.getEntityEdgeList().size()) { - LOG.debug( - "Cached and Input trace are not same. Reason: they are having different size either for entities"); - return true; - } - return false; - } } diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index 8a054c27d..1cab7995f 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory; public class StructuredTraceGraphBuilder { + private static final Logger LOG = LoggerFactory.getLogger(StructuredTraceGraphBuilder.class); private static final ThreadLocal cachedGraphThreadLocal = @@ -19,12 +20,7 @@ public static StructuredTraceGraph buildGraph(StructuredTrace trace) { StructuredTraceGraph cachedGraph = cachedGraphThreadLocal.get(); boolean shouldRebuildTraceEventsGraph = GraphBuilderUtil.isTraceEventsChanged(cachedTrace, trace); - boolean shouldRebuildTraceEntitiesGraph = - GraphBuilderUtil.isTraceEntitiesChanged(cachedTrace, trace); - - if (null == cachedGraph - || GraphBuilderUtil.isDifferentTrace(cachedTrace, trace) - || (shouldRebuildTraceEventsGraph && shouldRebuildTraceEntitiesGraph)) { + if (null == cachedGraph) { Instant start = Instant.now(); StructuredTraceGraph graph = new StructuredTraceGraph(trace); if (LOG.isDebugEnabled()) { @@ -39,13 +35,9 @@ public static StructuredTraceGraph buildGraph(StructuredTrace trace) { return graph; } - if (shouldRebuildTraceEventsGraph || shouldRebuildTraceEntitiesGraph) { + if (shouldRebuildTraceEventsGraph) { Instant start = Instant.now(); - if (shouldRebuildTraceEventsGraph) { - cachedGraph.reCreateTraceEventsGraph(trace); - } else { - cachedGraph.reCreateTraceEntitiesGraph(trace); - } + cachedGraph.reCreateTraceEventsGraph(trace); if (LOG.isDebugEnabled()) { LOG.debug( "Time taken in building TraceEventsGraph, duration_millis:{} for tenantId:{}", diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/test/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtilTest.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/test/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtilTest.java index bfe92f369..939d8cfe8 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/test/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtilTest.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/test/java/org/hypertrace/traceenricher/trace/util/GraphBuilderUtilTest.java @@ -115,7 +115,6 @@ public void testIsTraceEventsChanged() { when(underTestTrace.getEventEdgeList()).thenReturn(List.of(eventEdge1, eventEdge2)); Assertions.assertTrue(GraphBuilderUtil.isTraceEventsChanged(cachedTrace, underTestTrace)); - Assertions.assertFalse(GraphBuilderUtil.isTraceEntitiesChanged(cachedTrace, underTestTrace)); } @Test @@ -148,6 +147,5 @@ public void testIsTraceEntitiesChanged() { when(underTestTrace.getEventEdgeList()).thenReturn(List.of(eventEdge1, eventEdge2)); Assertions.assertFalse(GraphBuilderUtil.isTraceEventsChanged(cachedTrace, underTestTrace)); - Assertions.assertTrue(GraphBuilderUtil.isTraceEntitiesChanged(cachedTrace, underTestTrace)); } } From 98f389e5900d5124535c3ee373292583b1fdc454 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Fri, 15 Sep 2023 09:02:07 +0530 Subject: [PATCH 2/8] fix --- .../traceenricher/trace/util/StructuredTraceGraphBuilder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index 1cab7995f..38f2d4795 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -18,8 +18,6 @@ public class StructuredTraceGraphBuilder { public static StructuredTraceGraph buildGraph(StructuredTrace trace) { StructuredTrace cachedTrace = cachedTraceThreadLocal.get(); StructuredTraceGraph cachedGraph = cachedGraphThreadLocal.get(); - boolean shouldRebuildTraceEventsGraph = - GraphBuilderUtil.isTraceEventsChanged(cachedTrace, trace); if (null == cachedGraph) { Instant start = Instant.now(); StructuredTraceGraph graph = new StructuredTraceGraph(trace); @@ -35,6 +33,8 @@ public static StructuredTraceGraph buildGraph(StructuredTrace trace) { return graph; } + boolean shouldRebuildTraceEventsGraph = + GraphBuilderUtil.isTraceEventsChanged(cachedTrace, trace); if (shouldRebuildTraceEventsGraph) { Instant start = Instant.now(); cachedGraph.reCreateTraceEventsGraph(trace); From 44295936a525031ef6a3e41edddefa1fe09389dc Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Fri, 15 Sep 2023 09:05:33 +0530 Subject: [PATCH 3/8] fix --- .../trace/util/StructuredTraceGraphBuilder.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index 38f2d4795..aac4f825c 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -18,7 +18,7 @@ public class StructuredTraceGraphBuilder { public static StructuredTraceGraph buildGraph(StructuredTrace trace) { StructuredTrace cachedTrace = cachedTraceThreadLocal.get(); StructuredTraceGraph cachedGraph = cachedGraphThreadLocal.get(); - if (null == cachedGraph) { + if (null == cachedGraph || GraphBuilderUtil.isDifferentTrace(cachedTrace, trace)) { Instant start = Instant.now(); StructuredTraceGraph graph = new StructuredTraceGraph(trace); if (LOG.isDebugEnabled()) { @@ -60,7 +60,8 @@ private static void debugGraph( && (null == graph.getTraceEntitiesGraph() || null == graph.getTraceEventsGraph())) { LOG.info( logPrefix - + "StructuredTraceGraph is not built correctly, trace {}, Is events graph non-null: {}." + + "StructuredTraceGraph is not built correctly, trace {}, Is events graph non-null:" + + " {}." + " Is entities graph non-null: {}", trace, (null != graph.getTraceEventsGraph()), From 5911ebf2983c32b466c9f7656d9399c3cd4939e2 Mon Sep 17 00:00:00 2001 From: aman bansal Date: Thu, 28 Sep 2023 16:35:26 +0530 Subject: [PATCH 4/8] chore | remove entities in structured trace graph (#422) --- gradle/libs.versions.toml | 2 +- .../util/StructuredTraceGraphBuilder.java | 31 +++++-------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 263177735..ecca6eca7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -16,7 +16,7 @@ hypertrace-grpc-client-utils ={ module = "org.hypertrace.core.grpcutils:grpc-cli hypertrace-grpc-client-rxUtils = { module = "org.hypertrace.core.grpcutils:grpc-client-rx-utils", version.ref = "hypertrace-grpc-utils"} hypertrace-serviceFramework-metrics = { module = "org.hypertrace.core.serviceframework:platform-metrics", version.ref = "hypertrace-serviceFramework" } hypertrace-serviceFramework-framework = { module = "org.hypertrace.core.serviceframework:platform-service-framework", version.ref = "hypertrace-serviceFramework" } -hypertrace-data-model = { module ="org.hypertrace.core.datamodel:data-model", version = "0.1.27" } +hypertrace-data-model = { module ="org.hypertrace.core.datamodel:data-model", version = "0.1.29-SNAPSHOT" } hypertrace-kafkaStreams-framework = { module = "org.hypertrace.core.kafkastreams.framework:kafka-streams-framework", version.ref = "hypertrace-kafkaStreams" } hypertrace-kafkaStreams-avroPartitioners = { module = "org.hypertrace.core.kafkastreams.framework:avro-partitioners", version.ref = "hypertrace-kafkaStreams" } hypertrace-kafkaStreams-weightedGroupPartitioners = { module = "org.hypertrace.core.kafkastreams.framework:weighted-group-partitioner", version.ref = "hypertrace-kafkaStreams" } diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index aac4f825c..738482404 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -56,41 +56,24 @@ public static StructuredTraceGraph buildGraph(StructuredTrace trace) { private static void debugGraph( String logPrefix, StructuredTraceGraph graph, StructuredTrace trace) { - if (null != graph - && (null == graph.getTraceEntitiesGraph() || null == graph.getTraceEventsGraph())) { + if (null != graph && null == graph.getTraceEventsGraph()) { LOG.info( logPrefix + "StructuredTraceGraph is not built correctly, trace {}, Is events graph non-null:" - + " {}." - + " Is entities graph non-null: {}", + + " {}.", trace, - (null != graph.getTraceEventsGraph()), - (null != graph.getTraceEntitiesGraph())); + (null != graph.getTraceEventsGraph())); // build the graph again and check StructuredTraceGraph tempGraph = new StructuredTraceGraph(trace); LOG.info( - logPrefix - + "Recreating StructuredTraceGraph. Is events graph non-null: {}." - + " Is entities graph non-null: {}", - (null != tempGraph.getTraceEventsGraph()), - (null != tempGraph.getTraceEntitiesGraph())); + logPrefix + "Recreating StructuredTraceGraph. Is events graph non-null: {}.", + (null != tempGraph.getTraceEventsGraph())); tempGraph.reCreateTraceEventsGraph(trace); LOG.info( - logPrefix - + "Recreating events graph. Is events graph non-null: {}." - + " Is entities graph non-null: {}", - (null != tempGraph.getTraceEventsGraph()), - (null != tempGraph.getTraceEntitiesGraph())); - - tempGraph.reCreateTraceEntitiesGraph(trace); - LOG.info( - logPrefix - + "Recreating entities graph. Is events graph non-null: {}." - + " Is entities graph non-null: {}", - (null != tempGraph.getTraceEventsGraph()), - (null != tempGraph.getTraceEntitiesGraph())); + logPrefix + "Recreating events graph. Is events graph non-null: {}.", + (null != tempGraph.getTraceEventsGraph())); } } } From 2308264e60226f1a40cba11eeca9fd700050008a Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Wed, 18 Oct 2023 16:40:09 +0530 Subject: [PATCH 5/8] fix --- .../util/StructuredTraceGraphBuilder.java | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index 738482404..7b7490fb1 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -18,7 +18,8 @@ public class StructuredTraceGraphBuilder { public static StructuredTraceGraph buildGraph(StructuredTrace trace) { StructuredTrace cachedTrace = cachedTraceThreadLocal.get(); StructuredTraceGraph cachedGraph = cachedGraphThreadLocal.get(); - if (null == cachedGraph || GraphBuilderUtil.isDifferentTrace(cachedTrace, trace)) { + if (null == cachedGraph || GraphBuilderUtil.isDifferentTrace(cachedTrace, trace) + || GraphBuilderUtil.isTraceEventsChanged(cachedTrace, trace)) { Instant start = Instant.now(); StructuredTraceGraph graph = new StructuredTraceGraph(trace); if (LOG.isDebugEnabled()) { @@ -33,23 +34,6 @@ public static StructuredTraceGraph buildGraph(StructuredTrace trace) { return graph; } - boolean shouldRebuildTraceEventsGraph = - GraphBuilderUtil.isTraceEventsChanged(cachedTrace, trace); - if (shouldRebuildTraceEventsGraph) { - Instant start = Instant.now(); - cachedGraph.reCreateTraceEventsGraph(trace); - if (LOG.isDebugEnabled()) { - LOG.debug( - "Time taken in building TraceEventsGraph, duration_millis:{} for tenantId:{}", - Duration.between(start, Instant.now()).toMillis(), - trace.getCustomerId()); - } - cachedTraceThreadLocal.set(StructuredTrace.newBuilder(trace).build()); - cachedGraphThreadLocal.set(cachedGraph); - debugGraph("Case: Partially building the graph.", cachedGraph, trace); - return cachedGraph; - } - debugGraph("Case: Not building the graph.", cachedGraphThreadLocal.get(), trace); return cachedGraph; } From cdf0b7ce8cce323b6af46a825b42f1835203a18c Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Wed, 18 Oct 2023 16:49:56 +0530 Subject: [PATCH 6/8] fix vulnerability --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d0b4c51f1..ba2373001 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -52,7 +52,7 @@ uadetector-resources = { module = "net.sf.uadetector:uadetector-resources",versi reactivex-rxjava3 = { module = "io.reactivex.rxjava3:rxjava", version = "3.0.11" } -json-json = { module = "org.json:json", version = "20230618" } +json-json = { module = "org.json:json", version = "20231013" } #grpc dependency grpc-netty = { module = "io.grpc:grpc-netty", version.ref = "grpc" } From ca557100e6e73976f5bc7e16d8ac9162fd921b09 Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Wed, 18 Oct 2023 16:50:32 +0530 Subject: [PATCH 7/8] making it pretty --- .../traceenricher/trace/util/StructuredTraceGraphBuilder.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index 7b7490fb1..865ae60d9 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -18,7 +18,8 @@ public class StructuredTraceGraphBuilder { public static StructuredTraceGraph buildGraph(StructuredTrace trace) { StructuredTrace cachedTrace = cachedTraceThreadLocal.get(); StructuredTraceGraph cachedGraph = cachedGraphThreadLocal.get(); - if (null == cachedGraph || GraphBuilderUtil.isDifferentTrace(cachedTrace, trace) + if (null == cachedGraph + || GraphBuilderUtil.isDifferentTrace(cachedTrace, trace) || GraphBuilderUtil.isTraceEventsChanged(cachedTrace, trace)) { Instant start = Instant.now(); StructuredTraceGraph graph = new StructuredTraceGraph(trace); From c2cb43cfb5f15570faaa8ad2eea3c18b4b242cdf Mon Sep 17 00:00:00 2001 From: aman-bansal Date: Wed, 18 Oct 2023 17:34:29 +0530 Subject: [PATCH 8/8] fix debug --- .../trace/util/StructuredTraceGraphBuilder.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java index 865ae60d9..0a959e769 100644 --- a/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java +++ b/hypertrace-trace-enricher/hypertrace-trace-enricher-api/src/main/java/org/hypertrace/traceenricher/trace/util/StructuredTraceGraphBuilder.java @@ -54,11 +54,6 @@ private static void debugGraph( LOG.info( logPrefix + "Recreating StructuredTraceGraph. Is events graph non-null: {}.", (null != tempGraph.getTraceEventsGraph())); - - tempGraph.reCreateTraceEventsGraph(trace); - LOG.info( - logPrefix + "Recreating events graph. Is events graph non-null: {}.", - (null != tempGraph.getTraceEventsGraph())); } } }