diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/AgentJarIndex.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/AgentJarIndex.java index 168c916d994..cedc773bbfb 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/AgentJarIndex.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/AgentJarIndex.java @@ -15,8 +15,10 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.jar.JarFile; import java.util.zip.ZipEntry; @@ -101,6 +103,7 @@ static class IndexGenerator extends SimpleFileVisitor { private Path prefixRoot; private int prefixId; + private Map prefixMappings = new HashMap<>(); IndexGenerator(Path resourcesDir) { this.resourcesDir = resourcesDir; @@ -125,6 +128,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { prefixRoot = dir; prefixes.add(dir.getFileName() + "/"); prefixId = prefixes.size(); + prefixMappings.put(prefixId, dir.getFileName().toString()); } return FileVisitResult.CONTINUE; } @@ -143,10 +147,16 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { String entryKey = computeEntryKey(prefixRoot.relativize(file)); if (null != entryKey) { int existingPrefixId = prefixTrie.apply(entryKey); - if (-1 != existingPrefixId && prefixId != existingPrefixId) { + // warn if two subsections contain content under the same package prefix + // because we're then unable to redirect requests to the right submodule + // (ignore the two 'datadog.compiler' packages which allow duplication) + if (existingPrefixId > 0 && prefixId != existingPrefixId) { log.warn( - "Detected duplicate content under '{}'. Ensure your content is under a distinct directory.", - entryKey); + "Detected duplicate content '{}' under '{}', already seen in {}. Ensure your content is under a distinct directory.", + entryKey, + resourcesDir.relativize(file).getName(0), // prefix + prefixMappings.get(existingPrefixId) // previous prefix + ); } prefixTrie.put(entryKey, prefixId); if (entryKey.endsWith("*")) { @@ -185,10 +195,18 @@ private static String computeEntryKey(Path path) { } public static void main(String[] args) throws IOException { + if (args.length < 1) { + throw new IllegalArgumentException("Expected: resources-dir"); + } + Path resourcesDir = Paths.get(args[0]).toAbsolutePath(); + Path indexDir = resourcesDir; + if (args.length == 2) { + indexDir = Paths.get(args[1]).toAbsolutePath(); + } IndexGenerator indexGenerator = new IndexGenerator(resourcesDir); Files.walkFileTree(resourcesDir, indexGenerator); - indexGenerator.writeIndex(resourcesDir.resolve(AGENT_INDEX_FILE_NAME)); + indexGenerator.writeIndex(indexDir.resolve(AGENT_INDEX_FILE_NAME)); } } } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterIndex.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterIndex.java index 357c7118add..29537852213 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterIndex.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/InstrumenterIndex.java @@ -334,10 +334,10 @@ public void writeIndex(Path indexFile) throws IOException { */ public static void main(String[] args) throws IOException { if (args.length < 1) { - throw new IllegalArgumentException("Expected: resources-dir"); + throw new IllegalArgumentException("Expected: index-dir"); } - Path resourcesDir = Paths.get(args[0]).toAbsolutePath(); + Path indexDir = Paths.get(args[0]).toAbsolutePath(); // satisfy some instrumenters that cache matchers in initializers HierarchyMatchers.registerIfAbsent(HierarchyMatchers.simpleChecks()); @@ -345,7 +345,7 @@ public static void main(String[] args) throws IOException { IndexGenerator indexGenerator = new IndexGenerator(); indexGenerator.buildIndex(); - indexGenerator.writeIndex(resourcesDir.resolve(INSTRUMENTER_INDEX_NAME)); + indexGenerator.writeIndex(indexDir.resolve(INSTRUMENTER_INDEX_NAME)); } } } diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/KnownTypesIndex.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/KnownTypesIndex.java index d1edbf54af3..9f517b93cb3 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/KnownTypesIndex.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/KnownTypesIndex.java @@ -154,10 +154,10 @@ public void writeIndex(Path indexFile) throws IOException { */ public static void main(String[] args) throws IOException { if (args.length < 1) { - throw new IllegalArgumentException("Expected: resources-dir"); + throw new IllegalArgumentException("Expected: index-dir"); } - Path resourcesDir = Paths.get(args[0]).toAbsolutePath(); + Path indexDir = Paths.get(args[0]).toAbsolutePath(); // satisfy some instrumenters that cache matchers in initializers HierarchyMatchers.registerIfAbsent(HierarchyMatchers.simpleChecks()); @@ -165,7 +165,7 @@ public static void main(String[] args) throws IOException { IndexGenerator indexGenerator = new IndexGenerator(); indexGenerator.buildIndex(); - indexGenerator.writeIndex(resourcesDir.resolve(KNOWN_TYPES_INDEX_NAME)); + indexGenerator.writeIndex(indexDir.resolve(KNOWN_TYPES_INDEX_NAME)); } } } diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index a881ae920d3..6e9b2daf5e0 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -17,12 +17,21 @@ configurations { traceShadowInclude } +def includedAgentDir = project.layout.buildDirectory.dir("generated/included") +def includedJarFileTree = fileTree(includedAgentDir) + +tasks.named("processResources") { + dependsOn(includedJarFileTree) +} + // The special pre-check should be compiled with Java 6 to detect unsupported Java versions // and prevent issues for users that still using them. sourceSets { "main_java6" { java.srcDirs "${project.projectDir}/src/main/java6" + } + main.resources.srcDir(includedAgentDir) } def java6CompileTask = tasks.named("compileMain_java6Java") { @@ -52,9 +61,6 @@ def generalShadowJarConfig(ShadowJar shadowJarTask) { duplicatesStrategy = DuplicatesStrategy.FAIL - // Include AgentPreCheck compiled with Java 6. - from sourceSets.main_java6.output - // Remove some cruft from the final jar. // These patterns should NOT include **/META-INF/maven/**/pom.properties, which is // used to report our own dependencies, but we should remove the top-level metadata @@ -114,10 +120,13 @@ def generalShadowJarConfig(ShadowJar shadowJarTask) { } } -def includeShadowJar(TaskProvider includedShadowJarTask, String destinationDir) { - def opentracingFound = new AtomicBoolean() - project.tasks.named("processResources", ProcessResources) { - doFirst { +def includeShadowJar(TaskProvider includedShadowJarTask, String agentDir, FileTree includedJarFileTree) { + def expandTask = project.tasks.register("expandAgentShadowJar${agentDir.capitalize()}", Sync) { + it.group = LifecycleBasePlugin.BUILD_GROUP + it.description = "Expand the included shadow jar into the agent jar under ${agentDir}" + + def opentracingFound = new AtomicBoolean() + it.doFirst("detect-open-tracing") { eachFile { // We seem unlikely to use this name somewhere else. if (it.path.contains("opentracing") && it.name.contains("Format\$Builtin")) { @@ -125,18 +134,18 @@ def includeShadowJar(TaskProvider includedShadowJarTask, String desti } } } - doLast { + it.doLast("fail-on-detected-opentracing") { if (opentracingFound.get()) { throw new GradleException("OpenTracing direct dependency found!") } } - from(zipTree(includedShadowJarTask.map { it.archiveFile })) { - into destinationDir + it.into providers.provider { new File(includedJarFileTree.dir, agentDir) } + it.from(zipTree(includedShadowJarTask.map { it.archiveFile })) { rename '(^.*)\\.class$', '$1.classdata' // Rename LICENSE file since it clashes with license dir on non-case sensitive FSs (i.e. Mac) rename '^LICENSE$', 'LICENSE.renamed' - if (destinationDir == 'inst') { + if (agentDir == 'inst') { // byte-buddy now ships classes optimized for Java8+ under META-INF/versions/9 // since we target Java8+ we can promote these classes over the pre-Java8 ones duplicatesStrategy = DuplicatesStrategy.EXCLUDE @@ -148,39 +157,41 @@ def includeShadowJar(TaskProvider includedShadowJarTask, String desti } } - dependsOn includedShadowJarTask + it.dependsOn includedShadowJarTask } + includedJarFileTree.builtBy(expandTask) + includedShadowJarTask.configure { generalShadowJarConfig(it as ShadowJar) } } -def includeSubprojShadowJar(Project includedProjectJar, String destinationDir) { +def includeSubprojShadowJar(Project includedProjectJar, String destinationDir, FileTree includedJarFileTree) { evaluationDependsOn(includedProjectJar.path) - includeShadowJar(includedProjectJar.tasks.named("shadowJar", ShadowJar), destinationDir) + includeShadowJar(includedProjectJar.tasks.named("shadowJar", ShadowJar), destinationDir, includedJarFileTree) } -includeSubprojShadowJar(project(':dd-java-agent:instrumentation'), 'inst') -includeSubprojShadowJar(project(':dd-java-agent:agent-jmxfetch'), 'metrics') -includeSubprojShadowJar(project(':dd-java-agent:agent-profiling'), 'profiling') -includeSubprojShadowJar(project(':dd-java-agent:appsec'), 'appsec') -includeSubprojShadowJar(project(':dd-java-agent:agent-aiguard'), 'aiguard') -includeSubprojShadowJar(project(':dd-java-agent:agent-iast'), 'iast') -includeSubprojShadowJar(project(':dd-java-agent:agent-debugger'), 'debugger') -includeSubprojShadowJar(project(':dd-java-agent:agent-ci-visibility'), 'ci-visibility') -includeSubprojShadowJar(project(':dd-java-agent:agent-llmobs'), 'llm-obs') -includeSubprojShadowJar(project(':dd-java-agent:agent-logs-intake'), 'logs-intake') -includeSubprojShadowJar(project(':dd-java-agent:cws-tls'), 'cws-tls') +includeSubprojShadowJar(project(':dd-java-agent:instrumentation'), 'inst', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-jmxfetch'), 'metrics', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-profiling'), 'profiling', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:appsec'), 'appsec', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-aiguard'), 'aiguard', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-iast'), 'iast', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-debugger'), 'debugger', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-ci-visibility'), 'ci-visibility', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-llmobs'), 'llm-obs', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:agent-logs-intake'), 'logs-intake', includedJarFileTree) +includeSubprojShadowJar(project(':dd-java-agent:cws-tls'), 'cws-tls', includedJarFileTree) def sharedShadowJar = tasks.register('sharedShadowJar', ShadowJar) { - configurations = [project.configurations.sharedShadowInclude] + it.configurations = [project.configurations.sharedShadowInclude] // Put the jar in a different directory so we don't overwrite the normal shadow jar and // break caching, and also to not interfere with CI scripts that copy everything in the // libs directory it.destinationDirectory.set(project.layout.buildDirectory.dir("shared-lib")) // Add a classifier so we don't confuse the jar file with the normal shadow jar - archiveClassifier = 'shared' + it.archiveClassifier = 'shared' it.dependencies { exclude(project(':dd-java-agent:agent-bootstrap')) exclude(project(':dd-java-agent:agent-logging')) @@ -192,18 +203,21 @@ def sharedShadowJar = tasks.register('sharedShadowJar', ShadowJar) { exclude(dependency('org.slf4j::')) } } -includeShadowJar(sharedShadowJar, 'shared') +includeShadowJar(sharedShadowJar, 'shared', includedJarFileTree) // place the tracer in its own shadow jar separate to instrumentation def traceShadowJar = tasks.register('traceShadowJar', ShadowJar) { - configurations = [project.configurations.traceShadowInclude] + it.configurations = [project.configurations.traceShadowInclude] it.destinationDirectory.set(project.layout.buildDirectory.dir("trace-lib")) - archiveClassifier = 'trace' + it.archiveClassifier = 'trace' it.dependencies deps.excludeShared } -includeShadowJar(traceShadowJar, 'trace') +includeShadowJar(traceShadowJar, 'trace', includedJarFileTree) tasks.named("shadowJar", ShadowJar) { + // Include AgentPreCheck compiled with Java 6. + from sourceSets.main_java6.output + generalShadowJarConfig(it) configurations = [project.configurations.shadowInclude] @@ -221,24 +235,35 @@ tasks.named("shadowJar", ShadowJar) { } } -tasks.register('generateAgentJarIndex', JavaExec) { - def indexName = 'dd-java-agent.index' - def contentDir = "${sourceSets.main.output.resourcesDir}" - def indexFile = "${contentDir}/${indexName}" +// temporary config to add slf4j-simple so we get logging while indexing +project.configurations.register('slf4j-simple') { + it.dependencies.add(project.dependencyFactory.create("org.slf4j:slf4j-simple:${libs.versions.slf4j.get()}")) +} + +def generateAgentJarIndex = tasks.register('generateAgentJarIndex', JavaExec) { + def destinationDir = project.layout.buildDirectory.dir("generated/${it.name}") - it.group = 'Build' + it.group = LifecycleBasePlugin.BUILD_GROUP it.description = "Generate dd-java-agent.index" - it.inputs.files(fileTree(contentDir).exclude(indexName)) - it.outputs.files(indexFile) it.mainClass = 'datadog.trace.bootstrap.AgentJarIndex$IndexGenerator' - it.classpath = project.configurations.shadowInclude - it.args = [contentDir] - dependsOn 'processResources' - dependsOn 'writeVersionNumberFile' + it.inputs.files(includedJarFileTree) + it.inputs.files(it.classpath) + it.outputs.dir(destinationDir) + it.classpath = objects.fileCollection().tap { + it.from(project.configurations.named("shadowInclude")) + it.from(project.configurations.named('slf4j-simple')) + } + // debuggable within gradle using: + // it.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005") + it.argumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + return [includedAgentDir.get().asFile.path, destinationDir.get().asFile.path,] + } + }) } - -compileJava.dependsOn 'generateAgentJarIndex' +sourceSets.main.resources.srcDir(generateAgentJarIndex) subprojects { Project subProj -> // Don't need javadoc task run for internal projects. diff --git a/dd-java-agent/cws-tls/build.gradle b/dd-java-agent/cws-tls/build.gradle index 68fcbd9dc6b..df9efdd2608 100644 --- a/dd-java-agent/cws-tls/build.gradle +++ b/dd-java-agent/cws-tls/build.gradle @@ -1,4 +1,5 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar +import org.apache.maven.model.License plugins { id 'com.gradleup.shadow' @@ -22,6 +23,10 @@ dependencies { tasks.named("shadowJar", ShadowJar) { dependencies deps.excludeShared - // exclude this since it's available in the instrumentation jar - exclude 'com/sun/jna/**/*' + + // exclude 'jna' this since it's available in the instrumentation jar + dependencies { + exclude(dependency("net.java.dev.jna:jna")) + exclude(dependency("net.java.dev.jna:jna-platform")) + } } diff --git a/dd-java-agent/instrumentation/build.gradle b/dd-java-agent/instrumentation/build.gradle index 5486148c710..abe627fdd78 100644 --- a/dd-java-agent/instrumentation/build.gradle +++ b/dd-java-agent/instrumentation/build.gradle @@ -114,44 +114,43 @@ tasks.named('shadowJar', ShadowJar) { dependencies deps.excludeShared } -tasks.register('generateInstrumenterIndex', JavaExec) { - // temporary config to add slf4j-simple so we get logging from instrumenters while indexing - def slf4jSimple = project.configurations.maybeCreate('slf4j-simple') - project.dependencies.add('slf4j-simple', "org.slf4j:slf4j-simple:${libs.versions.slf4j.get()}") - - def resourcesDir = "${sourceSets.main.output.resourcesDir}" - def indexFile = "${resourcesDir}/instrumenter.index" - - it.group = 'Build' - it.description = "Generate instrumenter.index" - it.mainClass = 'datadog.trace.agent.tooling.InstrumenterIndex$IndexGenerator' - it.classpath = project.configurations.runtimeClasspath + slf4jSimple - it.inputs.files(it.classpath) - it.outputs.files(indexFile) - it.args = [resourcesDir] - - dependsOn 'processResources' +// temporary config to add slf4j-simple so we get logging from instrumenters while indexing +project.configurations.register('slf4j-simple') { + it.dependencies.add(project.dependencyFactory.create("org.slf4j:slf4j-simple:${libs.versions.slf4j.get()}")) } -tasks.register('generateKnownTypesIndex', JavaExec) { - // temporary config to add slf4j-simple so we get logging from instrumenters while indexing - def slf4jSimple = project.configurations.maybeCreate('slf4j-simple') - project.dependencies.add('slf4j-simple', "org.slf4j:slf4j-simple:${libs.versions.slf4j.get()}") +TaskProvider registerIndexTask(String indexTaskName, String indexer, String description ) { + def indexTask = tasks.register(indexTaskName, JavaExec) { + def destinationDir = project.layout.buildDirectory.dir("generated/${it.name}") - def resourcesDir = "${sourceSets.main.output.resourcesDir}" - def indexFile = "${resourcesDir}/known-types.index" - - it.group = 'Build' - it.description = "Generate known-types.index" - it.mainClass = 'datadog.trace.agent.tooling.KnownTypesIndex$IndexGenerator' - it.classpath = project.configurations.runtimeClasspath + slf4jSimple - it.inputs.files(it.classpath) - it.outputs.files(indexFile) - it.args = [resourcesDir] - - dependsOn 'processResources' + it.group = LifecycleBasePlugin.BUILD_GROUP + it.description = description + it.mainClass = indexer + it.classpath = objects.fileCollection().tap { + it.from(project.configurations.named("runtimeClasspath")) + it.from(project.configurations.named('slf4j-simple')) + } + it.inputs.files(it.classpath) + it.outputs.dir(destinationDir) + it.argumentProviders.add(new CommandLineArgumentProvider() { + @Override + Iterable asArguments() { + return destinationDir.map { [it.asFile.path] }.get() + } + }) + } + sourceSets.main.resources.srcDir(indexTask) + return indexTask } -tasks.named("shadowJar", ShadowJar) { - dependsOn('generateInstrumenterIndex', 'generateKnownTypesIndex') -} +registerIndexTask( + 'generateInstrumenterIndex', + 'datadog.trace.agent.tooling.InstrumenterIndex$IndexGenerator', + 'Generate instrumenter.index' + ) + +registerIndexTask( + 'generateKnownTypesIndex', + 'datadog.trace.agent.tooling.KnownTypesIndex$IndexGenerator', + 'Generate known-types.index' + ) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index f4c8bd35b08..abae8406a78 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -45,6 +45,9 @@ final class CachedData { exclude(dependency('com.github.jnr::')) exclude(dependency('org.ow2.asm::')) + // javac plugin client + exclude(dependency('com.datadoghq:dd-javac-plugin-client')) + // moshi and its transitives exclude(dependency('com.squareup.moshi::'))