From c2940ad996eb21ac65a631c91f08dc36c6aa4478 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Wed, 29 Jan 2025 15:51:01 +0100 Subject: [PATCH 1/8] Add checks, spotless, and test_published_artifact --- .gitlab-ci.yml | 421 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 414 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 334d20d22d0..7b5d8c521aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,21 +17,53 @@ stages: - ci-visibility-tests - generate-signing-key +#TODO display memory usage +#TODO test containers +#TODO paralization (muzzle) +#TODO publish tests to datadog +#TODO agent_integration_tests +#TODO no output timeout +#TODO xlarge_tests + variables: REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com BUILD_JOB_NAME: "build" DEPENDENCY_CACHE_POLICY: pull BUILD_CACHE_POLICY: pull GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties + JAVA_BUILD_IMAGE_VERSION: "v25.01" + PROFILE_TESTS: + description: "Enable profiling of tests" + value: "false" + SYSTEM_TESTS_COMMIT: + description: "Commit hash of system tests to use" + value: "8384a3947b9881d0fcdb5ac937b80aad9a70d4d4" + +.test_matrix: &test_matrix + - testJvm: + - "8" + - "semeru11" + - "oracle8" + - "21" + - "ubuntu17" + - "zulu8" + - "semeru8" + - "ibm8" + - "zulu11" + - "11" + - "17" + - "semeru17" default: tags: [ "arch:amd64" ] .gradle_build: &gradle_build - image: ghcr.io/datadog/dd-trace-java-docker-build:v25.01-base + image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base + stage: build variables: - GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'" - GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2" + GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + MAVEN_OPTS: "-Xms64M -Xmx512M" + GRADLE_WORKERS: 2 KUBERNETES_CPU_REQUEST: 8 KUBERNETES_MEMORY_REQUEST: 6Gi cache: @@ -42,7 +74,7 @@ default: - .gradle/caches - .gradle/notifications policy: $DEPENDENCY_CACHE_POLICY - - key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline + - key: $CI_PIPELINE_ID-$BUILD_CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type paths: - .gradle/caches/$GRADLE_VERSION - .gradle/$GRADLE_VERSION/executionHistory @@ -50,6 +82,7 @@ default: policy: $BUILD_CACHE_POLICY before_script: - export GRADLE_USER_HOME=`pwd`/.gradle + - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS" # for weird reasons, gradle will always "chmod 700" the .gradle folder # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails # This dance is a hack to have .gradle owned by the Gitlab runner user @@ -61,24 +94,37 @@ default: build: extends: .gradle_build - stage: build variables: BUILD_CACHE_POLICY: push + BUILD_CACHE_TYPE: lib DEPENDENCY_CACHE_POLICY: pull script: - - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS + - ./gradlew resolveAndLockAll --write-locks + - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env artifacts: + when: always paths: - 'workspace/dd-java-agent/build/libs/*.jar' - 'workspace/dd-trace-api/build/libs/*.jar' - 'workspace/dd-trace-ot/build/libs/*.jar' - 'upstream.env' + - '.gradle/daemon/*/*.out.log' reports: dotenv: build.env +build_tests: + extends: .gradle_build + variables: + BUILD_CACHE_POLICY: push + BUILD_CACHE_TYPE: test + DEPENDENCY_CACHE_POLICY: pull + script: + - ./gradlew resolveAndLockAll --write-locks + - ./gradlew clean :baseTest :instrumentationTest :instrumentationLatestDepTest :instrumentationTest :profilingTest -PskipTests $GRADLE_ARGS + build_and_populate_dep_cache: extends: build variables: @@ -90,6 +136,365 @@ build_and_populate_dep_cache: - when: manual allow_failure: true +spotless: + extends: .gradle_build + stage: tests + needs: [] + script: + - export JAVA_HOME=$JAVA_11_HOME + - ./gradlew spotlessCheck $GRADLE_ARGS + +.check_job: + extends: .gradle_build + needs: [ "build_tests" ] + stage: tests + rules: + - if: '$POPULATE_CACHE' + when: never + - when: always # TODO change to on_success once build_tests is stable + variables: + BUILD_CACHE_TYPE: test + script: + - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless $GRADLE_ARGS + after_script: + - .circleci/collect_reports.sh --destination ./check_reports --move + artifacts: + when: always + paths: + - ./check_reports + - '.gradle/daemon/*/*.out.log' + +check_base: + extends: .check_job + variables: + GRADLE_TARGET: ":baseCheck" + +check_inst: + extends: .check_job + variables: + GRADLE_TARGET: ":instrumentationCheck" + +check_smoke: + extends: .check_job + variables: + GRADLE_TARGET: ":smokeCheck" + +check_profiling: + extends: .check_job + variables: + GRADLE_TARGET: ":profilingCheck" + +check_debugger: + extends: .check_job + variables: + GRADLE_TARGET: ":debuggerCheck" + +test_published_artifacts: + extends: .gradle_build + image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests + stage: tests + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: on_success + variables: + BUILD_CACHE_TYPE: lib + script: + - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) + - rm -rf "${mvn_local_repo}/com/datadoghq" + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew publishToMavenLocal $GRADLE_ARGS + - cd test-published-dependencies + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew check --info $GRADLE_ARGS + after_script: + - .circleci/collect_reports.sh + artifacts: + when: always + paths: + - ./check_reports + +muzzle: + extends: .gradle_build + needs: [ build ] + stage: tests + parallel: 8 + rules: + - if: '$POPULATE_CACHE' + when: never + - when: on_success + variables: + BUILD_CACHE_TYPE: test + script: + - export SKIP_BUILDSCAN="true" + - echo CI_NODE_INDEX=$CI_NODE_INDEX + - echo CI_NODE_TOTAL=CI_NODE_TOTAL + - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS + - sort workspace/build/muzzleTasks > sortedMuzzleTasks + - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit + - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1)) + - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS + after_script: + - .circleci/collect_reports.sh + artifacts: + when: always + paths: + - ./reports + - '.gradle/daemon/*/*.out.log' + +muzzle-dep-report: + extends: .gradle_build + needs: [ build ] + stage: tests + variables: + BUILD_CACHE_TYPE: test + script: + - export SKIP_BUILDSCAN="true" + - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS + after_script: + - .circleci/collect_muzzle_deps.sh + artifacts: + when: always + paths: + - ./reports + - '.gradle/daemon/*/*.out.log' + +.test_job: + extends: .gradle_build + image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm + needs: [ build_tests ] + stage: tests + rules: + - if: '$POPULATE_CACHE' + when: never + - when: always # TODO change to on_success once build_tests is stable + variables: + BUILD_CACHE_TYPE: test + GRADLE_PARAMS: "" + CONTINUE_ON_FAILURE: "false" + script: + - > + if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; + then + export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true"; + fi + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M" + - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE + after_script: + - .circleci/collect_reports.sh + - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi + - .circleci/collect_results.sh + - .circleci/upload_ciapp.sh tests $testJvm +# TODO Get APM Test Agent Trace Check Results + artifacts: + when: always + paths: + - ./reports.tar + - ./profiles.tar + - ./results + - '.gradle/daemon/*/*.out.log' + +test_base: + extends: .test_job + variables: + GRADLE_TARGET: ":baseTest" + GRADLE_PARAMS: "-PskipFlakyTests" + parallel: + matrix: *test_matrix + script: + - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi + - !reference [.test_job, script] + +test_inst: + extends: .test_job + variables: + GRADLE_TARGET: ":instrumentationTest" + GRADLE_PARAMS: "-PskipFlakyTests" + parallel: + matrix: *test_matrix + +test_inst_latest: + extends: .test_job + variables: + GRADLE_TARGET: ":instrumentationLatestDepTest" + GRADLE_PARAMS: "-PskipFlakyTests" + parallel: + matrix: + - testJvm: ["8", "17", "21" ] + +test_flaky: + extends: .test_job + variables: + GRADLE_PARAMS: "-PrunFlakyTests" + testJvm: "8" + CONTINUE_ON_FAILURE: "true" + parallel: + matrix: + - GRADLE_TARGET: [":baseTest", ":instrumentationTest", ":smokeTest", ":debuggerTest"] + +test_profiling: + extends: .test_job + variables: + GRADLE_TARGET: ":profilingTest" + GRADLE_PARAMS: "-PskipFlakyTests" + parallel: + matrix: *test_matrix + +test_debugger: + extends: .test_job + variables: + GRADLE_TARGET: ":debuggerTest" + GRADLE_PARAMS: "-PskipFlakyTests" + parallel: + matrix: *test_matrix + +test_smoke: + extends: .test_job + variables: + GRADLE_TARGET: "stageMainDist :smokeTest" + GRADLE_PARAMS: "-PskipFlakyTests" + parallel: + matrix: *test_matrix + +test_ssi_smoke: + extends: .test_job + variables: + GRADLE_TARGET: "stageMainDist :smokeTest" + GRADLE_PARAMS: "-PskipFlakyTests" + DD_INJECT_FORCE: "true" + DD_INJECTION_ENABLED: "tracer" + parallel: + matrix: *test_matrix + +test_smoke_graalvm: + extends: .test_job + variables: + GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test" + parallel: + matrix: + - testJvm: ["graalvm17", "graalvm21"] + +test_smoke_semeru8_debugger: + extends: .test_job + variables: + GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" + GRADLE_PARAMS: "-PskipFlakyTests" + testJvm: "semeru8" + +.system-tests-job: + image: $SYSTEM_TESTS_IMAGE + tags: [ "runner:docker" ] + stage: tests + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: on_success + variables: + BUILD_CACHE_TYPE: lib + before_script: + - cd /system-tests + - git fetch origin $SYSTEM_TESTS_COMMIT + - git reset --hard FETCH_HEAD + - mkdir -p /system-tests/binaries/ + - cp $CI_PROJECT_DIR/workspace/dd-java-agent/build/libs/*.jar /system-tests/binaries/ + - cp $CI_PROJECT_DIR/workspace/dd-trace-api/build/libs/*.jar /system-tests/binaries/ + - cd /system-tests + - export SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID="" #TODO fix variable fetch + - export SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY="" + - export SYSTEM_TESTS_E2E_DD_API_KEY="" + - export SYSTEM_TESTS_E2E_DD_APP_KEY="" + - export SYSTEM_TESTS_DD_API_KEY="" + +system-tests: + extends: .system-tests-job + parallel: + matrix: + - WEBLOG_VARIANT: ['akka-http', 'jersey-grizzly2', 'play', 'resteasy-netty3', 'ratpack', 'spring-boot', 'spring-boot-jetty', 'spring-boot-openliberty', 'spring-boot-payara', 'spring-boot-undertow', 'spring-boot-wildfly', 'uds-spring-boot', 'vertx3', 'vertx4'] + SCENARIO: [ DEFAULT, APM_TRACING_E2E, APM_TRACING_E2E_SINGLE_SPAN, APPSEC_BLOCKING, APPSEC_REQUEST_BLOCKING, APPSEC_RASP, APPSEC_RUNTIME_ACTIVATION, REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD ] + - WEBLOG_VARIANT: ['spring-boot-3-native'] + SCENARIO: [ DEFAULT, APM_TRACING_E2E, APM_TRACING_E2E_SINGLE_SPAN ] + script: + - ./build.sh java --weblog-variant $WEBLOG_VARIANT + - > + if [[ $SCENARIO =~ .*_E2E.* ]]; then + export DD_SITE=datadoghq.com; + export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY; + export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY; + else + export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY; + fi + - echo "Running scenario $SCENARIO" + - ./run.sh $SCENARIO + after_script: + - | + mkdir -p artifacts + cd /system-tests + shopt -s nullglob + for log_dir in logs*; do + tar -cvzf ../artifacts/${log_dir}_${WEBLOG_VARIANT}.tar.gz $log_dir + done + artifacts: + when: always + paths: + - artifacts + - '.gradle/daemon/*/*.out.log' + +integration-system-tests: + extends: .system-tests-job + script: + - ./build.sh --library java --weblog-variant spring-boot + - export AWS_ACCESS_KEY_ID="$SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID" + - export AWS_SECRET_ACCESS_KEY="$SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY" + - export AWS_REGION="us-east-1" + - export AWS_DEFAULT_REGION="us-east-1" # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION` + - DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh INTEGRATIONS + - DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh CROSSED_TRACING_LIBRARIES + artifacts: + when: always + reports: + junit: /system-tests/logs_integrations/*.xml + paths: + - /system-tests/logs_integrations + - '.gradle/daemon/*/*.out.log' + +debugger-system-tests: + extends: .system-tests-job + script: + - ./build.sh --library java --weblog-variant spring-boot + - DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY ./run.sh DEBUGGER_SCENARIOS + after_script: + - | + mkdir -p logs_debugger + for dir in /system-tests/logs*/; do + cp -r "$dir" logs_debugger + done + artifacts: + when: always + reports: + junit: logs_debugger/*.xml + paths: + - logs_debugger + - '.gradle/daemon/*/*.out.log' + +parametric-tests: + extends: .system-tests-job + script: + - ./build.sh -i runner + - timeout 20m ./run.sh PARAMETRIC --library java --durations=30 -vv + after_script: + - tar -cvzf logs_java_parametric_dev.tar.gz -C /system-tests logs_parametric + artifacts: + when: always + reports: + junit: /system-tests/logs_parametric/*.xml + paths: + - logs_java_parametric_dev.tar.gz + - '.gradle/daemon/*/*.out.log' + deploy_to_profiling_backend: stage: publish needs: [ build ] @@ -165,6 +570,8 @@ deploy_to_sonatype: extends: .gradle_build stage: publish needs: [ build ] + variables: + BUILD_CACHE_TYPE: lib rules: - if: '$POPULATE_CACHE' when: never @@ -180,7 +587,7 @@ deploy_to_sonatype: - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text) - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository $GRADLE_ARGS + - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS artifacts: paths: - 'workspace/dd-java-agent/build/libs/*.jar' From ac94d55d1e4bcc98b7c0d19fc97db3ca47937d79 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 13 Feb 2025 12:11:14 -0500 Subject: [PATCH 2/8] reduce to jobs we care about --- .gitlab-ci.yml | 317 ------------------------------------------------- 1 file changed, 317 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7b5d8c521aa..06110eddb2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,14 +17,6 @@ stages: - ci-visibility-tests - generate-signing-key -#TODO display memory usage -#TODO test containers -#TODO paralization (muzzle) -#TODO publish tests to datadog -#TODO agent_integration_tests -#TODO no output timeout -#TODO xlarge_tests - variables: REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com BUILD_JOB_NAME: "build" @@ -32,27 +24,6 @@ variables: BUILD_CACHE_POLICY: pull GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties JAVA_BUILD_IMAGE_VERSION: "v25.01" - PROFILE_TESTS: - description: "Enable profiling of tests" - value: "false" - SYSTEM_TESTS_COMMIT: - description: "Commit hash of system tests to use" - value: "8384a3947b9881d0fcdb5ac937b80aad9a70d4d4" - -.test_matrix: &test_matrix - - testJvm: - - "8" - - "semeru11" - - "oracle8" - - "21" - - "ubuntu17" - - "zulu8" - - "semeru8" - - "ibm8" - - "zulu11" - - "11" - - "17" - - "semeru17" default: tags: [ "arch:amd64" ] @@ -115,16 +86,6 @@ build: reports: dotenv: build.env -build_tests: - extends: .gradle_build - variables: - BUILD_CACHE_POLICY: push - BUILD_CACHE_TYPE: test - DEPENDENCY_CACHE_POLICY: pull - script: - - ./gradlew resolveAndLockAll --write-locks - - ./gradlew clean :baseTest :instrumentationTest :instrumentationLatestDepTest :instrumentationTest :profilingTest -PskipTests $GRADLE_ARGS - build_and_populate_dep_cache: extends: build variables: @@ -144,51 +105,6 @@ spotless: - export JAVA_HOME=$JAVA_11_HOME - ./gradlew spotlessCheck $GRADLE_ARGS -.check_job: - extends: .gradle_build - needs: [ "build_tests" ] - stage: tests - rules: - - if: '$POPULATE_CACHE' - when: never - - when: always # TODO change to on_success once build_tests is stable - variables: - BUILD_CACHE_TYPE: test - script: - - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless $GRADLE_ARGS - after_script: - - .circleci/collect_reports.sh --destination ./check_reports --move - artifacts: - when: always - paths: - - ./check_reports - - '.gradle/daemon/*/*.out.log' - -check_base: - extends: .check_job - variables: - GRADLE_TARGET: ":baseCheck" - -check_inst: - extends: .check_job - variables: - GRADLE_TARGET: ":instrumentationCheck" - -check_smoke: - extends: .check_job - variables: - GRADLE_TARGET: ":smokeCheck" - -check_profiling: - extends: .check_job - variables: - GRADLE_TARGET: ":profilingCheck" - -check_debugger: - extends: .check_job - variables: - GRADLE_TARGET: ":debuggerCheck" - test_published_artifacts: extends: .gradle_build image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests @@ -262,239 +178,6 @@ muzzle-dep-report: - ./reports - '.gradle/daemon/*/*.out.log' -.test_job: - extends: .gradle_build - image: ghcr.io/datadog/dd-trace-java-docker-build:$testJvm - needs: [ build_tests ] - stage: tests - rules: - - if: '$POPULATE_CACHE' - when: never - - when: always # TODO change to on_success once build_tests is stable - variables: - BUILD_CACHE_TYPE: test - GRADLE_PARAMS: "" - CONTINUE_ON_FAILURE: "false" - script: - - > - if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; - then - export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true"; - fi - - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp' -Ddatadog.forkedMaxHeapSize=768M -Ddatadog.forkedMinHeapSize=128M" - - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE - after_script: - - .circleci/collect_reports.sh - - if [ "$PROFILE_TESTS" == "true" ]; then .circleci/collect_profiles.sh; fi - - .circleci/collect_results.sh - - .circleci/upload_ciapp.sh tests $testJvm -# TODO Get APM Test Agent Trace Check Results - artifacts: - when: always - paths: - - ./reports.tar - - ./profiles.tar - - ./results - - '.gradle/daemon/*/*.out.log' - -test_base: - extends: .test_job - variables: - GRADLE_TARGET: ":baseTest" - GRADLE_PARAMS: "-PskipFlakyTests" - parallel: - matrix: *test_matrix - script: - - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi - - !reference [.test_job, script] - -test_inst: - extends: .test_job - variables: - GRADLE_TARGET: ":instrumentationTest" - GRADLE_PARAMS: "-PskipFlakyTests" - parallel: - matrix: *test_matrix - -test_inst_latest: - extends: .test_job - variables: - GRADLE_TARGET: ":instrumentationLatestDepTest" - GRADLE_PARAMS: "-PskipFlakyTests" - parallel: - matrix: - - testJvm: ["8", "17", "21" ] - -test_flaky: - extends: .test_job - variables: - GRADLE_PARAMS: "-PrunFlakyTests" - testJvm: "8" - CONTINUE_ON_FAILURE: "true" - parallel: - matrix: - - GRADLE_TARGET: [":baseTest", ":instrumentationTest", ":smokeTest", ":debuggerTest"] - -test_profiling: - extends: .test_job - variables: - GRADLE_TARGET: ":profilingTest" - GRADLE_PARAMS: "-PskipFlakyTests" - parallel: - matrix: *test_matrix - -test_debugger: - extends: .test_job - variables: - GRADLE_TARGET: ":debuggerTest" - GRADLE_PARAMS: "-PskipFlakyTests" - parallel: - matrix: *test_matrix - -test_smoke: - extends: .test_job - variables: - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - parallel: - matrix: *test_matrix - -test_ssi_smoke: - extends: .test_job - variables: - GRADLE_TARGET: "stageMainDist :smokeTest" - GRADLE_PARAMS: "-PskipFlakyTests" - DD_INJECT_FORCE: "true" - DD_INJECTION_ENABLED: "tracer" - parallel: - matrix: *test_matrix - -test_smoke_graalvm: - extends: .test_job - variables: - GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test" - parallel: - matrix: - - testJvm: ["graalvm17", "graalvm21"] - -test_smoke_semeru8_debugger: - extends: .test_job - variables: - GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test" - GRADLE_PARAMS: "-PskipFlakyTests" - testJvm: "semeru8" - -.system-tests-job: - image: $SYSTEM_TESTS_IMAGE - tags: [ "runner:docker" ] - stage: tests - needs: [ build ] - rules: - - if: '$POPULATE_CACHE' - when: never - - when: on_success - variables: - BUILD_CACHE_TYPE: lib - before_script: - - cd /system-tests - - git fetch origin $SYSTEM_TESTS_COMMIT - - git reset --hard FETCH_HEAD - - mkdir -p /system-tests/binaries/ - - cp $CI_PROJECT_DIR/workspace/dd-java-agent/build/libs/*.jar /system-tests/binaries/ - - cp $CI_PROJECT_DIR/workspace/dd-trace-api/build/libs/*.jar /system-tests/binaries/ - - cd /system-tests - - export SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID="" #TODO fix variable fetch - - export SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY="" - - export SYSTEM_TESTS_E2E_DD_API_KEY="" - - export SYSTEM_TESTS_E2E_DD_APP_KEY="" - - export SYSTEM_TESTS_DD_API_KEY="" - -system-tests: - extends: .system-tests-job - parallel: - matrix: - - WEBLOG_VARIANT: ['akka-http', 'jersey-grizzly2', 'play', 'resteasy-netty3', 'ratpack', 'spring-boot', 'spring-boot-jetty', 'spring-boot-openliberty', 'spring-boot-payara', 'spring-boot-undertow', 'spring-boot-wildfly', 'uds-spring-boot', 'vertx3', 'vertx4'] - SCENARIO: [ DEFAULT, APM_TRACING_E2E, APM_TRACING_E2E_SINGLE_SPAN, APPSEC_BLOCKING, APPSEC_REQUEST_BLOCKING, APPSEC_RASP, APPSEC_RUNTIME_ACTIVATION, REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD ] - - WEBLOG_VARIANT: ['spring-boot-3-native'] - SCENARIO: [ DEFAULT, APM_TRACING_E2E, APM_TRACING_E2E_SINGLE_SPAN ] - script: - - ./build.sh java --weblog-variant $WEBLOG_VARIANT - - > - if [[ $SCENARIO =~ .*_E2E.* ]]; then - export DD_SITE=datadoghq.com; - export DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY; - export DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY; - else - export DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY; - fi - - echo "Running scenario $SCENARIO" - - ./run.sh $SCENARIO - after_script: - - | - mkdir -p artifacts - cd /system-tests - shopt -s nullglob - for log_dir in logs*; do - tar -cvzf ../artifacts/${log_dir}_${WEBLOG_VARIANT}.tar.gz $log_dir - done - artifacts: - when: always - paths: - - artifacts - - '.gradle/daemon/*/*.out.log' - -integration-system-tests: - extends: .system-tests-job - script: - - ./build.sh --library java --weblog-variant spring-boot - - export AWS_ACCESS_KEY_ID="$SYSTEM_TESTS_IDM_AWS_ACCESS_KEY_ID" - - export AWS_SECRET_ACCESS_KEY="$SYSTEM_TESTS_IDM_AWS_SECRET_ACCESS_KEY" - - export AWS_REGION="us-east-1" - - export AWS_DEFAULT_REGION="us-east-1" # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION` - - DD_SITE=datadoghq.com DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY DD_APPLICATION_KEY=$SYSTEM_TESTS_E2E_DD_APP_KEY ./run.sh INTEGRATIONS - - DD_API_KEY=$SYSTEM_TESTS_DD_API_KEY ./run.sh CROSSED_TRACING_LIBRARIES - artifacts: - when: always - reports: - junit: /system-tests/logs_integrations/*.xml - paths: - - /system-tests/logs_integrations - - '.gradle/daemon/*/*.out.log' - -debugger-system-tests: - extends: .system-tests-job - script: - - ./build.sh --library java --weblog-variant spring-boot - - DD_API_KEY=$SYSTEM_TESTS_E2E_DD_API_KEY ./run.sh DEBUGGER_SCENARIOS - after_script: - - | - mkdir -p logs_debugger - for dir in /system-tests/logs*/; do - cp -r "$dir" logs_debugger - done - artifacts: - when: always - reports: - junit: logs_debugger/*.xml - paths: - - logs_debugger - - '.gradle/daemon/*/*.out.log' - -parametric-tests: - extends: .system-tests-job - script: - - ./build.sh -i runner - - timeout 20m ./run.sh PARAMETRIC --library java --durations=30 -vv - after_script: - - tar -cvzf logs_java_parametric_dev.tar.gz -C /system-tests logs_parametric - artifacts: - when: always - reports: - junit: /system-tests/logs_parametric/*.xml - paths: - - logs_java_parametric_dev.tar.gz - - '.gradle/daemon/*/*.out.log' - deploy_to_profiling_backend: stage: publish needs: [ build ] From 6464d3ce3b041c0a71f98685a6a860dadf0d4a22 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 13 Feb 2025 12:14:01 -0500 Subject: [PATCH 3/8] remove from circleci config --- .circleci/config.continue.yml.j2 | 133 ------------------------------- 1 file changed, 133 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index c8e7e65f205..23b534ce8da 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -705,117 +705,6 @@ jobs: - DD_BIND_HOST=0.0.0.0 - DD_API_KEY=invalid_key_but_this_is_fine - test_published_artifacts: - <<: *defaults - resource_class: medium - docker: - - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}7 - - steps: - - setup_code - - restore_dependency_cache: - cacheType: lib - - restore_build_cache: - cacheType: lib - - - run: - name: Publish Artifacts Locally - command: | - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) - rm -rf "${mvn_local_repo}/com/datadoghq" - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew publishToMavenLocal << pipeline.parameters.gradle_flags >> --max-workers=3 - - - run: - name: Test Published Artifacts - command: | - cd test-published-dependencies - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew check --info --max-workers=3 - - - run: - name: Collect Reports - when: on_fail - command: .circleci/collect_reports.sh - - - store_artifacts: - path: ./reports - - - display_memory_usage - muzzle-dep-report: - <<: *defaults - resource_class: medium - steps: - - setup_code - - skip_unless_matching_files_changed: - pattern: "dd-java-agent/instrumentation" - - restore_dependency_cache: - cacheType: inst - - restore_build_cache: - cacheType: inst - - run: - name: Generate muzzle dep report - command: >- - SKIP_BUILDSCAN="true" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew generateMuzzleReport muzzleInstrumentationReport - - run: - name: Collect Reports - command: .circleci/collect_muzzle_deps.sh - - store_artifacts: - path: ./reports - - muzzle: - <<: *defaults - resource_class: medium+ - parallelism: 4 - steps: - - setup_code - - - skip_unless_matching_files_changed: - pattern: "dd-java-agent/instrumentation" - - # We are not running with a separate cache of all muzzle artifacts here because it gets very big and - # ends up taking more time restoring/saving than the actual increase in time it takes just - # downloading the artifacts each time. - # - # Let's at least restore the build cache to have something to start from. - - restore_dependency_cache: - cacheType: inst - - restore_build_cache: - cacheType: inst - - - run: - name: Gather muzzle tasks - command: >- - SKIP_BUILDSCAN="true" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew writeMuzzleTasksToFile - << pipeline.parameters.gradle_flags >> - --max-workers=3 - - - run: - name: Verify Muzzle - command: >- - SKIP_BUILDSCAN="true" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx3G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew `circleci tests split --split-by=timings workspace/build/muzzleTasks | xargs` - << pipeline.parameters.gradle_flags >> - --max-workers=4 - - - run: - name: Collect Reports - when: on_fail - command: .circleci/collect_reports.sh - - - store_artifacts: - path: ./reports - - - store_test_results: - path: workspace/build/muzzle-test-results - - - display_memory_usage - system-tests: machine: image: ubuntu-2404:current @@ -1424,24 +1313,6 @@ build_test_jobs: &build_test_jobs cacheType: base testJvm: "8" - - test_published_artifacts: - requires: - - ok_to_test - - - muzzle: - requires: - - ok_to_test - filters: - branches: - ignore: - - master - - project/* - - release/* - - - muzzle-dep-report: - requires: - - ok_to_test - - system-tests: requires: - ok_to_test @@ -1462,7 +1333,6 @@ build_test_jobs: &build_test_jobs - fan_in: requires: - - test_published_artifacts {% for jdk in all_jdks %} - "test_{{ jdk }}_profiling" {% endfor %} @@ -1471,7 +1341,6 @@ build_test_jobs: &build_test_jobs - fan_in: requires: - - test_published_artifacts {% for jdk in all_jdks %} - "test_{{ jdk }}_debugger" {% endfor %} @@ -1483,13 +1352,11 @@ build_test_jobs: &build_test_jobs - fan_in: requires: - check - - test_published_artifacts - agent_integration_tests {% for jdk in all_jdks %} - "test_{{ jdk }}" {% endfor %} - test_inst_latest - - muzzle - profiling - debugger - system-tests From 94a14a40e91bce5a65035b92638404071668410e Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 13 Feb 2025 12:16:51 -0500 Subject: [PATCH 4/8] remove spotless as well --- .circleci/config.continue.yml.j2 | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 23b534ce8da..c9f5e4de18b 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -365,21 +365,6 @@ jobs: - display_memory_usage - spotless: - <<: *defaults - resource_class: medium+ - - steps: - - setup_code - - - run: - name: Run spotless - command: >- - JAVA_HOME=$JAVA_11_HOME - ./gradlew spotlessCheck - << pipeline.parameters.gradle_flags >> - --max-workers=8 - check: <<: *defaults @@ -955,7 +940,6 @@ build_test_jobs: &build_test_jobs name: build_profiling gradleTarget: :profilingTest cacheType: profiling - - spotless - fan_in: requires: @@ -964,7 +948,6 @@ build_test_jobs: &build_test_jobs - build_inst - build_smoke - build_profiling - - spotless name: ok_to_test stage: ok_to_test From b029be8a445795f88e30d91d08151cad13236f0e Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 13 Feb 2025 13:11:13 -0500 Subject: [PATCH 5/8] add "required" job --- .gitlab-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 06110eddb2a..7c8ab5888df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,16 @@ variables: default: tags: [ "arch:amd64" ] +.fan_in: + stage: tests + image: registry.ddbuild.io/images/base/gbi-distroless + variables: + GRADLE_WORKERS: 2 + KUBERNETES_CPU_REQUEST: 1 + KUBERNETES_MEMORY_REQUEST: 1Gi + script: + - echo "done" + .gradle_build: &gradle_build image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base stage: build @@ -178,6 +188,13 @@ muzzle-dep-report: - ./reports - '.gradle/daemon/*/*.out.log' +required: + extends: .fan_in + needs: + - spotless + - muzzle + - test_published_artifacts + deploy_to_profiling_backend: stage: publish needs: [ build ] From e198d499b20654c682862412379ff7e9bbde62dc Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 13 Feb 2025 15:41:32 -0500 Subject: [PATCH 6/8] fix image tag for fan-in --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c8ab5888df..75f5ad5769b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,7 @@ default: .fan_in: stage: tests - image: registry.ddbuild.io/images/base/gbi-distroless + image: registry.ddbuild.io/images/base/gbi-distroless:release variables: GRADLE_WORKERS: 2 KUBERNETES_CPU_REQUEST: 1 From e223428a0821ef82396a8f92e5c7562bc56f52ac Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Thu, 13 Feb 2025 16:11:26 -0500 Subject: [PATCH 7/8] simplify --- .gitlab-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75f5ad5769b..8482af87798 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,11 +30,7 @@ default: .fan_in: stage: tests - image: registry.ddbuild.io/images/base/gbi-distroless:release - variables: - GRADLE_WORKERS: 2 - KUBERNETES_CPU_REQUEST: 1 - KUBERNETES_MEMORY_REQUEST: 1Gi + image: registry.ddbuild.io/images/base/gbi-ubuntu_2204-slim:release script: - echo "done" From 49ae36c0823806a914957302997e6fc2e67af418 Mon Sep 17 00:00:00 2001 From: Laplie Anderson Date: Tue, 18 Feb 2025 11:01:36 -0500 Subject: [PATCH 8/8] only relock during scheduled nightly --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8482af87798..244a9dd4ba0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,7 +76,7 @@ build: BUILD_CACHE_TYPE: lib DEPENDENCY_CACHE_POLICY: pull script: - - ./gradlew resolveAndLockAll --write-locks + - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks; fi - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env