diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index d1c7d297a75..98edf10d733 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -365,21 +365,6 @@ jobs: - display_memory_usage - spotless: - <<: *defaults - resource_class: medium+ - - steps: - - setup_code - - - run: - name: Run spotless - command: >- - JAVA_HOME=$JAVA_11_HOME - ./gradlew spotlessCheck - << pipeline.parameters.gradle_flags >> - --max-workers=8 - check: <<: *defaults @@ -705,117 +690,6 @@ jobs: - DD_BIND_HOST=0.0.0.0 - DD_API_KEY=invalid_key_but_this_is_fine - test_published_artifacts: - <<: *defaults - resource_class: medium - docker: - - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}7 - - steps: - - setup_code - - restore_dependency_cache: - cacheType: lib - - restore_build_cache: - cacheType: lib - - - run: - name: Publish Artifacts Locally - command: | - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) - rm -rf "${mvn_local_repo}/com/datadoghq" - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew publishToMavenLocal << pipeline.parameters.gradle_flags >> --max-workers=3 - - - run: - name: Test Published Artifacts - command: | - cd test-published-dependencies - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew check --info --max-workers=3 - - - run: - name: Collect Reports - when: on_fail - command: .circleci/collect_reports.sh - - - store_artifacts: - path: ./reports - - - display_memory_usage - muzzle-dep-report: - <<: *defaults - resource_class: medium - steps: - - setup_code - - skip_unless_matching_files_changed: - pattern: "dd-java-agent/instrumentation" - - restore_dependency_cache: - cacheType: inst - - restore_build_cache: - cacheType: inst - - run: - name: Generate muzzle dep report - command: >- - SKIP_BUILDSCAN="true" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew generateMuzzleReport muzzleInstrumentationReport - - run: - name: Collect Reports - command: .circleci/collect_muzzle_deps.sh - - store_artifacts: - path: ./reports - - muzzle: - <<: *defaults - resource_class: medium+ - parallelism: 4 - steps: - - setup_code - - - skip_unless_matching_files_changed: - pattern: "dd-java-agent/instrumentation" - - # We are not running with a separate cache of all muzzle artifacts here because it gets very big and - # ends up taking more time restoring/saving than the actual increase in time it takes just - # downloading the artifacts each time. - # - # Let's at least restore the build cache to have something to start from. - - restore_dependency_cache: - cacheType: inst - - restore_build_cache: - cacheType: inst - - - run: - name: Gather muzzle tasks - command: >- - SKIP_BUILDSCAN="true" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew writeMuzzleTasksToFile - << pipeline.parameters.gradle_flags >> - --max-workers=3 - - - run: - name: Verify Muzzle - command: >- - SKIP_BUILDSCAN="true" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx3G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" - ./gradlew `circleci tests split --split-by=timings workspace/build/muzzleTasks | xargs` - << pipeline.parameters.gradle_flags >> - --max-workers=4 - - - run: - name: Collect Reports - when: on_fail - command: .circleci/collect_reports.sh - - - store_artifacts: - path: ./reports - - - store_test_results: - path: workspace/build/muzzle-test-results - - - display_memory_usage - system-tests: machine: image: ubuntu-2404:current @@ -1069,7 +943,6 @@ build_test_jobs: &build_test_jobs name: build_profiling gradleTarget: :profilingTest cacheType: profiling - - spotless - fan_in: requires: @@ -1078,7 +951,6 @@ build_test_jobs: &build_test_jobs - build_inst - build_smoke - build_profiling - - spotless name: ok_to_test stage: ok_to_test @@ -1427,24 +1299,6 @@ build_test_jobs: &build_test_jobs cacheType: base testJvm: "8" - - test_published_artifacts: - requires: - - ok_to_test - - - muzzle: - requires: - - ok_to_test - filters: - branches: - ignore: - - master - - project/* - - release/* - - - muzzle-dep-report: - requires: - - ok_to_test - - system-tests: requires: - ok_to_test @@ -1465,7 +1319,6 @@ build_test_jobs: &build_test_jobs - fan_in: requires: - - test_published_artifacts {% for jdk in all_jdks %} - "test_{{ jdk }}_profiling" {% endfor %} @@ -1474,7 +1327,6 @@ build_test_jobs: &build_test_jobs - fan_in: requires: - - test_published_artifacts {% for jdk in all_jdks %} - "test_{{ jdk }}_debugger" {% endfor %} @@ -1486,13 +1338,11 @@ build_test_jobs: &build_test_jobs - fan_in: requires: - check - - test_published_artifacts - agent_integration_tests {% for jdk in all_jdks %} - "test_{{ jdk }}" {% endfor %} - test_inst_latest - - muzzle - profiling - debugger - system-tests diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 334d20d22d0..244a9dd4ba0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,15 +23,24 @@ variables: DEPENDENCY_CACHE_POLICY: pull BUILD_CACHE_POLICY: pull GRADLE_VERSION: "8.4" # must match gradle-wrapper.properties + JAVA_BUILD_IMAGE_VERSION: "v25.01" default: tags: [ "arch:amd64" ] +.fan_in: + stage: tests + image: registry.ddbuild.io/images/base/gbi-ubuntu_2204-slim:release + script: + - echo "done" + .gradle_build: &gradle_build - image: ghcr.io/datadog/dd-trace-java-docker-build:v25.01-base + image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-base + stage: build variables: - GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M'" - GRADLE_ARGS: " -PskipTests --build-cache --stacktrace --no-daemon --parallel --max-workers=2" + GRADLE_OPTS: "-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + MAVEN_OPTS: "-Xms64M -Xmx512M" + GRADLE_WORKERS: 2 KUBERNETES_CPU_REQUEST: 8 KUBERNETES_MEMORY_REQUEST: 6Gi cache: @@ -42,7 +51,7 @@ default: - .gradle/caches - .gradle/notifications policy: $DEPENDENCY_CACHE_POLICY - - key: $CI_PIPELINE_ID # Incremental build cache. Shared by all jobs in the pipeline + - key: $CI_PIPELINE_ID-$BUILD_CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type paths: - .gradle/caches/$GRADLE_VERSION - .gradle/$GRADLE_VERSION/executionHistory @@ -50,6 +59,7 @@ default: policy: $BUILD_CACHE_POLICY before_script: - export GRADLE_USER_HOME=`pwd`/.gradle + - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS" # for weird reasons, gradle will always "chmod 700" the .gradle folder # with Gitlab caching, .gradle is always owned by root and thus gradle's chmod invocation fails # This dance is a hack to have .gradle owned by the Gitlab runner user @@ -61,21 +71,24 @@ default: build: extends: .gradle_build - stage: build variables: BUILD_CACHE_POLICY: push + BUILD_CACHE_TYPE: lib DEPENDENCY_CACHE_POLICY: pull script: - - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar $GRADLE_ARGS + - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks; fi + - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env artifacts: + when: always paths: - 'workspace/dd-java-agent/build/libs/*.jar' - 'workspace/dd-trace-api/build/libs/*.jar' - 'workspace/dd-trace-ot/build/libs/*.jar' - 'upstream.env' + - '.gradle/daemon/*/*.out.log' reports: dotenv: build.env @@ -90,6 +103,94 @@ build_and_populate_dep_cache: - when: manual allow_failure: true +spotless: + extends: .gradle_build + stage: tests + needs: [] + script: + - export JAVA_HOME=$JAVA_11_HOME + - ./gradlew spotlessCheck $GRADLE_ARGS + +test_published_artifacts: + extends: .gradle_build + image: ghcr.io/datadog/dd-trace-java-docker-build:${JAVA_BUILD_IMAGE_VERSION}-7 # Needs Java7 for some tests + stage: tests + needs: [ build ] + rules: + - if: '$POPULATE_CACHE' + when: never + - when: on_success + variables: + BUILD_CACHE_TYPE: lib + script: + - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout) + - rm -rf "${mvn_local_repo}/com/datadoghq" + - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) + - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew publishToMavenLocal $GRADLE_ARGS + - cd test-published-dependencies + - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx512M -Xms512M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + - ./gradlew check --info $GRADLE_ARGS + after_script: + - .circleci/collect_reports.sh + artifacts: + when: always + paths: + - ./check_reports + +muzzle: + extends: .gradle_build + needs: [ build ] + stage: tests + parallel: 8 + rules: + - if: '$POPULATE_CACHE' + when: never + - when: on_success + variables: + BUILD_CACHE_TYPE: test + script: + - export SKIP_BUILDSCAN="true" + - echo CI_NODE_INDEX=$CI_NODE_INDEX + - echo CI_NODE_TOTAL=CI_NODE_TOTAL + - ./gradlew writeMuzzleTasksToFile $GRADLE_ARGS + - sort workspace/build/muzzleTasks > sortedMuzzleTasks + - split --number=l/$CI_NODE_TOTAL --suffix-length=1 --numeric-suffixes sortedMuzzleTasks muzzleSplit + - export NODE_ZERO_INDEX=$((CI_NODE_INDEX - 1)) + - ./gradlew `cat muzzleSplit${NODE_ZERO_INDEX} | xargs` $GRADLE_ARGS + after_script: + - .circleci/collect_reports.sh + artifacts: + when: always + paths: + - ./reports + - '.gradle/daemon/*/*.out.log' + +muzzle-dep-report: + extends: .gradle_build + needs: [ build ] + stage: tests + variables: + BUILD_CACHE_TYPE: test + script: + - export SKIP_BUILDSCAN="true" + - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS + after_script: + - .circleci/collect_muzzle_deps.sh + artifacts: + when: always + paths: + - ./reports + - '.gradle/daemon/*/*.out.log' + +required: + extends: .fan_in + needs: + - spotless + - muzzle + - test_published_artifacts + deploy_to_profiling_backend: stage: publish needs: [ build ] @@ -165,6 +266,8 @@ deploy_to_sonatype: extends: .gradle_build stage: publish needs: [ build ] + variables: + BUILD_CACHE_TYPE: lib rules: - if: '$POPULATE_CACHE' when: never @@ -180,7 +283,7 @@ deploy_to_sonatype: - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.sonatype_password --with-decryption --query "Parameter.Value" --out text) - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) - - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository $GRADLE_ARGS + - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS artifacts: paths: - 'workspace/dd-java-agent/build/libs/*.jar'