From 436220eee957483c9865a4ad9703fa26d0d39abc Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Fri, 2 Feb 2024 14:57:23 +0100 Subject: [PATCH] Run CI only for changed modules in pull requests --- .circleci/collect_results.sh | 13 +++++++++++-- .circleci/config.continue.yml.j2 | 17 +++++++++++++++++ .circleci/render_config.py | 19 ++++++++++++++----- build.gradle | 27 +++++++++++++++++++++++++-- 4 files changed, 67 insertions(+), 9 deletions(-) diff --git a/.circleci/collect_results.sh b/.circleci/collect_results.sh index 4c580e9b08b..44bc59f7f54 100755 --- a/.circleci/collect_results.sh +++ b/.circleci/collect_results.sh @@ -10,8 +10,17 @@ shopt -s globstar TEST_RESULTS_DIR=./results mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1 -echo "saving test results" mkdir -p $TEST_RESULTS_DIR -find workspace/**/build/test-results -name \*.xml -exec sh -c ' + +mkdir -p workspace +mapfile -t test_result_dirs < <(find workspace -name test-results -type d) + +if [[ ${#test_result_dirs[@]} -eq 0 ]]; then + echo "No test results found" + exit 0 +fi + +echo "saving test results" +find "${test_result_dirs[@]}" -name \*.xml -exec sh -c ' file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_") cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \; diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 189564bf262..43df62d9a5a 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -99,6 +99,11 @@ commands: setup_code: steps: - checkout +{% if use_git_changes %} + - run: + name: Fetch base branch + command: git fetch origin {{ pr_base_ref }} +{% endif %} - run: name: Checkout merge commit command: .circleci/checkout_merge_commit.sh @@ -301,6 +306,10 @@ jobs: GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" ./gradlew clean << parameters.gradleTarget >> +{% if use_git_changes %} + -Paffected_module_detector.enable + -PbaseBranch=origin/{{ pr_base_ref }} +{% endif %} -PskipTests << pipeline.parameters.gradle_flags >> --max-workers=8 @@ -393,6 +402,10 @@ jobs: GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" ./gradlew << parameters.gradleTarget >> +{% if use_git_changes %} + -Paffected_module_detector.enable + -PbaseBranch=origin/{{ pr_base_ref }} +{% endif %} -PskipTests -PrunBuildSrcTests -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX} @@ -539,6 +552,10 @@ jobs: ./gradlew << parameters.gradleTarget >> << parameters.gradleParameters >> +{% if use_git_changes %} + -Paffected_module_detector.enable + -PbaseBranch=origin/{{ pr_base_ref }} +{% endif %} -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX} <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><> << pipeline.parameters.gradle_flags >> diff --git a/.circleci/render_config.py b/.circleci/render_config.py index 10c72c67830..8e255129a71 100755 --- a/.circleci/render_config.py +++ b/.circleci/render_config.py @@ -28,7 +28,7 @@ } # Version to use for all the base Docker images, see # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build -DOCKER_IMAGE_VERSION="v24.01" +DOCKER_IMAGE_VERSION = "v24.01" # Get labels from pull requests to override some defaults for jobs to run. # `run-tests: all` will run all tests. @@ -52,7 +52,7 @@ ) resp.raise_for_status() except Exception as e: - print(f"Request filed: {e}") + print(f"Request failed: {e}") time.sleep(1) continue data = resp.json() @@ -63,12 +63,16 @@ labels = { l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ") } + # get the base reference (e.g. `master`), commit hash is also available at the `sha` field. + pr_base_ref = data.get("base", {}).get("ref") else: labels = set() - branch = os.environ.get("CIRCLE_BRANCH", "") -if branch == "master" or branch.startswith("release/v") or "all" in labels: +run_all = "all" in labels +is_master_or_release = branch == "master" or branch.startswith("release/v") + +if is_master_or_release or run_all: all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS else: all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels) @@ -83,6 +87,9 @@ is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true" is_regular = not is_nightly and not is_weekly +# Use git changes detection on PRs +use_git_changes = not run_all and not is_master_or_release and is_regular + vars = { "is_nightly": is_nightly, "is_weekly": is_weekly, @@ -92,12 +99,14 @@ "nocov_jdks": nocov_jdks, "flaky": branch == "master" or "flaky" in labels or "all" in labels, "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-", + "use_git_changes": use_git_changes, + "pr_base_ref": pr_base_ref, } print(f"Variables for this build: {vars}") loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR) -env = jinja2.Environment(loader=loader) +env = jinja2.Environment(loader=loader, trim_blocks=True) tpl = env.get_template(TPL_FILENAME) out = tpl.render(**vars) diff --git a/build.gradle b/build.gradle index 251b595146c..e2789b446f8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,7 @@ buildscript { dependencies { classpath "pl.allegro.tech.build:axion-release-plugin:1.14.4" + classpath "com.dropbox.affectedmoduledetector:affectedmoduledetector:0.2.2" } configurations.all { @@ -24,6 +25,8 @@ plugins { id "me.champeau.jmh" version "0.7.0" apply false id 'org.gradle.playframework' version '0.13' apply false id 'info.solidsoft.pitest' version '1.9.11' apply false + + id 'com.dropbox.affectedmoduledetector' version '0.2.2' } description = 'dd-trace-java' @@ -160,15 +163,35 @@ allprojects { project -> } } +affectedModuleDetector { + baseDir = "${project.rootDir}" + pathsAffectingAllModules = [".circleci/", "buildSrc/"] + specifiedBranch = project.hasProperty("baseBranch") ? project.property("baseBranch") : "origin/master" + compareFrom = "SpecifiedBranchCommit" + excludedModules = [] + includeUncommitted = true +} + +subprojects { subproject -> + subproject.tasks.withType(Test) { testTask -> + com.dropbox.affectedmoduledetector.AffectedModuleDetector.configureTaskGuard(testTask) + } +} def testAggregate(String baseTaskName, includePrefixes, excludePrefixes) { + def createRootTask = { rootTaskName, subProjTaskName -> - tasks.register(rootTaskName) { aggTest -> + tasks.register(rootTaskName) { aggTask -> subprojects { subproject -> if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { + def affected = com.dropbox.affectedmoduledetector.AffectedModuleDetector.isProjectAffected(subproject) def testTask = subproject.tasks.findByName(subProjTaskName) if (testTask != null) { - aggTest.dependsOn(testTask) + if (affected) { + aggTask.dependsOn(testTask) + } else { + logger.info("Skipped based on git changes: ${subproject.path}") + } } } }