diff --git a/.travis.yml b/.travis.yml index 709c408a47cf..d92b66c33367 100644 --- a/.travis.yml +++ b/.travis.yml @@ -382,29 +382,8 @@ jobs: after_success: - (cd web-console && travis_retry npm run codecov) # retry in case of network error - - name: "Build and test on ARM64 CPU architecture (1)" - stage: Tests - phase 2 - arch: arm64-graviton2 - dist: focal - virt: vm - group: edge - jdk: openjdk11 - env: - - MAVEN_PROJECTS='core,indexing-hadoop,indexing-service,processing' - script: ${MVN} test -B -pl ${MAVEN_PROJECTS} -Ddruid.console.skip=true -DargLine=-Xmx3000m - - - name: "Build and test on ARM64 CPU architecture (2)" - stage: Tests - phase 2 - arch: arm64-graviton2 - dist: focal - virt: vm - group: edge - jdk: openjdk11 - env: - - MAVEN_PROJECTS='core,sql,server,services' - script: ${MVN} test -B -pl ${MAVEN_PROJECTS} -Ddruid.console.skip=true -DargLine=-Xmx3000m - - name: "web console end-to-end test" + stage: Tests - phase 1 before_install: *setup_generate_license install: web-console/script/druid build before_script: @@ -415,6 +394,7 @@ jobs: after_script: web-console/script/druid stop - name: "docs" + stage: Tests - phase 1 install: ./check_test_suite.py && travis_terminate 0 || (cd website && npm install) script: |- (cd website && npm run lint && npm run spellcheck) || { echo " @@ -431,6 +411,29 @@ jobs: " && false; } + - name: "Build and test on ARM64 CPU architecture (1)" + stage: Tests - phase 2 + arch: arm64-graviton2 + dist: focal + virt: vm + group: edge + jdk: openjdk11 + env: + - MAVEN_PROJECTS='core,indexing-hadoop,indexing-service,processing' + script: ${MVN} test -B -pl ${MAVEN_PROJECTS} -Ddruid.console.skip=true -DargLine=-Xmx3000m + + - name: "Build and test on ARM64 CPU architecture (2)" + stage: Tests - phase 2 + arch: arm64-graviton2 + dist: focal + virt: vm + group: edge + jdk: openjdk11 + env: + - MAVEN_PROJECTS='core,sql,server,services' + script: ${MVN} test -B -pl ${MAVEN_PROJECTS} -Ddruid.console.skip=true -DargLine=-Xmx3000m + + # Integration tests Java Compile version is set by the machine environment jdk (set by the jdk key) # Integration tests Java Runtime version is set by the JVM_RUNTIME env property (set env key to -Djvm.runtime=) # Integration tests will either use MiddleManagers or Indexers diff --git a/check_test_suite.py b/check_test_suite.py index 3de4047eeed9..bdc05aef8d25 100755 --- a/check_test_suite.py +++ b/check_test_suite.py @@ -28,8 +28,11 @@ # of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed ignore_prefixes = ['.github', '.idea', '.asf.yaml', '.backportrc.json', '.codecov.yml', '.dockerignore', '.gitignore', '.lgtm.yml', 'CONTRIBUTING.md', 'setup-hooks.sh', 'upload.sh', 'dev', 'distribution/docker', - 'distribution/asf-release-process-guide.md', '.travis.yml', 'check_test_suite.py', - 'check_test_suite_test.py', 'owasp-dependency-check-suppressions.xml'] + 'distribution/asf-release-process-guide.md', '.travis.yml', + 'owasp-dependency-check-suppressions.xml'] + +script_prefixes = ['check_test_suite.py', 'check_test_suite_test.py'] +script_job = ['script checks'] # these files are docs changes # if changes are limited to this set then we can skip web-console and java @@ -43,6 +46,7 @@ web_console_prefixes = ['web-console/'] # travis web-console job name web_console_jobs = ['web console', 'web console end-to-end test'] +web_console_still_run_for_java_jobs = ['web console end-to-end test'] def check_ignore(file): @@ -51,6 +55,11 @@ def check_ignore(file): print("found ignorable file change: {}".format(file)) return is_always_ignore +def check_testable_script(file): + is_script = True in (file.startswith(prefix) for prefix in script_prefixes) + if is_script: + print("found script file change: {}".format(file)) + return is_script def check_docs(file): is_docs = True in (file.startswith(prefix) for prefix in docs_prefixes) @@ -85,6 +94,8 @@ def check_should_run_suite(suite, diff_files): any_console = False all_console = True any_java = False + any_testable_script = False + all_testable_script = True # go over all of the files in the diff and collect some information about the diff contents, we'll use this later # to decide whether or not to run the suite @@ -96,8 +107,11 @@ def check_should_run_suite(suite, diff_files): all_docs = all_docs and is_docs is_console = check_console(f) any_console = any_console or is_console - all_console = any_console and is_console - any_java = any_java or (not is_ignore and not is_docs and not is_console) + all_console = all_console and is_console + is_script = check_testable_script(f) + any_testable_script = any_testable_script or is_script + all_testable_script = all_testable_script and is_script + any_java = any_java or (not is_ignore and not is_docs and not is_console and not is_script) # if everything is ignorable, we can skip this suite if all_ignore: @@ -108,12 +122,18 @@ def check_should_run_suite(suite, diff_files): # if all of the changes are docs paths, but the current suite is not a docs job, we can skip if all_docs: return False + if suite in web_console_still_run_for_java_jobs: + return any_console or any_java # if the test suite is a web console job, return true if any of the changes are web console files if suite in web_console_jobs: return any_console # if all of the changes are web console paths, but the current suite is not a web console job, we can skip if all_console: return False + if suite in script_job: + return any_testable_script + if all_testable_script: + return False # if all of the files belong to known non-java groups, we can also skip java # note that this should probably be reworked to much more selectively run the java jobs depending on the diff diff --git a/check_test_suite_test.py b/check_test_suite_test.py index 18446ea8d992..e8b7f9b039c3 100755 --- a/check_test_suite_test.py +++ b/check_test_suite_test.py @@ -46,6 +46,8 @@ def test_docs(self): ) def test_web_console(self): + web_console_job = 'web console' + e2e_job = 'web console end-to-end test' self.assertEqual(False, check_test_suite.check_console('.travis.yml')) self.assertEqual(False, check_test_suite.check_console('check_test_suite_test.py')) self.assertEqual(False, check_test_suite.check_console('website/core/Footer.js')) @@ -53,25 +55,74 @@ def test_web_console(self): self.assertEqual(True, check_test_suite.check_console('web-console/src/views/index.ts')) self.assertEqual(True, check_test_suite.check_console('web-console/unified-console.html')) - for job in check_test_suite.web_console_jobs: - self.assertEqual( - True, - check_test_suite.check_should_run_suite( - job, - ['check_test_suite_test.py', 'web-console/unified-console.html'] - ) + self.assertEqual( + True, + check_test_suite.check_should_run_suite( + web_console_job, + ['check_test_suite_test.py', 'web-console/unified-console.html'] ) - self.assertEqual( - False, - check_test_suite.check_should_run_suite( - job, - ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] - ) + ) + self.assertEqual( + False, + check_test_suite.check_should_run_suite( + web_console_job, + ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + self.assertEqual( + True, + check_test_suite.check_should_run_suite( + e2e_job, + ['check_test_suite_test.py', 'web-console/unified-console.html'] + ) + ) + self.assertEqual( + True, + check_test_suite.check_should_run_suite( + e2e_job, + ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + + def test_testable_script(self): + self.assertEqual(False, check_test_suite.check_testable_script('.travis.yml')) + self.assertEqual(True, check_test_suite.check_testable_script('check_test_suite.py')) + self.assertEqual(True, check_test_suite.check_testable_script('check_test_suite_test.py')) + + script_job = 'script checks' + some_java_job = 'spotbugs checks' + self.assertEqual( + False, + check_test_suite.check_should_run_suite( + script_job, + ['core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + self.assertEqual( + True, + check_test_suite.check_should_run_suite( + some_java_job, + ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + self.assertEqual( + True, + check_test_suite.check_should_run_suite( + some_java_job, + ['check_test_suite_test.py', 'core/src/main/java/org/apache/druid/math/expr/Expr.java'] + ) + ) + self.assertEqual( + False, + check_test_suite.check_should_run_suite( + some_java_job, + ['check_test_suite_test.py'] ) + ) def test_some_java(self): - some_java_job = "spotbugs checks" + some_java_job = 'spotbugs checks' some_non_java_diffs = [ ['.travis.yml'], ['check_test_suite_test.py'],