diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8347f8a860..cfaeff6b8e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,8 +20,25 @@ on: # run every night at midnight - cron: '0 0 * * *' +# TODO: Our workflow is far from ideal. We need to refactor it into multiple +# ones and only run commands which are needed for some steps for those steps and +# not for all jobs: + pre_job: + name: Skip Duplicate Jobs Pre Job + runs-on: ubuntu-latest + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - id: skip_check + uses: fkirc/skip-duplicate-actions@4c656bbdb6906310fa6213604828008bc28fe55d # v3.3.0 + with: + cancel_others: 'true' + github_token: ${{ github.token }} + ci: + needs: pre_job + if: ${{ needs.pre_job.outputs.should_skip != 'true' }} name: '${{ matrix.name }} - python (${{ matrix.python-version }})' runs-on: ubuntu-latest strategy: @@ -40,13 +57,13 @@ jobs: - name: 'Unit Tests' task: 'ci-unit' python-version: '3.6' + - name: 'Integration Tests' + task: 'ci-integration' + python-version: '3.6' # This job is slow so we only run in on a daily basis # - name: 'Micro Benchmarks' # task: 'micro-benchmarks' # python-version: '3.6' - - name: 'Integration Tests' - task: 'ci-integration' - python-version: '3.6' services: mongo: image: mongo:4.0 @@ -82,6 +99,23 @@ jobs: #- 15671:15671/tcp # Management: SSL port #- 25672:25672/tcp # inter-node or CLI #- 4369:4369/tcp # epmd + # + + # Used for the coordination backend for integration tests + # TODO: Only start this for integration tests via job step + # https://github.community/t/conditional-services-in-a-job/135301/3 + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp env: TASK: '${{ matrix.task }}' @@ -191,7 +225,7 @@ jobs: - name: Install requirements run: | ./scripts/ci/install-requirements.sh - - name: Setup integration tests + - name: Setup Integration Tests run: | # prep a ci-specific dev conf file that uses runner instead of stanley # this user is the username of the user in GitHub actions, used for SSH, etc during diff --git a/scripts/github/prepare-integration.sh b/scripts/github/prepare-integration.sh index 767a4999b7..35e138c8ea 100755 --- a/scripts/github/prepare-integration.sh +++ b/scripts/github/prepare-integration.sh @@ -10,6 +10,25 @@ fi # shellcheck disable=SC1091 source ./virtualenv/bin/activate +# Enable coordination backend to avoid race conditions with orquesta tests due +# to the lack of the coordination backend +sed -i "s#\#url = redis://localhost#url = redis://127.0.0.1#g" ./conf/st2.dev.conf +sed -i "s#\#url = redis://localhost#url = redis://127.0.0.1#g" ./conf/st2.ci.conf || true + +echo "Used config for the tests" +echo "" +echo "st2.dev.conf" +echo "" +cat conf/st2.dev.conf +echo "" +echo "st2.ci.conf" +echo "" +cat conf/st2.ci.conf || true +echo "" + +# Needed by the coordination backend +pip install "redis==3.5.3" + # install st2 client python ./st2client/setup.py develop st2 --version diff --git a/st2common/st2common/service_setup.py b/st2common/st2common/service_setup.py index 03f9bcdfa7..065ce98833 100644 --- a/st2common/st2common/service_setup.py +++ b/st2common/st2common/service_setup.py @@ -44,6 +44,7 @@ from st2common.services import coordination from st2common.logging.misc import add_global_filters_for_all_loggers from st2common.constants.error_messages import PYTHON2_DEPRECATION +from st2common.services.coordination import get_driver_name # Note: This is here for backward compatibility. # Function has been moved in a standalone module to avoid expensive in-direct @@ -147,6 +148,9 @@ def setup( LOG.info("Using logging config: %s", logging_config_path) + LOG.info("Using coordination driver: %s", get_driver_name()) + LOG.info("Using metrics driver: %s", cfg.CONF.metrics.driver) + is_debug_enabled = cfg.CONF.debug or cfg.CONF.system.debug try: diff --git a/st2common/st2common/services/coordination.py b/st2common/st2common/services/coordination.py index 1a068632ab..fadfd2768f 100644 --- a/st2common/st2common/services/coordination.py +++ b/st2common/st2common/services/coordination.py @@ -175,6 +175,19 @@ def configured(): return backend_configured and not mock_backend +def get_driver_name() -> str: + """ + Return coordination driver name (aka protocol part from the URI / URL). + """ + url = cfg.CONF.coordination.url + + if not url: + return None + + driver_name = url.split("://")[0] + return driver_name + + def coordinator_setup(start_heart=True): """ Sets up the client for the coordination service. diff --git a/st2common/tests/integration/test_service_setup_log_level_filtering.py b/st2common/tests/integration/test_service_setup_log_level_filtering.py index 67479fc8e3..9062319808 100644 --- a/st2common/tests/integration/test_service_setup_log_level_filtering.py +++ b/st2common/tests/integration/test_service_setup_log_level_filtering.py @@ -74,11 +74,13 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): process.send_signal(signal.SIGKILL) # Verify first 4 environment related log messages - stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[:4]) + stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[:6]) self.assertIn("INFO [-] Using Python:", stdout) self.assertIn("INFO [-] Using fs encoding:", stdout) self.assertIn("INFO [-] Using config files:", stdout) self.assertIn("INFO [-] Using logging config:", stdout) + self.assertIn("INFO [-] Using coordination driver:", stdout) + self.assertIn("INFO [-] Using metrics driver:", stdout) # 1. INFO log level - audit messages should not be included process = self._start_process(config_path=ST2_CONFIG_INFO_LL_PATH) @@ -88,8 +90,8 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): eventlet.sleep(3) process.send_signal(signal.SIGKILL) - # First 4 log lines are debug messages about the environment which are always logged - stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:]) + # First 6 log lines are debug messages about the environment which are always logged + stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[6:]) self.assertIn("INFO [-]", stdout) self.assertNotIn("DEBUG [-]", stdout) @@ -103,8 +105,8 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): eventlet.sleep(5) process.send_signal(signal.SIGKILL) - # First 4 log lines are debug messages about the environment which are always logged - stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:]) + # First 6 log lines are debug messages about the environment which are always logged + stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[6:]) self.assertIn("INFO [-]", stdout) self.assertIn("DEBUG [-]", stdout) @@ -118,8 +120,8 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): eventlet.sleep(5) process.send_signal(signal.SIGKILL) - # First 4 log lines are debug messages about the environment which are always logged - stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:]) + # First 6 log lines are debug messages about the environment which are always logged + stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[6:]) self.assertNotIn("INFO [-]", stdout) self.assertNotIn("DEBUG [-]", stdout) @@ -133,8 +135,8 @@ def test_audit_log_level_is_filtered_if_log_level_is_not_debug_or_audit(self): eventlet.sleep(5) process.send_signal(signal.SIGKILL) - # First 4 log lines are debug messages about the environment which are always logged - stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[4:]) + # First 6 log lines are debug messages about the environment which are always logged + stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n")[6:]) self.assertIn("INFO [-]", stdout) self.assertIn("DEBUG [-]", stdout) diff --git a/st2common/tests/unit/services/test_synchronization.py b/st2common/tests/unit/services/test_synchronization.py index 991e6b9036..e0c42b50f4 100644 --- a/st2common/tests/unit/services/test_synchronization.py +++ b/st2common/tests/unit/services/test_synchronization.py @@ -39,16 +39,28 @@ def tearDownClass(cls): super(SynchronizationTest, cls).tearDownClass() def test_service_configured(self): + cfg.CONF.set_override(name="url", override=None, group="coordination") + self.assertEqual(coordination.get_driver_name(), None) + cfg.CONF.set_override( name="url", override="kazoo://127.0.0.1:2181", group="coordination" ) self.assertTrue(coordination.configured()) + self.assertEqual(coordination.get_driver_name(), "kazoo") cfg.CONF.set_override(name="url", override="file:///tmp", group="coordination") self.assertFalse(coordination.configured()) + self.assertEqual(coordination.get_driver_name(), "file") cfg.CONF.set_override(name="url", override="zake://", group="coordination") self.assertFalse(coordination.configured()) + self.assertEqual(coordination.get_driver_name(), "zake") + + cfg.CONF.set_override( + name="url", override="redis://foo:bar@127.0.0.1", group="coordination" + ) + self.assertTrue(coordination.configured()) + self.assertEqual(coordination.get_driver_name(), "redis") def test_lock(self): name = uuid.uuid4().hex