From 1c3090f2e1963dac6dfc28766d5bb63aeb457464 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 21 Jun 2024 10:48:38 +0100 Subject: [PATCH 01/10] Enable CIS benchmark hardening in AIO and Multinode Currently we do not have coverage of this feature in CI. --- doc/source/configuration/security-hardening.rst | 15 +++++++++++++++ etc/kayobe/environments/ci-aio/stackhpc.yml | 3 +++ etc/kayobe/environments/ci-multinode/stackhpc.yml | 3 +++ .../overcloud-host-configure/post.d/99-cis.yml | 4 ++++ etc/kayobe/stackhpc.yml | 6 ++++++ .../notes/adds-cis-hook-8cec8d42103d075e.yaml | 7 +++++++ 6 files changed, 38 insertions(+) create mode 100644 etc/kayobe/environments/ci-aio/stackhpc.yml create mode 100644 etc/kayobe/environments/ci-multinode/stackhpc.yml create mode 100644 etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml create mode 100644 releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml diff --git a/doc/source/configuration/security-hardening.rst b/doc/source/configuration/security-hardening.rst index f0cd77df39..effb6b8209 100644 --- a/doc/source/configuration/security-hardening.rst +++ b/doc/source/configuration/security-hardening.rst @@ -40,3 +40,18 @@ whether or not workloads or API requests are affected by any configuration chang kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cis.yml +Enabling the host configure hook +-------------------------------- + +A hook is pre-installed but its execution is guarded by the +``stackhpc_enable_cis_benchmark_hardening`` configuration option. +If you want the hardening playbooks to run automatically, as part of +host configure, simply set this flag to ``true``: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/stackhpc.yml + + stackhpc_enable_cis_benchmark_hardening: true + +Alternatively, this can be toggled on a per-environment basis by +setting it in an environment specific config file. diff --git a/etc/kayobe/environments/ci-aio/stackhpc.yml b/etc/kayobe/environments/ci-aio/stackhpc.yml new file mode 100644 index 0000000000..d31ba2a594 --- /dev/null +++ b/etc/kayobe/environments/ci-aio/stackhpc.yml @@ -0,0 +1,3 @@ +--- + +stackhpc_enable_cis_benchmark_hardening: true diff --git a/etc/kayobe/environments/ci-multinode/stackhpc.yml b/etc/kayobe/environments/ci-multinode/stackhpc.yml new file mode 100644 index 0000000000..d31ba2a594 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/stackhpc.yml @@ -0,0 +1,3 @@ +--- + +stackhpc_enable_cis_benchmark_hardening: true diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml new file mode 100644 index 0000000000..9c2e9e898a --- /dev/null +++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml @@ -0,0 +1,4 @@ +--- + +import_playbook: ../../../ansible/cis.yml +when: stackhpc_enable_cis_benchmark_hardening | bool diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index bd00b85d6b..0df969b133 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -148,3 +148,9 @@ stackhpc_docker_registry: "{{ pulp_url | regex_replace('^https?://', '') }}" # Username and password of container registry. stackhpc_docker_registry_username: "{{ pulp_username }}" stackhpc_docker_registry_password: "{{ pulp_password }}" + +############################################################################### +# Feature flags + +# Whether or not to run CIS benchmark hardening playbooks +stackhpc_enable_cis_benchmark_hardening: false diff --git a/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml b/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml new file mode 100644 index 0000000000..a78386f90c --- /dev/null +++ b/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml @@ -0,0 +1,7 @@ +--- +features: + - | + Adds a hook to automatically run the CIS benchmark hardening playbooks as + part of host configure. This is guarded by the + ``stackhpc_enable_cis_benchmark_hardening`` configuration option and is + disabled by default. From 4ded47e08fbff1f2a52725c4ea01c8902e4bf565 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 21 Jun 2024 11:25:33 +0100 Subject: [PATCH 02/10] Support extending default hardening group --- doc/source/configuration/security-hardening.rst | 3 ++- etc/kayobe/ansible/cis.yml | 2 +- etc/kayobe/inventory/group_vars/all/stackhpc | 6 ++++++ etc/kayobe/inventory/groups | 6 ++++++ etc/kayobe/stackhpc.yml | 4 ++-- 5 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 etc/kayobe/inventory/group_vars/all/stackhpc diff --git a/doc/source/configuration/security-hardening.rst b/doc/source/configuration/security-hardening.rst index effb6b8209..53a02ab05f 100644 --- a/doc/source/configuration/security-hardening.rst +++ b/doc/source/configuration/security-hardening.rst @@ -54,4 +54,5 @@ host configure, simply set this flag to ``true``: stackhpc_enable_cis_benchmark_hardening: true Alternatively, this can be toggled on a per-environment basis by -setting it in an environment specific config file. +setting it in an environment specific config file, or even on +targeted hosts by using group or host vars. diff --git a/etc/kayobe/ansible/cis.yml b/etc/kayobe/ansible/cis.yml index f35906344e..a97151f6be 100644 --- a/etc/kayobe/ansible/cis.yml +++ b/etc/kayobe/ansible/cis.yml @@ -1,7 +1,7 @@ --- - name: Security hardening - hosts: overcloud + hosts: cis-hardening become: true tasks: # TODO: Remove this when Red Hat FIPS policy has been updated to allow ed25519 keys. diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc b/etc/kayobe/inventory/group_vars/all/stackhpc new file mode 100644 index 0000000000..7c644efd97 --- /dev/null +++ b/etc/kayobe/inventory/group_vars/all/stackhpc @@ -0,0 +1,6 @@ +--- +############################################################################### +# Feature flags + +# Whether or not to run CIS benchmark hardening playbooks. Default is false. +stackhpc_enable_cis_benchmark_hardening: false \ No newline at end of file diff --git a/etc/kayobe/inventory/groups b/etc/kayobe/inventory/groups index d368b1bb2c..e957bbc7e4 100644 --- a/etc/kayobe/inventory/groups +++ b/etc/kayobe/inventory/groups @@ -125,3 +125,9 @@ rgws [mgrs] [osds] [rgws] + +############################################################################### +# Feature control groups + +[cis-hardening:children] +overcloud diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index 0df969b133..aea68f1ba5 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -152,5 +152,5 @@ stackhpc_docker_registry_password: "{{ pulp_password }}" ############################################################################### # Feature flags -# Whether or not to run CIS benchmark hardening playbooks -stackhpc_enable_cis_benchmark_hardening: false +# Whether or not to run CIS benchmark hardening playbooks. Default is false. +#stackhpc_enable_cis_benchmark_hardening: From fd2efb342f6e33420d15dd57a6834dff78a4a5ec Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 21 Jun 2024 11:40:20 +0100 Subject: [PATCH 03/10] Fix hook --- etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml index 9c2e9e898a..e9ce88d89a 100644 --- a/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml +++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml @@ -1,4 +1,4 @@ --- -import_playbook: ../../../ansible/cis.yml -when: stackhpc_enable_cis_benchmark_hardening | bool +- import_playbook: ../../../ansible/cis.yml + when: stackhpc_enable_cis_benchmark_hardening | bool From ee35591b3b4aa878c954a9672a2ca2d623355531 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 10 Jul 2024 11:59:57 +0100 Subject: [PATCH 04/10] Update name of variable --- .../configuration/security-hardening.rst | 18 ++++++++++++++++-- etc/kayobe/environments/ci-aio/stackhpc.yml | 2 +- .../environments/ci-multinode/stackhpc.yml | 2 +- .../overcloud-host-configure/post.d/99-cis.yml | 2 +- etc/kayobe/inventory/group_vars/all/stackhpc | 2 +- etc/kayobe/stackhpc.yml | 2 +- .../notes/adds-cis-hook-8cec8d42103d075e.yaml | 2 +- 7 files changed, 22 insertions(+), 8 deletions(-) diff --git a/doc/source/configuration/security-hardening.rst b/doc/source/configuration/security-hardening.rst index 53a02ab05f..40850767b3 100644 --- a/doc/source/configuration/security-hardening.rst +++ b/doc/source/configuration/security-hardening.rst @@ -40,18 +40,32 @@ whether or not workloads or API requests are affected by any configuration chang kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/cis.yml +Targetting additional hosts +--------------------------- + +The ``cis.yml`` playbook targets hosts in the ``cis-hardening`` group. By +default this includes the ``overcloud`` group. You can adjust this group +to suit your needs, e.g to add the seed VM: + +.. code-block:: yaml + :caption: $KAYOBE_CONFIG_PATH/inventory/groups + + [cis-hardening:children] + overcloud + seed + Enabling the host configure hook -------------------------------- A hook is pre-installed but its execution is guarded by the -``stackhpc_enable_cis_benchmark_hardening`` configuration option. +``stackhpc_enable_cis_benchmark_hardening_hook`` configuration option. If you want the hardening playbooks to run automatically, as part of host configure, simply set this flag to ``true``: .. code-block:: yaml :caption: $KAYOBE_CONFIG_PATH/stackhpc.yml - stackhpc_enable_cis_benchmark_hardening: true + stackhpc_enable_cis_benchmark_hardening_hook: true Alternatively, this can be toggled on a per-environment basis by setting it in an environment specific config file, or even on diff --git a/etc/kayobe/environments/ci-aio/stackhpc.yml b/etc/kayobe/environments/ci-aio/stackhpc.yml index d31ba2a594..a2c7858bb6 100644 --- a/etc/kayobe/environments/ci-aio/stackhpc.yml +++ b/etc/kayobe/environments/ci-aio/stackhpc.yml @@ -1,3 +1,3 @@ --- -stackhpc_enable_cis_benchmark_hardening: true +stackhpc_enable_cis_benchmark_hardening_hook: true diff --git a/etc/kayobe/environments/ci-multinode/stackhpc.yml b/etc/kayobe/environments/ci-multinode/stackhpc.yml index d31ba2a594..a2c7858bb6 100644 --- a/etc/kayobe/environments/ci-multinode/stackhpc.yml +++ b/etc/kayobe/environments/ci-multinode/stackhpc.yml @@ -1,3 +1,3 @@ --- -stackhpc_enable_cis_benchmark_hardening: true +stackhpc_enable_cis_benchmark_hardening_hook: true diff --git a/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml index e9ce88d89a..9c132efbc1 100644 --- a/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml +++ b/etc/kayobe/hooks/overcloud-host-configure/post.d/99-cis.yml @@ -1,4 +1,4 @@ --- - import_playbook: ../../../ansible/cis.yml - when: stackhpc_enable_cis_benchmark_hardening | bool + when: stackhpc_enable_cis_benchmark_hardening_hook | bool diff --git a/etc/kayobe/inventory/group_vars/all/stackhpc b/etc/kayobe/inventory/group_vars/all/stackhpc index 7c644efd97..528733ebc8 100644 --- a/etc/kayobe/inventory/group_vars/all/stackhpc +++ b/etc/kayobe/inventory/group_vars/all/stackhpc @@ -3,4 +3,4 @@ # Feature flags # Whether or not to run CIS benchmark hardening playbooks. Default is false. -stackhpc_enable_cis_benchmark_hardening: false \ No newline at end of file +stackhpc_enable_cis_benchmark_hardening_hook: false \ No newline at end of file diff --git a/etc/kayobe/stackhpc.yml b/etc/kayobe/stackhpc.yml index ffeee66d34..c149e9d7be 100644 --- a/etc/kayobe/stackhpc.yml +++ b/etc/kayobe/stackhpc.yml @@ -157,4 +157,4 @@ stackhpc_docker_registry_password: "{{ pulp_password }}" # Feature flags # Whether or not to run CIS benchmark hardening playbooks. Default is false. -#stackhpc_enable_cis_benchmark_hardening: +#stackhpc_enable_cis_benchmark_hardening_hook: diff --git a/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml b/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml index a78386f90c..63c3bd0e05 100644 --- a/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml +++ b/releasenotes/notes/adds-cis-hook-8cec8d42103d075e.yaml @@ -3,5 +3,5 @@ features: - | Adds a hook to automatically run the CIS benchmark hardening playbooks as part of host configure. This is guarded by the - ``stackhpc_enable_cis_benchmark_hardening`` configuration option and is + ``stackhpc_enable_cis_benchmark_hardening_hook`` configuration option and is disabled by default. From 4bfa6177bd7a9165f1a964594ecf68c69d1933de Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 10 Jul 2024 12:13:36 +0100 Subject: [PATCH 05/10] Add note about needing to reboot --- doc/source/configuration/security-hardening.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/configuration/security-hardening.rst b/doc/source/configuration/security-hardening.rst index 40850767b3..e958011f7b 100644 --- a/doc/source/configuration/security-hardening.rst +++ b/doc/source/configuration/security-hardening.rst @@ -31,6 +31,14 @@ about what each variable does. The documentation can be found here: Running the playbooks --------------------- +.. note:: + + The hosts may need rebooting to fully pick up all of the changes. The CIS + roles will warn you when this needs to be done, but the actual reboot is left + as a manual operation to allow you to select a convenient time. Generally, if + you are applying the hardening for the first time, then you will need to + reboot. + As there is potential for unintended side effects when applying the hardening playbooks, the playbooks are not currently enabled by default. It is recommended that they are first applied to a representative staging environment to determine From 10cca926c5f48f87741643ad09ad1d50d436f19c Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Wed, 10 Jul 2024 17:01:38 +0100 Subject: [PATCH 06/10] Add cis tag --- etc/kayobe/ansible/cis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/kayobe/ansible/cis.yml b/etc/kayobe/ansible/cis.yml index a97151f6be..e71e17ff3e 100644 --- a/etc/kayobe/ansible/cis.yml +++ b/etc/kayobe/ansible/cis.yml @@ -3,6 +3,8 @@ - name: Security hardening hosts: cis-hardening become: true + tags: + - cis tasks: # TODO: Remove this when Red Hat FIPS policy has been updated to allow ed25519 keys. # https://gitlab.com/gitlab-org/gitlab/-/issues/367429#note_1840422075 From 6308f6c945a27d3ac4a58798c4d24057e701de82 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 11 Jul 2024 10:00:20 +0000 Subject: [PATCH 07/10] Ensure we have an IP on breth1 --- .github/workflows/stackhpc-all-in-one.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 5f8409a500..6668bbf36a 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -355,6 +355,19 @@ jobs: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} if: inputs.upgrade + - name: Ensure we have IP on breth1 to reach the instances + # NOTE(wszumski): Whilst we don't need to create resources again, in some circumstances + # we can lose the IP address that allows us to connect to the instances. This playbook + # also fixes that issue. + run: | + docker run -t --rm \ + -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ + -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ + ${{ steps.kayobe_image.outputs.kayobe_image }} \ + /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml + env: + KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + - name: Tempest tests id: tempest run: | From 425a0ac713fbb8fc006008d25e980e3bb87f6cb1 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Thu, 11 Jul 2024 15:41:50 +0100 Subject: [PATCH 08/10] Only run configure-aio-resources again on upgrade --- .github/workflows/stackhpc-all-in-one.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 6668bbf36a..88028561b0 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -367,6 +367,7 @@ jobs: /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml env: KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} + if: inputs.upgrade - name: Tempest tests id: tempest From 70fad7bf135334dd353045671fd4cfae4d0a7c26 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Fri, 12 Jul 2024 19:04:43 +0100 Subject: [PATCH 09/10] Move CIS group_vars to cis-hardening group --- etc/kayobe/inventory/group_vars/{overcloud => cis-hardening}/cis | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename etc/kayobe/inventory/group_vars/{overcloud => cis-hardening}/cis (100%) diff --git a/etc/kayobe/inventory/group_vars/overcloud/cis b/etc/kayobe/inventory/group_vars/cis-hardening/cis similarity index 100% rename from etc/kayobe/inventory/group_vars/overcloud/cis rename to etc/kayobe/inventory/group_vars/cis-hardening/cis From 3763af78d22e98fb05f9cb056e7e833b4edf5208 Mon Sep 17 00:00:00 2001 From: Will Szumski Date: Mon, 15 Jul 2024 09:25:50 +0100 Subject: [PATCH 10/10] Update docs for group_vars change of location --- doc/source/configuration/security-hardening.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/configuration/security-hardening.rst b/doc/source/configuration/security-hardening.rst index e958011f7b..1bb8216b50 100644 --- a/doc/source/configuration/security-hardening.rst +++ b/doc/source/configuration/security-hardening.rst @@ -19,7 +19,7 @@ Configuration -------------- Some overrides to the role defaults are provided in -``$KAYOBE_CONFIG_PATH/inventory/group_vars/overcloud/cis``. These may not be +``$KAYOBE_CONFIG_PATH/inventory/group_vars/cis-hardening/cis``. These may not be suitable for all deployments and so some fine tuning may be required. For instance, you may want different rules on a network node compared to a controller. It is best to consult the upstream role documentation for details