allow kvstore upgrade

roles/splunk_common/tasks/get_facts.yml

@@ -57,23 +57,8 @@
   include_tasks: get_facts_target_version.yml
   when: splunk_target_version is not defined or splunk_target_version == none
 
-# We can apply the same logic to the current version by checking which manifest file is in Splunk
-- name: "Find manifests"
-  find:
-    paths: "{{ splunk.home }}"
-    patterns: ".*-manifest$"
-    use_regex: yes
-  become: yes
-  become_user: "{{ splunk.user }}"
-  register: manifests
-
 - name: "Set current version fact"
-  set_fact:
-    splunk_current_version: "{{ manifests.files[0].path | regex_search(regexp, '\\1') if (manifests.matched == 1) else '0' }}"
-    splunk_current_build_hash: "{{ manifests.files[0].path | regex_search(regexp, '\\3') if (manifests.matched == 1) else '0' }}"
-    splunk_target_build_hash: "{{ splunk.build_location | string | regex_search(regexp, '\\3') | default('0') }}"
-  vars:
-    regexp: 'splunk\D*?-(\d+\.\d+\.\d+(\.\d+)?)-(.*?)-.*?'
+  include_tasks: get_facts_current_version.yml
 
 # We are upgrading if it is not a fresh installation and the current version is different from the target version,
 # and allowing upgrades between new and old hashes of the same version.
@@ -131,3 +116,21 @@
 # we want to manually specify it. This is where we detect it was manually set.
 - name: "Detect service name"
   include_tasks: get_facts_service_name.yml
+
+# Check if there is existing data in $SPLUNK_HOME/var
+# This is used primarily for ephemeral hosts with a persistent storage for data.
+- name: Check if var/lib directory exists
+  ansible.builtin.stat:
+    path: "{{ splunk.home }}/var/lib"
+  become: yes
+  become_user: "{{ splunk.user }}"
+  register: var_lib
+
+# Checking if the KVStore has already been migrated to the WiredTiger engine
+# The mmapv1 engine for KVStore is deprecated, and will not work on newer versions of splunk
+- name: Check if KVStore storage engine is already on wiredTiger
+  ansible.builtin.stat:
+    path: "{{ splunk.home }}/var/lib/splunk/kvstore/mongo/WiredTiger"
+  become: yes
+  become_user: "{{ splunk.user }}"
+  register: wiredtiger_file

roles/splunk_common/tasks/get_facts_current_version.yml

@@ -0,0 +1,48 @@
+---
+# If a splunk.version file exists, use that to get the current version,
+# otherwise try to use the manifest.
+
+- name: "Check for splunk.version file"
+  stat:
+    path: "{{ splunk.home }}/etc/splunk.version"
+  register: splunk_version_file
+
+- block:
+  - name: "Get splunk_current_version from splunk.version"
+    shell: "grep VERSION {{ splunk.home }}/etc/splunk.version | awk -F'=' '{print $2}'"
+    register: splunk_current_version
+    become: yes
+    become_user: "{{ splunk.user }}"
+    changed_when: false
+
+  - name: "Get splunk_current_build_hash from splunk.version"
+    shell: "grep BUILD {{ splunk.home }}/etc/splunk.version | awk -F'=' '{print $2}'"
+    register: splunk_current_build_hash
+    become: yes
+    become_user: "{{ splunk.user }}"
+    changed_when: false
+
+  - name: Set splunk_current_version
+    set_fact:
+      splunk_current_version: "{{ splunk_current_version.stdout }}"
+      splunk_current_build_hash: " {{ splunk_current_build_hash.stdout }}"
+  when: splunk_version_file.stat.exists
+
+- block:
+  - name: "Find manifests"
+    find:
+      paths: "{{ splunk.home }}"
+      patterns: ".*-manifest$"
+      use_regex: yes
+    become: yes
+    become_user: "{{ splunk.user }}"
+    register: manifests
+
+  - name: "Set current version fact"
+    set_fact:
+      splunk_current_version: "{{ manifests.files[0].path | regex_search(regexp, '\\1') if (manifests.matched == 1) else '0' }}"
+      splunk_current_build_hash: "{{ manifests.files[0].path | regex_search(regexp, '\\3') if (manifests.matched == 1) else '0' }}"
+      splunk_target_build_hash: "{{ splunk.build_location | string | regex_search(regexp, '\\3') | default('0') }}"
+    vars:
+      regexp: 'splunk\D*?-(\d+\.\d+\.\d+(\.\d+)?)-(.*?)-.*?'
+  when: not splunk_version_file.stat.exists

roles/splunk_common/tasks/main.yml

@@ -121,6 +121,13 @@
 
 - include_tasks: start_splunk.yml
 
+# KVStore upgrades and actions can only be done after the server.pem has been generated
+# Only upgrade standalone instances
+- include_tasks: upgrade_kvstore.yml
+  when:
+    - var_lib.stat.exists
+    - splunk.role == "splunk_standalone"
+
 - include_tasks: set_certificate_prefix.yml
 
 - include_tasks: clean_user_seed.yml

roles/splunk_common/tasks/update_etc.yml

@@ -14,3 +14,9 @@
   become_user: "{{ splunk.user }}"
   environment:
     SPLUNK_HOME: "{{ splunk.home }}"
+
+# Reset the current version if the splunk.version file was populated.
+- name: "Set current version fact"
+  include_tasks: get_facts_current_version.yml
+  when:
+    - updateetc_stat_result.stat.exists

roles/splunk_common/tasks/upgrade_kvstore.yml

@@ -0,0 +1,40 @@
+---
+# Splunk needs to be stopped for KVStore upgrade
+- include_tasks: stop_splunk.yml
+
+# Splunk version 9.1 does not ship with MongoDB 3.6, 
+# so we need to upgrade mongo before we do anything.
+- name: Upgrade MongoDB version from 3.6 to 4.0
+  command: "{{ splunk.exec }} migrate migrate-kvstore-36-40"
+  become: yes
+  become_user: "{{ splunk.user }}"
+  when:
+    - splunk_current_version is version('9', '>=')
+
+- block:
+  - name: Configure storageEngineMigration if on version 8.x
+    ini_file:
+      dest: "{{ splunk.home }}/etc/system/local/server.conf"
+      section: kvstore
+      option: "storageEngineMigration"
+      value: "true"
+      owner: "{{ splunk.user }}"
+      group: "{{ splunk.group }}"
+    when:
+      - splunk_current_version is version('9', '<')
+      - not wiredtiger_file.stat.exists
+
+- name: Upgrade KVStore to wiredTiger
+  command: "{{ splunk.exec }} migrate kvstore-storage-engine --target-engine wiredTiger"
+  become: yes
+  become_user: "{{ splunk.user }}"
+  when:
+    - not wiredtiger_file.stat.exists
+
+# Always migrate to the latest version
+- name: Upgrade KVStire to latest version
+  command: "{{ splunk.exec }} migrate migrate-kvstore"
+  become: yes
+  become_user: "{{ splunk.user }}"
+
+- include_tasks: start_splunk.yml