From ab9c64b3759824e622350c070845c9bff4c3277c Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Thu, 11 Feb 2021 22:58:28 +0100 Subject: [PATCH 01/14] Add test environment setup --- docker-compose.yml | 63 ++++++++++++++++++++++++++++++++++++++++++++ test/bkp_default.yml | 29 ++++++++++++++++++++ test/custom_init.sh | 15 +++++++++++ test/sh_default.yml | 15 +++++++++++ 4 files changed, 122 insertions(+) create mode 100644 docker-compose.yml create mode 100644 test/bkp_default.yml create mode 100755 test/custom_init.sh create mode 100644 test/sh_default.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d345b67 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,63 @@ +version: "3.7" +networks: + splunknet: + driver: bridge + attachable: true + +services: + splunk_sh: + networks: + splunknet: + aliases: + - splunk_sh + hostname: splunk_sh + container_name: splunk_sh + image: splunk/splunk:latest + ports: + - 8003:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + volumes: + - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups + - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static + + splunk_moc: + networks: + splunknet: + aliases: + - splunk_moc + hostname: splunk_moc + container_name: splunk_moc + image: splunk/splunk:latest + entrypoint: + - /bin/sh + - /usr/sbin/custom_init.sh + ports: + - 8004:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + - SPLUNK_ROLE=splunk_monitor + - DEBUG=true + + volumes: + - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups + - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ./test/bkp_default.yml:/tmp/defaults/default.yml + - ./test/install_git.yml:/tmp/install_git.yml + - ./test/custom_init.sh:/usr/sbin/custom_init.sh diff --git a/test/bkp_default.yml b/test/bkp_default.yml new file mode 100644 index 0000000..01911ff --- /dev/null +++ b/test/bkp_default.yml @@ -0,0 +1,29 @@ +--- +#ansible_post_tasks: file:///tmp/install_git.yml +splunk: + conf: + - key: inputs + value: + directory: /opt/splunk/etc/apps/SplunkVersionControl/local + content: + splunkversioncontrol_backup://SH: + srcURL: https://splunk_sh:8089 + srcUsername: admin + srcPassword: password:SH + gitTempDir: /opt/splunk/vcs/git_tmp_backup + gitRepoURL: file:///opt/splunk/vcs/backup.git + # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + debugMode: true + useLocalAuth: false + interval: 60 + splunkversioncontrol_restore://SH: + destURL: https://splunk_sh:8089 + destUsername: admin + destPassword: password:SH + gitTempDir: /opt/splunk/vcs/git_tmp_restore + gitRepoURL: file:///opt/splunk/vcs/backup.git + # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + debugMode: true + useLocalAuth: false + auditLogsLookupBackTime: -2h + timewait: 30 diff --git a/test/custom_init.sh b/test/custom_init.sh new file mode 100755 index 0000000..d5ac403 --- /dev/null +++ b/test/custom_init.sh @@ -0,0 +1,15 @@ +set -e +sudo microdnf install git +sudo -u splunk mkdir -p /opt/splunk/vcs/{git_tmp_backup,git_tmp_restore} +sudo -u splunk mkdir -p /opt/splunk/vcs/backup.git +echo "dirs" +sudo -u splunk sh -c "cd /opt/splunk/vcs/backup.git && git init --bare --shared" +echo "init" +sudo -u splunk sh -c "cd /opt/splunk/vcs/ && git clone backup.git" +sudo -u splunk sh -c "git config --global user.email 'backup@example.local'" +sudo -u splunk sh -c "git config --global user.name 'BackupUser'" +echo "clone" +sudo -u splunk sh -c 'cd /opt/splunk/vcs/backup && touch test && git add test && git commit -a -m "init" && git push' +echo "file" + +/usr/sbin/entrypoint.sh start diff --git a/test/sh_default.yml b/test/sh_default.yml new file mode 100644 index 0000000..cc23d8c --- /dev/null +++ b/test/sh_default.yml @@ -0,0 +1,15 @@ +--- +#ansible_post_tasks: file:///tmp/install_git.yml +splunk: + conf: + - key: macros + value: + directory: /opt/splunk/etc/apps/SplunkVersionControl/local + content: + splunk_vc_name: + definition: SH + splunk_vc_url: + definition: https://splunk_moc:8089 + sslVerify: + definiton: false + # definition: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem From 5a910f5b674fc9e637a5375cda7abd70eea0f516 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Thu, 11 Feb 2021 23:10:49 +0100 Subject: [PATCH 02/14] Add check when rerunning same container --- test/custom_init.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/custom_init.sh b/test/custom_init.sh index d5ac403..05d3e4f 100755 --- a/test/custom_init.sh +++ b/test/custom_init.sh @@ -2,14 +2,11 @@ set -e sudo microdnf install git sudo -u splunk mkdir -p /opt/splunk/vcs/{git_tmp_backup,git_tmp_restore} sudo -u splunk mkdir -p /opt/splunk/vcs/backup.git -echo "dirs" sudo -u splunk sh -c "cd /opt/splunk/vcs/backup.git && git init --bare --shared" -echo "init" +if [ ! -d "/opt/splunk/vcs/backup.git" ]; then sudo -u splunk sh -c "cd /opt/splunk/vcs/ && git clone backup.git" sudo -u splunk sh -c "git config --global user.email 'backup@example.local'" sudo -u splunk sh -c "git config --global user.name 'BackupUser'" -echo "clone" sudo -u splunk sh -c 'cd /opt/splunk/vcs/backup && touch test && git add test && git commit -a -m "init" && git push' -echo "file" - +fi /usr/sbin/entrypoint.sh start From 4eeeb2b4001651d62c3bb7840a722903a0508c74 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Thu, 11 Feb 2021 23:11:50 +0100 Subject: [PATCH 03/14] Add credentials --- test/bkp_default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bkp_default.yml b/test/bkp_default.yml index 01911ff..b2afbcc 100644 --- a/test/bkp_default.yml +++ b/test/bkp_default.yml @@ -9,7 +9,7 @@ splunk: splunkversioncontrol_backup://SH: srcURL: https://splunk_sh:8089 srcUsername: admin - srcPassword: password:SH + srcPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_backup gitRepoURL: file:///opt/splunk/vcs/backup.git # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem @@ -19,7 +19,7 @@ splunk: splunkversioncontrol_restore://SH: destURL: https://splunk_sh:8089 destUsername: admin - destPassword: password:SH + destPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_restore gitRepoURL: file:///opt/splunk/vcs/backup.git # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem From 4e5440cc9b3b7e005749b7b52382df92f9810a08 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:16:23 +0100 Subject: [PATCH 04/14] Add macro config --- docker-compose.yml | 1 + test/sh_default.yml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d345b67..33dcd74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ./test/sh_default.yml:/tmp/defaults/default.yml splunk_moc: networks: diff --git a/test/sh_default.yml b/test/sh_default.yml index cc23d8c..8df1e55 100644 --- a/test/sh_default.yml +++ b/test/sh_default.yml @@ -1,5 +1,4 @@ --- -#ansible_post_tasks: file:///tmp/install_git.yml splunk: conf: - key: macros @@ -9,7 +8,7 @@ splunk: splunk_vc_name: definition: SH splunk_vc_url: - definition: https://splunk_moc:8089 + definition: https://splunk_moc:8089/services/splunkversioncontrol_rest_restore sslVerify: definiton: false # definition: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem From ac872be4ffd4fedf41a4ba937955dcc9d4b5b4a2 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:38:41 +0100 Subject: [PATCH 05/14] Fix recreation check --- test/custom_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/custom_init.sh b/test/custom_init.sh index 05d3e4f..36d5336 100755 --- a/test/custom_init.sh +++ b/test/custom_init.sh @@ -1,9 +1,9 @@ set -e sudo microdnf install git sudo -u splunk mkdir -p /opt/splunk/vcs/{git_tmp_backup,git_tmp_restore} +if [ ! -d "/opt/splunk/vcs/backup.git" ]; then sudo -u splunk mkdir -p /opt/splunk/vcs/backup.git sudo -u splunk sh -c "cd /opt/splunk/vcs/backup.git && git init --bare --shared" -if [ ! -d "/opt/splunk/vcs/backup.git" ]; then sudo -u splunk sh -c "cd /opt/splunk/vcs/ && git clone backup.git" sudo -u splunk sh -c "git config --global user.email 'backup@example.local'" sudo -u splunk sh -c "git config --global user.name 'BackupUser'" From e51872e3de740bbb6b351a3d0191c22401faed5e Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:46:55 +0100 Subject: [PATCH 06/14] Remove lookup from container mapping In order to regenerate the lookups especially the taglist on recreation the lookups folder is not shared. --- test/docker-compose.yml | 61 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 test/docker-compose.yml diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 0000000..1d03326 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.7" +networks: + splunknet: + driver: bridge + attachable: true + +services: + splunk_sh: + networks: + splunknet: + aliases: + - splunk_sh + hostname: splunk_sh + container_name: splunk_sh + image: splunk/splunk:latest + ports: + - 8003:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + volumes: + - ../bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ../default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ../lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ../metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ../README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ../static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ../test/sh_default.yml:/tmp/defaults/default.yml + + splunk_moc: + networks: + splunknet: + aliases: + - splunk_moc + hostname: splunk_moc + container_name: splunk_moc + image: splunk/splunk:latest + entrypoint: + - /bin/sh + - /usr/sbin/custom_init.sh + ports: + - 8004:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + - SPLUNK_ROLE=splunk_monitor + - DEBUG=true + + volumes: + - ../bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ../default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ../lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ../metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ../README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ../static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ../test/bkp_default.yml:/tmp/defaults/default.yml + - ../test/custom_init.sh:/usr/sbin/custom_init.sh From 4dd598fea4069b315c3b48e774c39bef2dce01b2 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:52:51 +0100 Subject: [PATCH 07/14] Remove old docker-compose.yml --- docker-compose.yml | 64 ---------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 33dcd74..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: "3.7" -networks: - splunknet: - driver: bridge - attachable: true - -services: - splunk_sh: - networks: - splunknet: - aliases: - - splunk_sh - hostname: splunk_sh - container_name: splunk_sh - image: splunk/splunk:latest - ports: - - 8003:8000 - - 8089 - environment: - - SPLUNK_PASSWORD=g8nGXBQBF - - SPLUNK_START_ARGS=--accept-license - - SPLUNK_STANDALONE_URL=splunk_sh - volumes: - - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin - - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default - - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib - - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups - - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata - - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README - - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static - - ./test/sh_default.yml:/tmp/defaults/default.yml - - splunk_moc: - networks: - splunknet: - aliases: - - splunk_moc - hostname: splunk_moc - container_name: splunk_moc - image: splunk/splunk:latest - entrypoint: - - /bin/sh - - /usr/sbin/custom_init.sh - ports: - - 8004:8000 - - 8089 - environment: - - SPLUNK_PASSWORD=g8nGXBQBF - - SPLUNK_START_ARGS=--accept-license - - SPLUNK_STANDALONE_URL=splunk_sh - - SPLUNK_ROLE=splunk_monitor - - DEBUG=true - - volumes: - - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin - - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default - - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib - - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups - - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata - - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README - - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static - - ./test/bkp_default.yml:/tmp/defaults/default.yml - - ./test/install_git.yml:/tmp/install_git.yml - - ./test/custom_init.sh:/usr/sbin/custom_init.sh From c336b4776902c0deeddf3d85707e0c4c05aa4537 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 12:34:45 +0100 Subject: [PATCH 08/14] Add certification creation scripts --- test/certificates/.gitignore | 5 ++ test/certificates/createca.sh | 13 ++++ test/certificates/createcerts.sh | 129 +++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 test/certificates/.gitignore create mode 100644 test/certificates/createca.sh create mode 100644 test/certificates/createcerts.sh diff --git a/test/certificates/.gitignore b/test/certificates/.gitignore new file mode 100644 index 0000000..423a60f --- /dev/null +++ b/test/certificates/.gitignore @@ -0,0 +1,5 @@ +*.crt +*.key +*.pem +*.csr +castuff diff --git a/test/certificates/createca.sh b/test/certificates/createca.sh new file mode 100644 index 0000000..9491f1d --- /dev/null +++ b/test/certificates/createca.sh @@ -0,0 +1,13 @@ +set -e +CA_KEY=ca.key +CA_CRT=ca.crt +SUBJ="/C=CH/ST=Bern/L=Bern/O=Example Company/CN=CA Domain1" + +if [ ! -f $CA_KEY ]; then + openssl genrsa -out $CA_KEY 4096 +fi + +if [ ! -f $CA_CRT ]; then + openssl req -new -x509 -key ${CA_KEY} -out ${CA_CRT} -subj "$SUBJ" -days 3000 +fi + diff --git a/test/certificates/createcerts.sh b/test/certificates/createcerts.sh new file mode 100644 index 0000000..61e9a36 --- /dev/null +++ b/test/certificates/createcerts.sh @@ -0,0 +1,129 @@ +set -e + +if [ $# -ne 3 ] + then + cat < $dir/serial +[ ! -f $dir/index.txt ] && touch $dir/index.txt +# SIGN Request +openssl ca \ + -in ${CSR} \ + -out ${CERT} \ + -extensions ${CRT_TYPE} \ + -config <( +cat <<-EOF +[ ca ] +default_ca = CA_default +[ CA_default ] +copy_extensions = copy +dir = . +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand + +private_key = ./ca.key +certificate = ./ca.crt + +crlnumber = $dir/crlnumber +crl = $dir/crl/ca.crl.pem +crl_extensions = crl_ext +default_crl_days = 30 + +# SHA-1 is deprecated, so use SHA-2 instead. +default_md = sha256 + +name_opt = ca_default +cert_opt = ca_default +default_days = 375 +preserve = no +policy = policy_loose +unique_subject = no + +[ policy_loose ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ usr_cert ] +basicConstraints = CA:FALSE +nsCertType = client +nsComment = "Client Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, emailProtection + +[ server_cert ] +basicConstraints = CA:FALSE +nsCertType = server +nsComment = "OpenSSL Generated Server Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth + +EOF +) + +cat ${CERT} ${KEYOUT} > ${COMBINED} +#rm ${CERT} ${CSR} ${KEYOUT} From c23d5b4a10c4cd4fb4e3dccb7d5e0ef1b1a52e57 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 14:42:01 +0100 Subject: [PATCH 09/14] Add certificate creation scripts --- test/bkp_default.yml | 9 +++++++++ test/certificates/Dockerfile | 9 +++++++++ test/certificates/createcerts.sh | 1 + test/certificates/entrypoint.sh | 21 +++++++++++++++++++++ test/docker-compose.yml | 21 +++++++++++++++++++++ test/sh_default.yml | 9 +++++++++ 6 files changed, 70 insertions(+) create mode 100644 test/certificates/Dockerfile create mode 100755 test/certificates/entrypoint.sh diff --git a/test/bkp_default.yml b/test/bkp_default.yml index b2afbcc..2385add 100644 --- a/test/bkp_default.yml +++ b/test/bkp_default.yml @@ -27,3 +27,12 @@ splunk: useLocalAuth: false auditLogsLookupBackTime: -2h timewait: 30 + - key: server + value: + directory: /opt/splunk/etc/system/local/ + content: + sslConfig: + enableSplunkdSSL: true + serverCert: /cert_dir/splunk_sh.pem + sslRootCAPath: /cert_dir/ca.crt + sslPassword: password diff --git a/test/certificates/Dockerfile b/test/certificates/Dockerfile new file mode 100644 index 0000000..411a7c2 --- /dev/null +++ b/test/certificates/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:latest + +RUN apk update && \ + apk add --no-cache openssl bash && \ + rm -rf "/var/cache/apk/*" + +COPY createca.sh createcerts.sh entrypoint.sh /scripts/ + +ENTRYPOINT [ "bash", "/scripts/entrypoint.sh" ] diff --git a/test/certificates/createcerts.sh b/test/certificates/createcerts.sh index 61e9a36..3c0a2cc 100644 --- a/test/certificates/createcerts.sh +++ b/test/certificates/createcerts.sh @@ -60,6 +60,7 @@ mkdir -p $dir/certs [ ! -f $dir/index.txt ] && touch $dir/index.txt # SIGN Request openssl ca \ + -batch \ -in ${CSR} \ -out ${CERT} \ -extensions ${CRT_TYPE} \ diff --git a/test/certificates/entrypoint.sh b/test/certificates/entrypoint.sh new file mode 100755 index 0000000..ede1288 --- /dev/null +++ b/test/certificates/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e +echo "Starting Certificate Creation" +mkdir -p /cert_dir +cd /cert_dir +sh /scripts/createca.sh +while read -r line; do + # Get the string before = (the var name) + name="${line%=*}" + eval value="\$$name" + if [[ $name = 'CERTIFICATE'* ]] + then + echo "name: ${name}, value: ${value}" + bash /scripts/createcerts.sh ${value} + fi +done < Date: Fri, 12 Feb 2021 15:58:04 +0100 Subject: [PATCH 10/14] Configuration for ssl --- test/bkp_default.yml | 6 +++--- test/sh_default.yml | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/bkp_default.yml b/test/bkp_default.yml index 2385add..8876445 100644 --- a/test/bkp_default.yml +++ b/test/bkp_default.yml @@ -12,7 +12,7 @@ splunk: srcPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_backup gitRepoURL: file:///opt/splunk/vcs/backup.git - # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + sslVerify: /cert_dir/ca.crt debugMode: true useLocalAuth: false interval: 60 @@ -22,7 +22,7 @@ splunk: destPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_restore gitRepoURL: file:///opt/splunk/vcs/backup.git - # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + sslVerify: /cert_dir/ca.crt debugMode: true useLocalAuth: false auditLogsLookupBackTime: -2h @@ -33,6 +33,6 @@ splunk: content: sslConfig: enableSplunkdSSL: true - serverCert: /cert_dir/splunk_sh.pem + serverCert: /cert_dir/splunk_moc.pem sslRootCAPath: /cert_dir/ca.crt sslPassword: password diff --git a/test/sh_default.yml b/test/sh_default.yml index ef26525..ab5de78 100644 --- a/test/sh_default.yml +++ b/test/sh_default.yml @@ -10,8 +10,11 @@ splunk: splunk_vc_url: definition: https://splunk_moc:8089/services/splunkversioncontrol_rest_restore sslVerify: - definiton: false - # definition: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem +# definiton: false + definition: /cert_dir/ca.crt + requestingAddress: + # is used to connect back + definition: https://splunk_sh:8089 - key: server value: directory: /opt/splunk/etc/system/local/ From f09b152c06725be3e271f36289bd99f4dd921520 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 15:58:16 +0100 Subject: [PATCH 11/14] Adjustments for SSL support --- bin/postversioncontrolrestore.py | 13 +++++++++++-- bin/splunkversioncontrol_backup_class.py | 2 +- bin/splunkversioncontrol_rest_restore.py | 20 ++++++++++++++------ bin/splunkversioncontrol_utility.py | 4 ++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index 73cd511..d34049f 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -10,7 +10,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import dispatch, GeneratingCommand, Configuration, Option -from splunklib.searchcommands.validators import Validator, Boolean, File +from splunklib.searchcommands.validators import Validator, Boolean from splunklib.binding import HTTPError class OrValidator(Validator): @@ -31,6 +31,13 @@ def format(self, value): except: return self.b.format(value) +class Filename(Validator): + # TODO Validate file path + def __call__(self, value): + return value + + def format(self, value): + return value splunkLogsDir = os.environ['SPLUNK_HOME'] + "/var/log/splunk" #Setup the logging @@ -80,7 +87,8 @@ class SVCPostRestore(GeneratingCommand): restoreAsUser = Option(require=True) scope = Option(require=True) timeout = Option(require=True) - sslVerify = Option(require=False, default=False, validate=OrValidator(File(), Boolean())) + sslVerify = Option(require=False, default=False, validate=OrValidator(Boolean(), Filename())) + requestingAddress = Option(require=False, default=False) def generate(self): """ @@ -108,6 +116,7 @@ def generate(self): body['restoreAsUser'] = self.restoreAsUser body['scope'] = self.scope body['timeout'] = self.timeout + body['requestingAddress'] = self.requestingAddress logger.info("Attempting POST request to url=%s with body=\"%s\"" % (url, body)) diff --git a/bin/splunkversioncontrol_backup_class.py b/bin/splunkversioncontrol_backup_class.py index 688d6ed..b2a54ea 100644 --- a/bin/splunkversioncontrol_backup_class.py +++ b/bin/splunkversioncontrol_backup_class.py @@ -1037,7 +1037,7 @@ def run_script(self): headers={'Authorization': 'Splunk %s' % config['session_key']} url = 'https://localhost:8089/services/shcluster/captain/info?output_mode=json' - res = requests.get(url, headers=headers, verify=self.sslVerify) + res = requests.get(url, headers=headers, verify=False) if (res.status_code == 503): logger.debug("i=\"%s\" Non-shcluster / standalone instance, safe to run on this node" % (self.stanzaName)) elif (res.status_code != requests.codes.ok): diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 2b76e72..5a0a968 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -59,12 +59,15 @@ class SVCRestore(splunk.rest.BaseRestHandler): - def query_back_for_user_and_permissions(self, authorization_token, *, sslVerify): + def query_back_for_user_and_permissions(self, requestingAddress, authorization_token, *, sslVerify): headers = { "Authorization" : authorization_token } #Run a query back against the source system to check the username/role - remoteAddr = self.request['remoteAddr'] - url = "https://" + remoteAddr + ":8089/services/authentication/current-context?output_mode=json" + if requestingAddress: + remoteAddr = requestingAddress + else: + remoteAddr = "https://" + self.request['remoteAddr'] + ":8089" + url = remoteAddr + "/services/authentication/current-context?output_mode=json" logger.info("Received remote request checking username and role related to the token on url=%s" % (url)) logger.debug("token=%s" % (authorization_token)) @@ -160,7 +163,12 @@ def handle_POST(self): else: time_wait = 600 - username, roles = self.query_back_for_user_and_permissions(payload['Authorization'][0], sslVerify=sslVerify) + if 'requestingAddress' in payload: + requestingAddress = payload['requestingAddress'][0] + else: + requestingAddress = None + + username, roles = self.query_back_for_user_and_permissions(requestingAddress, payload['Authorization'][0], sslVerify=sslVerify) logger.info("username=%s roles=%s" % (username, roles)) app = payload['app'][0] @@ -207,7 +215,7 @@ def handle_POST(self): headers = { "Authorization" : "Splunk " + self.request['systemAuth'] } curtime = calendar.timegm(time.gmtime()) url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status" - res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=sslVerify) + res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=False) if not res: return @@ -216,7 +224,7 @@ def handle_POST(self): if not len(res) == 0: if not 'start_time' in res[0]: logger.warn("Warning invalid kvstore data, will wipe it and continue in collection splunkversioncontrol_rest_restore_status on url=%s, value returned res=\"%s\"" % (url, payload)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=sslVerify) + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=False) else: kvstore_start_time = res[0]['start_time'] target_time = curtime - time_wait diff --git a/bin/splunkversioncontrol_utility.py b/bin/splunkversioncontrol_utility.py index 55689f3..0ec0b57 100644 --- a/bin/splunkversioncontrol_utility.py +++ b/bin/splunkversioncontrol_utility.py @@ -55,7 +55,7 @@ def get_password(password, session_key, logger, *, sslVerify=False): url = "https://localhost:8089/servicesNS/-/" + context + "/storage/passwords?output_mode=json&f=clear_password&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) headers = {'Authorization': 'Splunk %s' % session_key} - res = requests.get(url, headers=headers, verify=sslVerify) + res = requests.get(url, headers=headers, verify=False) dict = json.loads(res.text) clear_password = False if not 'entry' in dict: @@ -73,7 +73,7 @@ def get_password(password, session_key, logger, *, sslVerify=False): url = "https://localhost:8089/servicesNS/-/-/storage/passwords?output_mode=json&f=clear_password&count=0&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) - res = requests.get(url, headers=headers, verify=sslVerify) + res = requests.get(url, headers=headers, verify=False) dict = json.loads(res.text) if not 'entry' in dict: logger.warn("dict=%s did not contain the entries expected on url=%s while looking for password=%s" % (dict, url, password)) From 99571a77eec98c498a67f7b848a3349533387406 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:09:44 +0100 Subject: [PATCH 12/14] Add additional macro --- .../data/ui/views/splunkversioncontrol_restore_dynamic.xml | 2 +- default/macros.conf | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml index 9a961a4..839c909 100644 --- a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml +++ b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml @@ -129,7 +129,7 @@ SplunkVersionControl Results (please wait a few minutes for results) - | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` sslVerify=`sslVerify` + | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` sslVerify=`sslVerify` requestingAddress=`requestingAddress` -5m now 1 diff --git a/default/macros.conf b/default/macros.conf index 37089d5..31c7670 100644 --- a/default/macros.conf +++ b/default/macros.conf @@ -22,3 +22,8 @@ iseval = 0 [sslVerify] definition = false iseval = 0 + +[requestingAddress] +definition = false +iseval = 0 + From 52d18d2d49cc0656329d85c6237e62c6830f4f17 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:10:31 +0100 Subject: [PATCH 13/14] Add validation and disable SSL for localhost --- bin/postversioncontrolrestore.py | 3 ++- bin/splunkversioncontrol_rest_restore.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index d34049f..f6ca563 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -116,7 +116,8 @@ def generate(self): body['restoreAsUser'] = self.restoreAsUser body['scope'] = self.scope body['timeout'] = self.timeout - body['requestingAddress'] = self.requestingAddress + if self.requestingAddress: + body['requestingAddress'] = self.requestingAddress logger.info("Attempting POST request to url=%s with body=\"%s\"" % (url, body)) diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 5a0a968..9869278 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -231,7 +231,7 @@ def handle_POST(self): if kvstore_start_time < target_time: logger.warn("Found existing entry from %s but time is %s, this is past the limit of current time minus %s (%s)" % (kvstore_start_time, curtime, time_wait, target_time)) #More than 10 minutes ago, delete the entry and move on - self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=sslVerify) + self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=False) else: removal_target = kvstore_start_time + time_wait + 1 logger.warn("Attempted to run but found a running restore instance with time=%s and current_time=%s, will delete and move on after current_time_minus=%s seconds (override_time=%s)" % (kvstore_start_time, curtime, time_wait, removal_target)) From 5bd5aaabf35deac3af8cbced13f1478a0555a642 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:18:38 +0100 Subject: [PATCH 14/14] Remove validation for localhost --- bin/splunkversioncontrol_rest_restore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 9869278..3142aa2 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -242,7 +242,7 @@ def handle_POST(self): payload = json.dumps({ 'start_time': curtime }) headers['Content-Type'] = 'application/json' #update kvstore with runtime - res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=sslVerify) + res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=False) if not res: return res @@ -254,7 +254,7 @@ def handle_POST(self): self.response.write("Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) logger.warn("Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=sslVerify) + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=False) #Run a Splunk query via the search/jobs endpoint def runSearchJob(self, url, appname, headers, auth, username, earliest_time, *, sslVerify=False):