From 1ede611491931574e3b5fd96933e0557e707f315 Mon Sep 17 00:00:00 2001 From: me Date: Thu, 10 Dec 2020 21:35:51 +0100 Subject: [PATCH 01/17] Add sslVerify variable --- README/inputs.conf.spec | 5 ++ bin/postversioncontrolrestore.py | 3 +- bin/splunkversioncontrol_backup.py | 13 +++- bin/splunkversioncontrol_backup_class.py | 18 ++--- bin/splunkversioncontrol_rest_restore.py | 77 ++++++++++--------- bin/splunkversioncontrol_restore.py | 14 +++- bin/splunkversioncontrol_restore_class.py | 24 +++--- bin/splunkversioncontrol_utility.py | 6 +- .../splunkversioncontrol_restore_dynamic.xml | 2 +- default/macros.conf | 4 + 10 files changed, 103 insertions(+), 63 deletions(-) diff --git a/README/inputs.conf.spec b/README/inputs.conf.spec index 868421f..4d80549 100644 --- a/README/inputs.conf.spec +++ b/README/inputs.conf.spec @@ -9,6 +9,8 @@ gitTempDir = * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist) gitRepoURL = * git repository URL to store the objects (SSH URL only) +sslVerify = +* Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) noPrivate = * disable the backup of user level / private objects (true/false), default false noDisabled = @@ -47,6 +49,9 @@ gitTempDir = * location where to store the output of the script on the filesystem (note this directory will be deleted/re-created but the parent dir must exist) gitRepoURL = * git repository URL to store the objects (SSH URL only) +sslVerify = +* Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) + auditLogsLookupBackTime = * This is how far back the audit logs will be checked to ensure that a restore entry is valid, this should be set to your interval time or slightly more, defaults to -1h (use Splunk format) debugMode = diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index 18afd92..7ff5d3b 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -60,6 +60,7 @@ class SVCPostRestore(GeneratingCommand): restoreAsUser = Option(require=True) scope = Option(require=True) timeout = Option(require=True) + sslVerify = Option(require=False, default=False) def generate(self): """ @@ -94,7 +95,7 @@ def generate(self): logger.debug("Using token %s" % (body['Authorization'])) - attempt = requests.post(url, verify=False, data=body) + attempt = requests.post(url, verify=self.sslVerify, data=body) if attempt.status_code != 200: logger.error("POST request failed with status_code=%s, reason=%s, text=%s on url=%s" % (attempt.status_code, attempt.reason, attempt.text, url)) yield {'result': 'Unknown failure, received a non-200 response code of %s on the url %s, reason %s, text result is %s' % (attempt.status_code, url, attempt.reason, attempt.text)} diff --git a/bin/splunkversioncontrol_backup.py b/bin/splunkversioncontrol_backup.py index b795ac7..96479b2 100644 --- a/bin/splunkversioncontrol_backup.py +++ b/bin/splunkversioncontrol_backup.py @@ -47,6 +47,12 @@ gitRepoURL git repository URL to store the objects (SSH URL only) + + sslVerify + Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) + is_bool('sslVerify') + false + noPrivate disable the backup of user level / private objects (true/false), default false @@ -203,10 +209,13 @@ def validate_arguments(): else: ssh_command = "ssh" + sslVerify = False + if 'sslVerify' in val_data: + sslVerify = val_data['sslVerify'] + #Run a sanity check and make sure we can connect into the remote Splunk instance if not useLocalAuth: url = val_data['srcURL'] + "/servicesNS/nobody/%s/search/jobs/export?search=makeresults" % (appName) - #Verify=false is hardcoded to workaround local SSL issues srcUsername = val_data['srcUsername'] srcPassword = val_data['srcPassword'] if srcPassword.find("password:") == 0: @@ -224,7 +233,7 @@ def validate_arguments(): try: logger.debug("Running query against URL %s with username %s proxies_length=%s" % (url, srcUsername, len(proxies))) - res = requests.get(url, auth=(srcUsername, srcPassword), verify=False, proxies=proxies) + res = requests.get(url, auth=(srcUsername, srcPassword), verify=self.sslVerify, proxies=proxies) logger.debug("End query against URL %s with username %s" % (url, srcUsername)) if (res.status_code != requests.codes.ok): diff --git a/bin/splunkversioncontrol_backup_class.py b/bin/splunkversioncontrol_backup_class.py index 74254be..688d6ed 100644 --- a/bin/splunkversioncontrol_backup_class.py +++ b/bin/splunkversioncontrol_backup_class.py @@ -77,6 +77,7 @@ class SplunkVersionControlBackup: gitRepoURL = None stanzaName = None lastRunEpoch = None + sslVerify = False # read XML configuration passed from splunkd def get_config(self): @@ -136,8 +137,7 @@ def getAllAppsList(self): else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.fatal("i=\"%s\" Could not obtain a list of all apps, URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text)) sys.exit(-1) @@ -190,8 +190,7 @@ def runQueries(self, app, endpoint, type, fieldIgnoreList, aliasAttributes={}, v else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" URL=%s in app=%s statuscode=%s reason=%s response=\"%s\"" % (self.stanzaName, url, app, res.status_code, res.reason, res.text)) @@ -433,8 +432,7 @@ def macros(self, app): else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" Type macro in app=%s, URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, app, url, res.status_code, res.reason, res.text)) @@ -749,7 +747,7 @@ def runSearchJob(self, query): headers = {'Authorization': 'Splunk %s' % self.session_key } else: auth = HTTPBasicAuth(self.srcUsername, self.srcPassword) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=data, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=data, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" URL=%s statuscode=%s reason=%s response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text)) res = json.loads(res.text) @@ -1022,6 +1020,9 @@ def run_script(self): self.proxies = proxies + if 'sslVerify' in config: + self.sslVerify = config['sslVerify'] + #From server self.splunk_rest = config['srcURL'] excludedList = [ "srcPassword", "session_key" ] @@ -1035,9 +1036,8 @@ def run_script(self): headers={'Authorization': 'Splunk %s' % config['session_key']} - #Verify=false is hardcoded to workaround local SSL issues url = 'https://localhost:8089/services/shcluster/captain/info?output_mode=json' - res = requests.get(url, headers=headers, verify=False) + res = requests.get(url, headers=headers, verify=self.sslVerify) if (res.status_code == 503): logger.debug("i=\"%s\" Non-shcluster / standalone instance, safe to run on this node" % (self.stanzaName)) elif (res.status_code != requests.codes.ok): diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index ac2563d..987eb9d 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -59,6 +59,26 @@ class SVCRestore(splunk.rest.BaseRestHandler): + def query_back_for_user_and_permissions(self, authorization_token, remoteAddr,*, sslVerify): + headers = { "Authorization" : authorization_token } + + #Run a query back against the source system to check the username/role + remoteAddr = self.request['remoteAddr'] + url = "https://" + remoteAddr + ":8089/services/authentication/current-context?output_mode=json" + logger.info("Received remote request checking username and role related to the token on url=%s" % (url)) + logger.debug("token=%s" % (authorization_token)) + + res = self.runHttpRequest(url, headers, None, "get", "checking username with token (based on incoming ip address)", sslVerify=sslVerify) + if not res: + return + + json_dict = json.loads(res.text) + #self.response.write(str(json_dict) + "\n\n\n") + username = json_dict['entry'][0]['content']['username'] + roles = json_dict['entry'][0]['content']['roles'] + return username, roles + + def handle_POST(self): starttime = calendar.timegm(time.gmtime()) payload = six.moves.urllib.parse.parse_qs(self.request['payload']) @@ -75,26 +95,6 @@ def handle_POST(self): self.response.write("Received remote call but attr=%s was missing from arguments, received=\"%s\"" % (attr, payload)) return - headers = { "Authorization" : payload['Authorization'][0] } - - #Run a query back against the source system to check the username/role - remoteAddr = self.request['remoteAddr'] - url = "https://" + remoteAddr + ":8089/services/authentication/current-context?output_mode=json" - logger.info("Received remote request checking username and role related to the token on url=%s" % (url)) - logger.debug("token=%s" % (payload['Authorization'][0])) - - res = self.runHttpRequest(url, headers, None, "get", "checking username with token (based on incoming ip address)") - if not res: - return - - json_dict = json.loads(res.text) - #self.response.write(str(json_dict) + "\n\n\n") - username = json_dict['entry'][0]['content']['username'] - roles = json_dict['entry'][0]['content']['roles'] - - #self.response.write(username + "\n") - #self.response.write(str(roles) + "\n") - logger.info("username=%s roles=%s" % (username, roles)) splunk_vc_name = payload['splunk_vc_name'][0] #self.response.write(splunk_vc_name + "\n\n\n") @@ -104,7 +104,7 @@ def handle_POST(self): url = "https://localhost:8089/servicesNS/-/-/data/inputs/splunkversioncontrol_restore/" + six.moves.urllib.parse.quote(splunk_vc_name) + "?output_mode=json" logger.debug("Now running query against url=%s to obtain config information" % (url)) - res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name)) + res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name), sslVerify=sslVerify) if not res: return @@ -134,6 +134,10 @@ def handle_POST(self): destURL = json_dict['destURL'] + sslVerify = False + if 'sslVerify' in json_dict: + sslVerify = json_dict['sslVerify'] + headers = {} auth = None @@ -156,6 +160,9 @@ def handle_POST(self): else: time_wait = 600 + username, roles = self.query_back_for_user_and_permissions(payload['Authorization'][0], remoteAddr) + logger.info("username=%s roles=%s" % (username, roles)) + app = payload['app'][0] type = payload['type'][0] obj_name = payload['obj_name'][0] @@ -178,7 +185,7 @@ def handle_POST(self): starttime = starttime-60-timeout - json_res = self.runSearchJob(destURL, remoteAppName, headers, auth, username, starttime) + json_res = self.runSearchJob(destURL, remoteAppName, headers, auth, username, starttime, sslVerify=sslVerify) if 'error' in json_res: self.response.write("An error occurred: %s" % (json_res['error'])) @@ -200,7 +207,7 @@ def handle_POST(self): headers = { "Authorization" : "Splunk " + self.request['systemAuth'] } curtime = calendar.timegm(time.gmtime()) url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status" - res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status") + res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=sslVerify) if not res: return @@ -209,14 +216,14 @@ def handle_POST(self): if not len(res) == 0: if not 'start_time' in res[0]: logger.warn("Warning invalid kvstore data, will wipe it and continue in collection splunkversioncontrol_rest_restore_status on url=%s, value returned res=\"%s\"" % (url, payload)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status') + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=sslVerify) else: kvstore_start_time = res[0]['start_time'] target_time = curtime - time_wait if kvstore_start_time < target_time: logger.warn("Found existing entry from %s but time is %s, this is past the limit of current time minus %s (%s)" % (kvstore_start_time, curtime, time_wait, target_time)) #More than 10 minutes ago, delete the entry and move on - self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time)) + self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=sslVerify) else: removal_target = kvstore_start_time + time_wait + 1 logger.warn("Attempted to run but found a running restore instance with time=%s and current_time=%s, will delete and move on after current_time_minus=%s seconds (override_time=%s)" % (kvstore_start_time, curtime, time_wait, removal_target)) @@ -227,7 +234,7 @@ def handle_POST(self): payload = json.dumps({ 'start_time': curtime }) headers['Content-Type'] = 'application/json' #update kvstore with runtime - res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status') + res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=sslVerify) if not res: return res @@ -239,16 +246,16 @@ def handle_POST(self): self.response.write("Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) logger.warn("Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run') + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=sslVerify) #Run a Splunk query via the search/jobs endpoint - def runSearchJob(self, url, appname, headers, auth, username, earliest_time): + def runSearchJob(self, url, appname, headers, auth, username, earliest_time, *, sslVerify=False): url = url + "/servicesNS/-/%s/search/jobs" % (appname) query = "savedsearch \"Splunk Version Control Audit Query POST\" username=\"%s\" | stats count | where count>0" % (username) logger.debug("Running requests.post() on url=%s query=\"%s\"" % (url, query)) data = { "search" : query, "output_mode" : "json", "exec_mode" : "oneshot", "earliest_time" : earliest_time } - res = requests.post(url, auth=auth, headers=headers, verify=False, data=data) + res = requests.post(url, auth=auth, headers=headers, verify=sslVerify, data=data) if (res.status_code != requests.codes.ok): logger.error("url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text)) return { "error": "url=%s status_code=%s reason=%s, response=\"%s\"" % (url, res.status_code, res.reason, res.text) } @@ -264,17 +271,17 @@ def runSearchJob(self, url, appname, headers, auth, username, earliest_time): logger.warn("messages from query=\"%s\" were messages=\"%s\"" % (query, res["messages"])) return res - def runHttpRequest(self, url, headers, data, type, text): + def runHttpRequest(self, url, headers, data, type, text, *, sslVerify=False): if type == "delete": - res = requests.delete(url, headers=headers, verify=False) + res = requests.delete(url, headers=headers, verify=sslVerify) elif type == "post": - res = requests.post(url, headers=headers, verify=False, data=data) + res = requests.post(url, headers=headers, verify=sslVerify, data=data) elif type == "get": - res = requests.get(url, headers=headers, verify=False) + res = requests.get(url, headers=headers, verify=sslVerify) if (res.status_code != requests.codes.ok and res.status_code != 201): - logger.error("Unexpected response code while %s, on url=%s, statuscode=%s reason=%s, response=\"%s\", payload=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data)) - self.response.write("Error unexpected response code while %s, on url %s, statuscode %s reason %s, response \"%s\", payload=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data)) + logger.error("Unexpected response code while %s, on url=%s, statuscode=%s reason=%s, response=\"%s\", payload=\"%s\", verify=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data, sslVerify)) + self.response.write("Error unexpected response code while %s, on url %s, statuscode %s reason %s, response \"%s\", payload=\"%s\", verify=\"%s\"" % (text, url, res.status_code, res.reason, res.text, data, sslVerify)) return return res diff --git a/bin/splunkversioncontrol_restore.py b/bin/splunkversioncontrol_restore.py index 0e31ef7..6c490c1 100644 --- a/bin/splunkversioncontrol_restore.py +++ b/bin/splunkversioncontrol_restore.py @@ -49,6 +49,13 @@ gitRepoURL git repository URL to store the objects (SSH URL only) + + sslVerify + Set to 'true' or 'false' to enable/disable SSL verification for REST requests to `srcUrl`. Set to a path to specify a file with valid CA. (https://2.python-requests.org/en/master/user/advanced/#ssl-cert-verification) + is_bool('sslVerify') + false + + auditLogsLookupBackTime This is how far back the audit logs will be checked to ensure that a restore entry is valid, this should be set to your interval time or slightly more, defaults to -1h (use Splunk format) @@ -176,12 +183,15 @@ def validate_arguments(): print_error("Unable to convert timeout field to a valid value, this must be an integer value in seconds, value provided was %s" % (val_data['timewait'])) sys.exit(1) + sslVerify = False + if 'sslVerify' in val_data: + sslVerify = val_data['sslVerify'] + session_key = val_data['session_key'] #Run a sanity check and make sure we can connect into the remote Splunk instance if not useLocalAuth: url = val_data['destURL'] + "/servicesNS/nobody/%s/search/jobs/export?search=makeresults" % (appName) - #Verify=false is hardcoded to workaround local SSL issues destUsername = val_data['destUsername'] destPassword = val_data['destPassword'] @@ -199,7 +209,7 @@ def validate_arguments(): try: logger.debug("Running query against URL %s with username %s proxies_length=%s" % (url, destUsername, len(proxies))) - res = requests.get(url, auth=(destUsername, destPassword), verify=False, proxies=proxies) + res = requests.get(url, auth=(destUsername, destPassword), verify=sslVerify, proxies=proxies) logger.debug("End query against URL %s with username %s" % (url, destUsername)) if (res.status_code != requests.codes.ok): print_error("Attempt to validate access to Splunk failed with code %s, reason %s, text %s on URL %s" % (res.status_code, res.reason, res.text, url)) diff --git a/bin/splunkversioncontrol_restore_class.py b/bin/splunkversioncontrol_restore_class.py index 5ab04ad..7cce8ff 100644 --- a/bin/splunkversioncontrol_restore_class.py +++ b/bin/splunkversioncontrol_restore_class.py @@ -69,6 +69,7 @@ class SplunkVersionControlRestore: appName = "SplunkVersionControl" gitRepoURL = None stanzaName = None + sslVerify = False # read XML configuration passed from splunkd def get_config(self): @@ -161,7 +162,7 @@ def runQueries(self, app, endpoint, type, name, scope, user, restoreAsUser, admi res_result = False #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) objExists = False #If we get 404 it definitely does not exist or it has a name override @@ -348,7 +349,7 @@ def runRestore(self, config, type, endpoint, app, name, user, restoreAsUser, adm objExistsURL = "%s/%s?output_mode=json" % (url, origName) logger.debug("i=\"%s\" URL=%s re-checking object exists URL due to name override from %s to original name of %s proxies_length=%s" % (self.stanzaName, objExistsURL, name, origName, len(self.proxies))) #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(objExistsURL, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(objExistsURL, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) #If we get 404 it definitely does not exist or it has a name override if (res.status_code == 404): @@ -399,14 +400,14 @@ def runRestore(self, config, type, endpoint, app, name, user, restoreAsUser, adm del payload['disabled'] logger.debug("i=\"%s\" Attempting to %s type=%s with name=%s on URL=%s with payload=\"%s\" in app=%s proxies_length=%s" % (self.stanzaName, createOrUpdate, type, name, url, payload, app, len(self.proxies))) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): logger.error("i=\"%s\" user=%s, name=%s of type=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", in app=%s, owner=%s" % (self.stanzaName, user, name, type, url, res.status_code, res.reason, res.text, app, owner)) #Saved Searches sometimes fail due to the VSID field, auto-retry in case that solves the problem... if type=="savedsearches": if 'vsid' in payload: del payload['vsid'] - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): logger.error("i=\"%s\" user=%s, re-attempted without vsid but result for name=%s of type=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", in app=%s, owner=%s" % (self.stanzaName, user, name, type, url, res.status_code, res.reason, res.text, app, owner)) result = False @@ -442,7 +443,7 @@ def runRestore(self, config, type, endpoint, app, name, user, restoreAsUser, adm url = "%s/acl" % (objURL) payload = { "owner": owner, "sharing" : sharing } logger.info("i=\"%s\" Attempting to change ownership of type=%s with name=%s via URL=%s to owner=%s in app=%s with sharing=%s" % (self.stanzaName, type, name, url, owner, app, sharing)) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) #If re-own fails log this for investigation if (res.status_code != requests.codes.ok): @@ -502,7 +503,7 @@ def runRestoreMacro(self, config, app, name, username, restoreAsUser, adminLevel #servicesNS/-/search/properties/macros #__stanza = - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): message = "name=%s of type=macro in app=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", owner=%s" % (name, app, url, res.status_code, res.reason, res.text, owner) logger.error("i=\"" + self.stanzaName + "\"" + message) @@ -529,7 +530,7 @@ def runRestoreMacro(self, config, app, name, username, restoreAsUser, adminLevel payload = config logger.debug("i=\"%s\" Attempting to modify type=macro name=%s on URL=%s with payload=\"%s\" in app=%s proxies_length=%s" % (self.stanzaName, name, url, payload, app, len(self.proxies))) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok and res.status_code != 201): logger.error("i=\"%s\" name=%s of type=macro in app=%s with URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, name, app, url, res.status_code, res.reason, res.text)) result = False @@ -538,7 +539,7 @@ def runRestoreMacro(self, config, app, name, username, restoreAsUser, adminLevel url = "%s/servicesNS/%s/%s/configs/conf-macros/%s/acl" % (self.splunk_rest, owner, app, name) payload = { "owner": owner, "sharing" : sharing } logger.info("i=\"%s\" Attempting to change ownership of type=macro name=%s via URL=%s to owner=%s in app=%s with sharing=%s" % (self.stanzaName, name, url, owner, app, sharing)) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=payload, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=payload, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" name=%s of type=macro in app=%s with URL=%s statuscode=%s reason=%s, response=\"%s\", owner=%s sharing=%s" % (self.stanzaName, name, app, url, res.status_code, res.reason, res.text, owner, sharing)) else: @@ -581,7 +582,7 @@ def macros(self, app, name, scope, user, restoreAsUser, adminLevel): auth = HTTPBasicAuth(self.destUsername, self.destPassword) #Verify=false is hardcoded to workaround local SSL issues - res = requests.get(url, auth=auth, headers=headers, verify=False, proxies=self.proxies) + res = requests.get(url, auth=auth, headers=headers, verify=self.sslVerify, proxies=self.proxies) objExists = False if (res.status_code == 404): logger.debug("i=\"%s\" URL=%s is throwing a 404, assuming new object creation" % (self.stanzaName, url)) @@ -843,7 +844,7 @@ def runSearchJob(self, query, earliest_time="-1h"): else: auth = HTTPBasicAuth(self.destUsername, self.destPassword) - res = requests.post(url, auth=auth, headers=headers, verify=False, data=data, proxies=self.proxies) + res = requests.post(url, auth=auth, headers=headers, verify=self.sslVerify, data=data, proxies=self.proxies) if (res.status_code != requests.codes.ok): logger.error("i=\"%s\" URL=%s statuscode=%s reason=%s, response=\"%s\"" % (self.stanzaName, url, res.status_code, res.reason, res.text)) res = json.loads(res.text) @@ -949,6 +950,9 @@ def run_script(self, restlist_override=None, config_override=None): self.proxies = proxies + if 'sslVerify' in config: + self.sslVerify = config['sslVerify'] + dirExists = os.path.isdir(self.gitTempDir) if dirExists and len(os.listdir(self.gitTempDir)) != 0: if not ".git" in os.listdir(self.gitTempDir): diff --git a/bin/splunkversioncontrol_utility.py b/bin/splunkversioncontrol_utility.py index bed452a..689ec15 100644 --- a/bin/splunkversioncontrol_utility.py +++ b/bin/splunkversioncontrol_utility.py @@ -36,7 +36,7 @@ def runOSProcess(command, logger, timeout=60, shell=False): return str(stdout), str(stderr), res # use the password endpoint to obtain the clear_password passed in, start with the context of this app and then try all contexts -def get_password(password, session_key, logger): +def get_password(password, session_key, logger, *, sslVerify=False): #TODO move this into shared function to obtain passwords: context = os.path.dirname(os.path.dirname(__file__)) @@ -55,7 +55,7 @@ def get_password(password, session_key, logger): url = "https://localhost:8089/servicesNS/-/" + context + "/storage/passwords?output_mode=json&f=clear_password&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) headers = {'Authorization': 'Splunk %s' % session_key} - res = requests.get(url, headers=headers, verify=False) + res = requests.get(url, headers=headers, verify=sslVerify) dict = json.loads(res.text) clear_password = False if not 'entry' in dict: @@ -73,7 +73,7 @@ def get_password(password, session_key, logger): url = "https://localhost:8089/servicesNS/-/-/storage/passwords?output_mode=json&f=clear_password&count=0&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) - res = requests.get(url, headers=headers, verify=False) + res = requests.get(url, headers=headers, verify=verify) dict = json.loads(res.text) if not 'entry' in dict: logger.warn("dict=%s did not contain the entries expected on url=%s while looking for password=%s" % (dict, url, password)) diff --git a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml index 977a12a..9a961a4 100644 --- a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml +++ b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml @@ -129,7 +129,7 @@ SplunkVersionControl Results (please wait a few minutes for results) - | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` + | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` sslVerify=`sslVerify` -5m now 1 diff --git a/default/macros.conf b/default/macros.conf index fe69706..37089d5 100644 --- a/default/macros.conf +++ b/default/macros.conf @@ -18,3 +18,7 @@ iseval = 0 [splunk_vc_timeout] definition = 30 iseval = 0 + +[sslVerify] +definition = false +iseval = 0 From ab9c64b3759824e622350c070845c9bff4c3277c Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Thu, 11 Feb 2021 22:58:28 +0100 Subject: [PATCH 02/17] Add test environment setup --- docker-compose.yml | 63 ++++++++++++++++++++++++++++++++++++++++++++ test/bkp_default.yml | 29 ++++++++++++++++++++ test/custom_init.sh | 15 +++++++++++ test/sh_default.yml | 15 +++++++++++ 4 files changed, 122 insertions(+) create mode 100644 docker-compose.yml create mode 100644 test/bkp_default.yml create mode 100755 test/custom_init.sh create mode 100644 test/sh_default.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d345b67 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,63 @@ +version: "3.7" +networks: + splunknet: + driver: bridge + attachable: true + +services: + splunk_sh: + networks: + splunknet: + aliases: + - splunk_sh + hostname: splunk_sh + container_name: splunk_sh + image: splunk/splunk:latest + ports: + - 8003:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + volumes: + - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups + - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static + + splunk_moc: + networks: + splunknet: + aliases: + - splunk_moc + hostname: splunk_moc + container_name: splunk_moc + image: splunk/splunk:latest + entrypoint: + - /bin/sh + - /usr/sbin/custom_init.sh + ports: + - 8004:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + - SPLUNK_ROLE=splunk_monitor + - DEBUG=true + + volumes: + - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups + - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ./test/bkp_default.yml:/tmp/defaults/default.yml + - ./test/install_git.yml:/tmp/install_git.yml + - ./test/custom_init.sh:/usr/sbin/custom_init.sh diff --git a/test/bkp_default.yml b/test/bkp_default.yml new file mode 100644 index 0000000..01911ff --- /dev/null +++ b/test/bkp_default.yml @@ -0,0 +1,29 @@ +--- +#ansible_post_tasks: file:///tmp/install_git.yml +splunk: + conf: + - key: inputs + value: + directory: /opt/splunk/etc/apps/SplunkVersionControl/local + content: + splunkversioncontrol_backup://SH: + srcURL: https://splunk_sh:8089 + srcUsername: admin + srcPassword: password:SH + gitTempDir: /opt/splunk/vcs/git_tmp_backup + gitRepoURL: file:///opt/splunk/vcs/backup.git + # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + debugMode: true + useLocalAuth: false + interval: 60 + splunkversioncontrol_restore://SH: + destURL: https://splunk_sh:8089 + destUsername: admin + destPassword: password:SH + gitTempDir: /opt/splunk/vcs/git_tmp_restore + gitRepoURL: file:///opt/splunk/vcs/backup.git + # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + debugMode: true + useLocalAuth: false + auditLogsLookupBackTime: -2h + timewait: 30 diff --git a/test/custom_init.sh b/test/custom_init.sh new file mode 100755 index 0000000..d5ac403 --- /dev/null +++ b/test/custom_init.sh @@ -0,0 +1,15 @@ +set -e +sudo microdnf install git +sudo -u splunk mkdir -p /opt/splunk/vcs/{git_tmp_backup,git_tmp_restore} +sudo -u splunk mkdir -p /opt/splunk/vcs/backup.git +echo "dirs" +sudo -u splunk sh -c "cd /opt/splunk/vcs/backup.git && git init --bare --shared" +echo "init" +sudo -u splunk sh -c "cd /opt/splunk/vcs/ && git clone backup.git" +sudo -u splunk sh -c "git config --global user.email 'backup@example.local'" +sudo -u splunk sh -c "git config --global user.name 'BackupUser'" +echo "clone" +sudo -u splunk sh -c 'cd /opt/splunk/vcs/backup && touch test && git add test && git commit -a -m "init" && git push' +echo "file" + +/usr/sbin/entrypoint.sh start diff --git a/test/sh_default.yml b/test/sh_default.yml new file mode 100644 index 0000000..cc23d8c --- /dev/null +++ b/test/sh_default.yml @@ -0,0 +1,15 @@ +--- +#ansible_post_tasks: file:///tmp/install_git.yml +splunk: + conf: + - key: macros + value: + directory: /opt/splunk/etc/apps/SplunkVersionControl/local + content: + splunk_vc_name: + definition: SH + splunk_vc_url: + definition: https://splunk_moc:8089 + sslVerify: + definiton: false + # definition: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem From 5a910f5b674fc9e637a5375cda7abd70eea0f516 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Thu, 11 Feb 2021 23:10:49 +0100 Subject: [PATCH 03/17] Add check when rerunning same container --- test/custom_init.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/test/custom_init.sh b/test/custom_init.sh index d5ac403..05d3e4f 100755 --- a/test/custom_init.sh +++ b/test/custom_init.sh @@ -2,14 +2,11 @@ set -e sudo microdnf install git sudo -u splunk mkdir -p /opt/splunk/vcs/{git_tmp_backup,git_tmp_restore} sudo -u splunk mkdir -p /opt/splunk/vcs/backup.git -echo "dirs" sudo -u splunk sh -c "cd /opt/splunk/vcs/backup.git && git init --bare --shared" -echo "init" +if [ ! -d "/opt/splunk/vcs/backup.git" ]; then sudo -u splunk sh -c "cd /opt/splunk/vcs/ && git clone backup.git" sudo -u splunk sh -c "git config --global user.email 'backup@example.local'" sudo -u splunk sh -c "git config --global user.name 'BackupUser'" -echo "clone" sudo -u splunk sh -c 'cd /opt/splunk/vcs/backup && touch test && git add test && git commit -a -m "init" && git push' -echo "file" - +fi /usr/sbin/entrypoint.sh start From 4eeeb2b4001651d62c3bb7840a722903a0508c74 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Thu, 11 Feb 2021 23:11:50 +0100 Subject: [PATCH 04/17] Add credentials --- test/bkp_default.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/bkp_default.yml b/test/bkp_default.yml index 01911ff..b2afbcc 100644 --- a/test/bkp_default.yml +++ b/test/bkp_default.yml @@ -9,7 +9,7 @@ splunk: splunkversioncontrol_backup://SH: srcURL: https://splunk_sh:8089 srcUsername: admin - srcPassword: password:SH + srcPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_backup gitRepoURL: file:///opt/splunk/vcs/backup.git # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem @@ -19,7 +19,7 @@ splunk: splunkversioncontrol_restore://SH: destURL: https://splunk_sh:8089 destUsername: admin - destPassword: password:SH + destPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_restore gitRepoURL: file:///opt/splunk/vcs/backup.git # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem From 4e5440cc9b3b7e005749b7b52382df92f9810a08 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:16:23 +0100 Subject: [PATCH 05/17] Add macro config --- docker-compose.yml | 1 + test/sh_default.yml | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d345b67..33dcd74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,7 @@ services: - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ./test/sh_default.yml:/tmp/defaults/default.yml splunk_moc: networks: diff --git a/test/sh_default.yml b/test/sh_default.yml index cc23d8c..8df1e55 100644 --- a/test/sh_default.yml +++ b/test/sh_default.yml @@ -1,5 +1,4 @@ --- -#ansible_post_tasks: file:///tmp/install_git.yml splunk: conf: - key: macros @@ -9,7 +8,7 @@ splunk: splunk_vc_name: definition: SH splunk_vc_url: - definition: https://splunk_moc:8089 + definition: https://splunk_moc:8089/services/splunkversioncontrol_rest_restore sslVerify: definiton: false # definition: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem From 12d9194fb4f3670665108f18b34896766b85740b Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:18:39 +0100 Subject: [PATCH 06/17] Add validation and fix variable bugs --- bin/postversioncontrolrestore.py | 22 +++++++++++++++++++++- bin/splunkversioncontrol_rest_restore.py | 6 +++--- bin/splunkversioncontrol_utility.py | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index 7ff5d3b..73cd511 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -10,8 +10,28 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import dispatch, GeneratingCommand, Configuration, Option +from splunklib.searchcommands.validators import Validator, Boolean, File from splunklib.binding import HTTPError +class OrValidator(Validator): + def __init__(self, a, b): + self.a = a + self.b = b + def __call__(self, value): + """Returns b if a raises an exception otherwise a.""" + try: + return self.a.__call__(value) + except ValueError: + return self.b.__call__(value) + + def format(self, value): + """Returns b if a raises an exception otherwise a.""" + try: + return self.a.format(value) + except: + return self.b.format(value) + + splunkLogsDir = os.environ['SPLUNK_HOME'] + "/var/log/splunk" #Setup the logging logging_config = dict( @@ -60,7 +80,7 @@ class SVCPostRestore(GeneratingCommand): restoreAsUser = Option(require=True) scope = Option(require=True) timeout = Option(require=True) - sslVerify = Option(require=False, default=False) + sslVerify = Option(require=False, default=False, validate=OrValidator(File(), Boolean())) def generate(self): """ diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 987eb9d..2b76e72 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -59,7 +59,7 @@ class SVCRestore(splunk.rest.BaseRestHandler): - def query_back_for_user_and_permissions(self, authorization_token, remoteAddr,*, sslVerify): + def query_back_for_user_and_permissions(self, authorization_token, *, sslVerify): headers = { "Authorization" : authorization_token } #Run a query back against the source system to check the username/role @@ -104,7 +104,7 @@ def handle_POST(self): url = "https://localhost:8089/servicesNS/-/-/data/inputs/splunkversioncontrol_restore/" + six.moves.urllib.parse.quote(splunk_vc_name) + "?output_mode=json" logger.debug("Now running query against url=%s to obtain config information" % (url)) - res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name), sslVerify=sslVerify) + res = self.runHttpRequest(url, headers, None, "get", "querying the inputs for splunkversioncontrol_restore with name %s" % (splunk_vc_name), sslVerify=False) if not res: return @@ -160,7 +160,7 @@ def handle_POST(self): else: time_wait = 600 - username, roles = self.query_back_for_user_and_permissions(payload['Authorization'][0], remoteAddr) + username, roles = self.query_back_for_user_and_permissions(payload['Authorization'][0], sslVerify=sslVerify) logger.info("username=%s roles=%s" % (username, roles)) app = payload['app'][0] diff --git a/bin/splunkversioncontrol_utility.py b/bin/splunkversioncontrol_utility.py index 689ec15..55689f3 100644 --- a/bin/splunkversioncontrol_utility.py +++ b/bin/splunkversioncontrol_utility.py @@ -73,7 +73,7 @@ def get_password(password, session_key, logger, *, sslVerify=False): url = "https://localhost:8089/servicesNS/-/-/storage/passwords?output_mode=json&f=clear_password&count=0&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) - res = requests.get(url, headers=headers, verify=verify) + res = requests.get(url, headers=headers, verify=sslVerify) dict = json.loads(res.text) if not 'entry' in dict: logger.warn("dict=%s did not contain the entries expected on url=%s while looking for password=%s" % (dict, url, password)) From ac872be4ffd4fedf41a4ba937955dcc9d4b5b4a2 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:38:41 +0100 Subject: [PATCH 07/17] Fix recreation check --- test/custom_init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/custom_init.sh b/test/custom_init.sh index 05d3e4f..36d5336 100755 --- a/test/custom_init.sh +++ b/test/custom_init.sh @@ -1,9 +1,9 @@ set -e sudo microdnf install git sudo -u splunk mkdir -p /opt/splunk/vcs/{git_tmp_backup,git_tmp_restore} +if [ ! -d "/opt/splunk/vcs/backup.git" ]; then sudo -u splunk mkdir -p /opt/splunk/vcs/backup.git sudo -u splunk sh -c "cd /opt/splunk/vcs/backup.git && git init --bare --shared" -if [ ! -d "/opt/splunk/vcs/backup.git" ]; then sudo -u splunk sh -c "cd /opt/splunk/vcs/ && git clone backup.git" sudo -u splunk sh -c "git config --global user.email 'backup@example.local'" sudo -u splunk sh -c "git config --global user.name 'BackupUser'" From e51872e3de740bbb6b351a3d0191c22401faed5e Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:46:55 +0100 Subject: [PATCH 08/17] Remove lookup from container mapping In order to regenerate the lookups especially the taglist on recreation the lookups folder is not shared. --- test/docker-compose.yml | 61 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 test/docker-compose.yml diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 0000000..1d03326 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.7" +networks: + splunknet: + driver: bridge + attachable: true + +services: + splunk_sh: + networks: + splunknet: + aliases: + - splunk_sh + hostname: splunk_sh + container_name: splunk_sh + image: splunk/splunk:latest + ports: + - 8003:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + volumes: + - ../bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ../default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ../lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ../metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ../README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ../static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ../test/sh_default.yml:/tmp/defaults/default.yml + + splunk_moc: + networks: + splunknet: + aliases: + - splunk_moc + hostname: splunk_moc + container_name: splunk_moc + image: splunk/splunk:latest + entrypoint: + - /bin/sh + - /usr/sbin/custom_init.sh + ports: + - 8004:8000 + - 8089 + environment: + - SPLUNK_PASSWORD=g8nGXBQBF + - SPLUNK_START_ARGS=--accept-license + - SPLUNK_STANDALONE_URL=splunk_sh + - SPLUNK_ROLE=splunk_monitor + - DEBUG=true + + volumes: + - ../bin:/opt/splunk/etc/apps/SplunkVersionControl/bin + - ../default:/opt/splunk/etc/apps/SplunkVersionControl/default + - ../lib:/opt/splunk/etc/apps/SplunkVersionControl/lib + - ../metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata + - ../README:/opt/splunk/etc/apps/SplunkVersionControl/README + - ../static:/opt/splunk/etc/apps/SplunkVersionControl/static + - ../test/bkp_default.yml:/tmp/defaults/default.yml + - ../test/custom_init.sh:/usr/sbin/custom_init.sh From 4dd598fea4069b315c3b48e774c39bef2dce01b2 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 00:52:51 +0100 Subject: [PATCH 09/17] Remove old docker-compose.yml --- docker-compose.yml | 64 ---------------------------------------------- 1 file changed, 64 deletions(-) delete mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 33dcd74..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: "3.7" -networks: - splunknet: - driver: bridge - attachable: true - -services: - splunk_sh: - networks: - splunknet: - aliases: - - splunk_sh - hostname: splunk_sh - container_name: splunk_sh - image: splunk/splunk:latest - ports: - - 8003:8000 - - 8089 - environment: - - SPLUNK_PASSWORD=g8nGXBQBF - - SPLUNK_START_ARGS=--accept-license - - SPLUNK_STANDALONE_URL=splunk_sh - volumes: - - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin - - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default - - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib - - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups - - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata - - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README - - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static - - ./test/sh_default.yml:/tmp/defaults/default.yml - - splunk_moc: - networks: - splunknet: - aliases: - - splunk_moc - hostname: splunk_moc - container_name: splunk_moc - image: splunk/splunk:latest - entrypoint: - - /bin/sh - - /usr/sbin/custom_init.sh - ports: - - 8004:8000 - - 8089 - environment: - - SPLUNK_PASSWORD=g8nGXBQBF - - SPLUNK_START_ARGS=--accept-license - - SPLUNK_STANDALONE_URL=splunk_sh - - SPLUNK_ROLE=splunk_monitor - - DEBUG=true - - volumes: - - ./bin:/opt/splunk/etc/apps/SplunkVersionControl/bin - - ./default:/opt/splunk/etc/apps/SplunkVersionControl/default - - ./lib:/opt/splunk/etc/apps/SplunkVersionControl/lib - - ./lookups:/opt/splunk/etc/apps/SplunkVersionControl/lookups - - ./metadata:/opt/splunk/etc/apps/SplunkVersionControl/metadata - - ./README:/opt/splunk/etc/apps/SplunkVersionControl/README - - ./static:/opt/splunk/etc/apps/SplunkVersionControl/static - - ./test/bkp_default.yml:/tmp/defaults/default.yml - - ./test/install_git.yml:/tmp/install_git.yml - - ./test/custom_init.sh:/usr/sbin/custom_init.sh From c336b4776902c0deeddf3d85707e0c4c05aa4537 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 12:34:45 +0100 Subject: [PATCH 10/17] Add certification creation scripts --- test/certificates/.gitignore | 5 ++ test/certificates/createca.sh | 13 ++++ test/certificates/createcerts.sh | 129 +++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 test/certificates/.gitignore create mode 100644 test/certificates/createca.sh create mode 100644 test/certificates/createcerts.sh diff --git a/test/certificates/.gitignore b/test/certificates/.gitignore new file mode 100644 index 0000000..423a60f --- /dev/null +++ b/test/certificates/.gitignore @@ -0,0 +1,5 @@ +*.crt +*.key +*.pem +*.csr +castuff diff --git a/test/certificates/createca.sh b/test/certificates/createca.sh new file mode 100644 index 0000000..9491f1d --- /dev/null +++ b/test/certificates/createca.sh @@ -0,0 +1,13 @@ +set -e +CA_KEY=ca.key +CA_CRT=ca.crt +SUBJ="/C=CH/ST=Bern/L=Bern/O=Example Company/CN=CA Domain1" + +if [ ! -f $CA_KEY ]; then + openssl genrsa -out $CA_KEY 4096 +fi + +if [ ! -f $CA_CRT ]; then + openssl req -new -x509 -key ${CA_KEY} -out ${CA_CRT} -subj "$SUBJ" -days 3000 +fi + diff --git a/test/certificates/createcerts.sh b/test/certificates/createcerts.sh new file mode 100644 index 0000000..61e9a36 --- /dev/null +++ b/test/certificates/createcerts.sh @@ -0,0 +1,129 @@ +set -e + +if [ $# -ne 3 ] + then + cat < $dir/serial +[ ! -f $dir/index.txt ] && touch $dir/index.txt +# SIGN Request +openssl ca \ + -in ${CSR} \ + -out ${CERT} \ + -extensions ${CRT_TYPE} \ + -config <( +cat <<-EOF +[ ca ] +default_ca = CA_default +[ CA_default ] +copy_extensions = copy +dir = . +certs = $dir/certs +crl_dir = $dir/crl +new_certs_dir = $dir/newcerts +database = $dir/index.txt +serial = $dir/serial +RANDFILE = $dir/private/.rand + +private_key = ./ca.key +certificate = ./ca.crt + +crlnumber = $dir/crlnumber +crl = $dir/crl/ca.crl.pem +crl_extensions = crl_ext +default_crl_days = 30 + +# SHA-1 is deprecated, so use SHA-2 instead. +default_md = sha256 + +name_opt = ca_default +cert_opt = ca_default +default_days = 375 +preserve = no +policy = policy_loose +unique_subject = no + +[ policy_loose ] +countryName = optional +stateOrProvinceName = optional +localityName = optional +organizationName = optional +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[ usr_cert ] +basicConstraints = CA:FALSE +nsCertType = client +nsComment = "Client Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, emailProtection + +[ server_cert ] +basicConstraints = CA:FALSE +nsCertType = server +nsComment = "OpenSSL Generated Server Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth + +EOF +) + +cat ${CERT} ${KEYOUT} > ${COMBINED} +#rm ${CERT} ${CSR} ${KEYOUT} From c23d5b4a10c4cd4fb4e3dccb7d5e0ef1b1a52e57 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 14:42:01 +0100 Subject: [PATCH 11/17] Add certificate creation scripts --- test/bkp_default.yml | 9 +++++++++ test/certificates/Dockerfile | 9 +++++++++ test/certificates/createcerts.sh | 1 + test/certificates/entrypoint.sh | 21 +++++++++++++++++++++ test/docker-compose.yml | 21 +++++++++++++++++++++ test/sh_default.yml | 9 +++++++++ 6 files changed, 70 insertions(+) create mode 100644 test/certificates/Dockerfile create mode 100755 test/certificates/entrypoint.sh diff --git a/test/bkp_default.yml b/test/bkp_default.yml index b2afbcc..2385add 100644 --- a/test/bkp_default.yml +++ b/test/bkp_default.yml @@ -27,3 +27,12 @@ splunk: useLocalAuth: false auditLogsLookupBackTime: -2h timewait: 30 + - key: server + value: + directory: /opt/splunk/etc/system/local/ + content: + sslConfig: + enableSplunkdSSL: true + serverCert: /cert_dir/splunk_sh.pem + sslRootCAPath: /cert_dir/ca.crt + sslPassword: password diff --git a/test/certificates/Dockerfile b/test/certificates/Dockerfile new file mode 100644 index 0000000..411a7c2 --- /dev/null +++ b/test/certificates/Dockerfile @@ -0,0 +1,9 @@ +FROM alpine:latest + +RUN apk update && \ + apk add --no-cache openssl bash && \ + rm -rf "/var/cache/apk/*" + +COPY createca.sh createcerts.sh entrypoint.sh /scripts/ + +ENTRYPOINT [ "bash", "/scripts/entrypoint.sh" ] diff --git a/test/certificates/createcerts.sh b/test/certificates/createcerts.sh index 61e9a36..3c0a2cc 100644 --- a/test/certificates/createcerts.sh +++ b/test/certificates/createcerts.sh @@ -60,6 +60,7 @@ mkdir -p $dir/certs [ ! -f $dir/index.txt ] && touch $dir/index.txt # SIGN Request openssl ca \ + -batch \ -in ${CSR} \ -out ${CERT} \ -extensions ${CRT_TYPE} \ diff --git a/test/certificates/entrypoint.sh b/test/certificates/entrypoint.sh new file mode 100755 index 0000000..ede1288 --- /dev/null +++ b/test/certificates/entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e +echo "Starting Certificate Creation" +mkdir -p /cert_dir +cd /cert_dir +sh /scripts/createca.sh +while read -r line; do + # Get the string before = (the var name) + name="${line%=*}" + eval value="\$$name" + if [[ $name = 'CERTIFICATE'* ]] + then + echo "name: ${name}, value: ${value}" + bash /scripts/createcerts.sh ${value} + fi +done < Date: Fri, 12 Feb 2021 15:58:04 +0100 Subject: [PATCH 12/17] Configuration for ssl --- test/bkp_default.yml | 6 +++--- test/sh_default.yml | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/bkp_default.yml b/test/bkp_default.yml index 2385add..8876445 100644 --- a/test/bkp_default.yml +++ b/test/bkp_default.yml @@ -12,7 +12,7 @@ splunk: srcPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_backup gitRepoURL: file:///opt/splunk/vcs/backup.git - # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + sslVerify: /cert_dir/ca.crt debugMode: true useLocalAuth: false interval: 60 @@ -22,7 +22,7 @@ splunk: destPassword: g8nGXBQBF gitTempDir: /opt/splunk/vcs/git_tmp_restore gitRepoURL: file:///opt/splunk/vcs/backup.git - # sslVerify: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem + sslVerify: /cert_dir/ca.crt debugMode: true useLocalAuth: false auditLogsLookupBackTime: -2h @@ -33,6 +33,6 @@ splunk: content: sslConfig: enableSplunkdSSL: true - serverCert: /cert_dir/splunk_sh.pem + serverCert: /cert_dir/splunk_moc.pem sslRootCAPath: /cert_dir/ca.crt sslPassword: password diff --git a/test/sh_default.yml b/test/sh_default.yml index ef26525..ab5de78 100644 --- a/test/sh_default.yml +++ b/test/sh_default.yml @@ -10,8 +10,11 @@ splunk: splunk_vc_url: definition: https://splunk_moc:8089/services/splunkversioncontrol_rest_restore sslVerify: - definiton: false - # definition: /opt/splunk/etc/apps/SplunkVersionControl/auth/ca.pem +# definiton: false + definition: /cert_dir/ca.crt + requestingAddress: + # is used to connect back + definition: https://splunk_sh:8089 - key: server value: directory: /opt/splunk/etc/system/local/ From f09b152c06725be3e271f36289bd99f4dd921520 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 15:58:16 +0100 Subject: [PATCH 13/17] Adjustments for SSL support --- bin/postversioncontrolrestore.py | 13 +++++++++++-- bin/splunkversioncontrol_backup_class.py | 2 +- bin/splunkversioncontrol_rest_restore.py | 20 ++++++++++++++------ bin/splunkversioncontrol_utility.py | 4 ++-- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index 73cd511..d34049f 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -10,7 +10,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "lib")) from splunklib.searchcommands import dispatch, GeneratingCommand, Configuration, Option -from splunklib.searchcommands.validators import Validator, Boolean, File +from splunklib.searchcommands.validators import Validator, Boolean from splunklib.binding import HTTPError class OrValidator(Validator): @@ -31,6 +31,13 @@ def format(self, value): except: return self.b.format(value) +class Filename(Validator): + # TODO Validate file path + def __call__(self, value): + return value + + def format(self, value): + return value splunkLogsDir = os.environ['SPLUNK_HOME'] + "/var/log/splunk" #Setup the logging @@ -80,7 +87,8 @@ class SVCPostRestore(GeneratingCommand): restoreAsUser = Option(require=True) scope = Option(require=True) timeout = Option(require=True) - sslVerify = Option(require=False, default=False, validate=OrValidator(File(), Boolean())) + sslVerify = Option(require=False, default=False, validate=OrValidator(Boolean(), Filename())) + requestingAddress = Option(require=False, default=False) def generate(self): """ @@ -108,6 +116,7 @@ def generate(self): body['restoreAsUser'] = self.restoreAsUser body['scope'] = self.scope body['timeout'] = self.timeout + body['requestingAddress'] = self.requestingAddress logger.info("Attempting POST request to url=%s with body=\"%s\"" % (url, body)) diff --git a/bin/splunkversioncontrol_backup_class.py b/bin/splunkversioncontrol_backup_class.py index 688d6ed..b2a54ea 100644 --- a/bin/splunkversioncontrol_backup_class.py +++ b/bin/splunkversioncontrol_backup_class.py @@ -1037,7 +1037,7 @@ def run_script(self): headers={'Authorization': 'Splunk %s' % config['session_key']} url = 'https://localhost:8089/services/shcluster/captain/info?output_mode=json' - res = requests.get(url, headers=headers, verify=self.sslVerify) + res = requests.get(url, headers=headers, verify=False) if (res.status_code == 503): logger.debug("i=\"%s\" Non-shcluster / standalone instance, safe to run on this node" % (self.stanzaName)) elif (res.status_code != requests.codes.ok): diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 2b76e72..5a0a968 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -59,12 +59,15 @@ class SVCRestore(splunk.rest.BaseRestHandler): - def query_back_for_user_and_permissions(self, authorization_token, *, sslVerify): + def query_back_for_user_and_permissions(self, requestingAddress, authorization_token, *, sslVerify): headers = { "Authorization" : authorization_token } #Run a query back against the source system to check the username/role - remoteAddr = self.request['remoteAddr'] - url = "https://" + remoteAddr + ":8089/services/authentication/current-context?output_mode=json" + if requestingAddress: + remoteAddr = requestingAddress + else: + remoteAddr = "https://" + self.request['remoteAddr'] + ":8089" + url = remoteAddr + "/services/authentication/current-context?output_mode=json" logger.info("Received remote request checking username and role related to the token on url=%s" % (url)) logger.debug("token=%s" % (authorization_token)) @@ -160,7 +163,12 @@ def handle_POST(self): else: time_wait = 600 - username, roles = self.query_back_for_user_and_permissions(payload['Authorization'][0], sslVerify=sslVerify) + if 'requestingAddress' in payload: + requestingAddress = payload['requestingAddress'][0] + else: + requestingAddress = None + + username, roles = self.query_back_for_user_and_permissions(requestingAddress, payload['Authorization'][0], sslVerify=sslVerify) logger.info("username=%s roles=%s" % (username, roles)) app = payload['app'][0] @@ -207,7 +215,7 @@ def handle_POST(self): headers = { "Authorization" : "Splunk " + self.request['systemAuth'] } curtime = calendar.timegm(time.gmtime()) url = "https://localhost:8089/servicesNS/nobody/SplunkVersionControl/storage/collections/data/splunkversioncontrol_rest_restore_status" - res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=sslVerify) + res = self.runHttpRequest(url, headers, None, "get", "checking kvstore collection splunkversioncontrol_rest_restore_status", sslVerify=False) if not res: return @@ -216,7 +224,7 @@ def handle_POST(self): if not len(res) == 0: if not 'start_time' in res[0]: logger.warn("Warning invalid kvstore data, will wipe it and continue in collection splunkversioncontrol_rest_restore_status on url=%s, value returned res=\"%s\"" % (url, payload)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=sslVerify) + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status', sslVerify=False) else: kvstore_start_time = res[0]['start_time'] target_time = curtime - time_wait diff --git a/bin/splunkversioncontrol_utility.py b/bin/splunkversioncontrol_utility.py index 55689f3..0ec0b57 100644 --- a/bin/splunkversioncontrol_utility.py +++ b/bin/splunkversioncontrol_utility.py @@ -55,7 +55,7 @@ def get_password(password, session_key, logger, *, sslVerify=False): url = "https://localhost:8089/servicesNS/-/" + context + "/storage/passwords?output_mode=json&f=clear_password&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) headers = {'Authorization': 'Splunk %s' % session_key} - res = requests.get(url, headers=headers, verify=sslVerify) + res = requests.get(url, headers=headers, verify=False) dict = json.loads(res.text) clear_password = False if not 'entry' in dict: @@ -73,7 +73,7 @@ def get_password(password, session_key, logger, *, sslVerify=False): url = "https://localhost:8089/servicesNS/-/-/storage/passwords?output_mode=json&f=clear_password&count=0&search=" + password logger.debug("Trying url=%s with session_key to obtain name=%s" % (url, password)) - res = requests.get(url, headers=headers, verify=sslVerify) + res = requests.get(url, headers=headers, verify=False) dict = json.loads(res.text) if not 'entry' in dict: logger.warn("dict=%s did not contain the entries expected on url=%s while looking for password=%s" % (dict, url, password)) From 99571a77eec98c498a67f7b848a3349533387406 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:09:44 +0100 Subject: [PATCH 14/17] Add additional macro --- .../data/ui/views/splunkversioncontrol_restore_dynamic.xml | 2 +- default/macros.conf | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml index 9a961a4..839c909 100644 --- a/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml +++ b/default/data/ui/views/splunkversioncontrol_restore_dynamic.xml @@ -129,7 +129,7 @@
SplunkVersionControl Results (please wait a few minutes for results) - | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` sslVerify=`sslVerify` + | postversioncontrolrestore app="$app$" obj_name="$name$" restoreAsUser="$restoreAsUser$" scope="$scope$" splunk_vc_name=`splunk_vc_name` tag="$tag$" url=`splunk_vc_url` type="$type$" timeout=`splunk_vc_timeout` sslVerify=`sslVerify` requestingAddress=`requestingAddress` -5m now 1 diff --git a/default/macros.conf b/default/macros.conf index 37089d5..31c7670 100644 --- a/default/macros.conf +++ b/default/macros.conf @@ -22,3 +22,8 @@ iseval = 0 [sslVerify] definition = false iseval = 0 + +[requestingAddress] +definition = false +iseval = 0 + From 52d18d2d49cc0656329d85c6237e62c6830f4f17 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:10:31 +0100 Subject: [PATCH 15/17] Add validation and disable SSL for localhost --- bin/postversioncontrolrestore.py | 3 ++- bin/splunkversioncontrol_rest_restore.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/postversioncontrolrestore.py b/bin/postversioncontrolrestore.py index d34049f..f6ca563 100644 --- a/bin/postversioncontrolrestore.py +++ b/bin/postversioncontrolrestore.py @@ -116,7 +116,8 @@ def generate(self): body['restoreAsUser'] = self.restoreAsUser body['scope'] = self.scope body['timeout'] = self.timeout - body['requestingAddress'] = self.requestingAddress + if self.requestingAddress: + body['requestingAddress'] = self.requestingAddress logger.info("Attempting POST request to url=%s with body=\"%s\"" % (url, body)) diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 5a0a968..9869278 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -231,7 +231,7 @@ def handle_POST(self): if kvstore_start_time < target_time: logger.warn("Found existing entry from %s but time is %s, this is past the limit of current time minus %s (%s)" % (kvstore_start_time, curtime, time_wait, target_time)) #More than 10 minutes ago, delete the entry and move on - self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=sslVerify) + self.runHttpRequest(url, headers, None, "delete", "wiping kvstore splunkversioncontrol_rest_restore_status due to record %s older than %s time period" % (kvstore_start_time, target_time), sslVerify=False) else: removal_target = kvstore_start_time + time_wait + 1 logger.warn("Attempted to run but found a running restore instance with time=%s and current_time=%s, will delete and move on after current_time_minus=%s seconds (override_time=%s)" % (kvstore_start_time, curtime, time_wait, removal_target)) From 5bd5aaabf35deac3af8cbced13f1478a0555a642 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Fri, 12 Feb 2021 16:18:38 +0100 Subject: [PATCH 16/17] Remove validation for localhost --- bin/splunkversioncontrol_rest_restore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 9869278..3142aa2 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -242,7 +242,7 @@ def handle_POST(self): payload = json.dumps({ 'start_time': curtime }) headers['Content-Type'] = 'application/json' #update kvstore with runtime - res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=sslVerify) + res = self.runHttpRequest(url, headers, payload, 'post', 'updating kvstore collection splunkversioncontrol_rest_restore_status', sslVerify=False) if not res: return res @@ -254,7 +254,7 @@ def handle_POST(self): self.response.write("Restore has failed to complete successfully in app %s, object of type %s, with name %s, from tag %s, scope %s with restoreAsUser %s and your username of %s. Message is %s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) logger.warn("Restore has failed to complete successfully in app=%s, object of type=%s, with name=%s, from tag=%s, scope=%s with restoreAsUser=%s and requested by username=%s, message=%s" % (app, type, obj_name, tag, scope, restoreAsUser, username, message)) - self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=sslVerify) + self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=False) #Run a Splunk query via the search/jobs endpoint def runSearchJob(self, url, appname, headers, auth, username, earliest_time, *, sslVerify=False): From d4d3d2caa26baf0f539d1dbd44a4d096bb752563 Mon Sep 17 00:00:00 2001 From: calesanz <8714917+calesanz@users.noreply.github.com> Date: Sat, 13 Feb 2021 19:32:26 +0100 Subject: [PATCH 17/17] Remove named argument enforcement for python2 --- bin/splunkversioncontrol_rest_restore.py | 6 +++--- bin/splunkversioncontrol_utility.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/splunkversioncontrol_rest_restore.py b/bin/splunkversioncontrol_rest_restore.py index 3142aa2..ce9eb35 100644 --- a/bin/splunkversioncontrol_rest_restore.py +++ b/bin/splunkversioncontrol_rest_restore.py @@ -59,7 +59,7 @@ class SVCRestore(splunk.rest.BaseRestHandler): - def query_back_for_user_and_permissions(self, requestingAddress, authorization_token, *, sslVerify): + def query_back_for_user_and_permissions(self, requestingAddress, authorization_token, sslVerify): headers = { "Authorization" : authorization_token } #Run a query back against the source system to check the username/role @@ -257,7 +257,7 @@ def handle_POST(self): self.runHttpRequest(url, headers, None, 'delete', 'wiping kvstore splunkversioncontrol_rest_restore_status after completed run', sslVerify=False) #Run a Splunk query via the search/jobs endpoint - def runSearchJob(self, url, appname, headers, auth, username, earliest_time, *, sslVerify=False): + def runSearchJob(self, url, appname, headers, auth, username, earliest_time, sslVerify=False): url = url + "/servicesNS/-/%s/search/jobs" % (appname) query = "savedsearch \"Splunk Version Control Audit Query POST\" username=\"%s\" | stats count | where count>0" % (username) logger.debug("Running requests.post() on url=%s query=\"%s\"" % (url, query)) @@ -279,7 +279,7 @@ def runSearchJob(self, url, appname, headers, auth, username, earliest_time, *, logger.warn("messages from query=\"%s\" were messages=\"%s\"" % (query, res["messages"])) return res - def runHttpRequest(self, url, headers, data, type, text, *, sslVerify=False): + def runHttpRequest(self, url, headers, data, type, text, sslVerify=False): if type == "delete": res = requests.delete(url, headers=headers, verify=sslVerify) elif type == "post": diff --git a/bin/splunkversioncontrol_utility.py b/bin/splunkversioncontrol_utility.py index 0ec0b57..97818df 100644 --- a/bin/splunkversioncontrol_utility.py +++ b/bin/splunkversioncontrol_utility.py @@ -36,7 +36,7 @@ def runOSProcess(command, logger, timeout=60, shell=False): return str(stdout), str(stderr), res # use the password endpoint to obtain the clear_password passed in, start with the context of this app and then try all contexts -def get_password(password, session_key, logger, *, sslVerify=False): +def get_password(password, session_key, logger, sslVerify=False): #TODO move this into shared function to obtain passwords: context = os.path.dirname(os.path.dirname(__file__))