From a135a2d17009dc02374610db411dc56d8cca17cc Mon Sep 17 00:00:00 2001 From: nirs Date: Thu, 7 Nov 2019 09:33:41 +0200 Subject: [PATCH 1/9] Experimental - a web server emulating the RIAK over a File Sytem and possibly more --- experimental/emulated_vault/README | 23 ++ .../etc/systemd/system/emulated-vault.service | 13 + .../opt/emulated-vault/adaptor/__init__.py | 1 + .../opt/emulated-vault/adaptor/fs.py | 149 +++++++++++ .../opt/emulated-vault/conf/cfg.json | 13 + .../opt/emulated-vault/vault-cmd | 145 +++++++++++ .../opt/emulated-vault/vault-server | 232 ++++++++++++++++++ 7 files changed, 576 insertions(+) create mode 100644 experimental/emulated_vault/README create mode 100644 experimental/emulated_vault/etc/systemd/system/emulated-vault.service create mode 100644 experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py create mode 100644 experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py create mode 100644 experimental/emulated_vault/opt/emulated-vault/conf/cfg.json create mode 100644 experimental/emulated_vault/opt/emulated-vault/vault-cmd create mode 100644 experimental/emulated_vault/opt/emulated-vault/vault-server diff --git a/experimental/emulated_vault/README b/experimental/emulated_vault/README new file mode 100644 index 0000000000..c3a71526bc --- /dev/null +++ b/experimental/emulated_vault/README @@ -0,0 +1,23 @@ +The emulated_vault module supplies a HTTP server mimicking RIAK behavior for the usage as traffic-control vault. +The server may use different type of persistent storage (e.g. file-system), using the proper adaptor. +Basic requirements: Centos ver >= 7; Python + +In order to deploy the code on a server please: +1. Copy the module files to the server's root +2. Add the certificate and key to you favorite path +3. Adjust opt/emulated_vault/conf/cfg.json - pointing at your certificate and key +4. chkconfig the service + +Logs may be found under /opt/emulated_vault/var/log + +Developers notes: +------------------------ +If you just want to play around with the module, you may of course run the server script on its own. +Before doing that, you would probably need to adjust the opt/emulated_vault/conf/cfg.json: +1. Changing the db-path to one you have access to +2. Disable ssl (just to make it easier) + +Additionally, the vault-cmd script is also available to work against the DB with command line. +It is mostly useful when developing a new adaptor. + +For any question, please approach Nir B. Sopher (nir@apache.org) diff --git a/experimental/emulated_vault/etc/systemd/system/emulated-vault.service b/experimental/emulated_vault/etc/systemd/system/emulated-vault.service new file mode 100644 index 0000000000..8117ee6aaa --- /dev/null +++ b/experimental/emulated_vault/etc/systemd/system/emulated-vault.service @@ -0,0 +1,13 @@ +[Unit] +Description=Emulated Vault Service +After=network.target + +[Service] +Type=simple +User=root +ExecStart=/opt/emulated-vault/vault-server +Restart=on-failure + + +[Install] +WantedBy=multi-user.target diff --git a/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py b/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py @@ -0,0 +1 @@ + diff --git a/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py b/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py new file mode 100644 index 0000000000..5dbfb10642 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py @@ -0,0 +1,149 @@ +import os + +class Fs(object): + def __init__ (self, logger, basePath, pingRelPath): + self.logger = logger + self.basePath = basePath + self.pingRelPath = pingRelPath + + def init_ping(self): + pingPath = os.path.join(self.basePath, self.pingRelPath) + value = "OK" + success = self._set_param(pingPath, value) + if not success: + self.logger.error("Failed to set variable %s", pingPath) + return False + return True + + def ping(self): + pingPath = os.path.join(self.basePath, self.pingRelPath) + success, value = self._get_param(pingPath) + if not success or value is None: + self.logger.error("no ping response") + return (False, "") + self.logger.debug("ping response: %s", value) + return (True, value) + + def getParameter(self, parameterPath): + parameterPath = parameterPath.lstrip("/") + _parameterPath = os.path.join(self.basePath, parameterPath) + success, value = self._get_param(_parameterPath) + if not success: + self.logger.error("Failed to bring variable %s", parameterPath) + return False, "" + if value is None: + self.logger.error("Could not find variable %s", parameterPath) + return True, None + self.logger.debug("Parameter get response for %s ready", parameterPath) + #self.logger.debug("Parameter get response: %s", value) + return True, value + + def searchParameters(self, parameterPathPrefix, keyFilters = {}, filters = {}): + parameterPathPrefix = parameterPathPrefix.lstrip("/") + _parametersPathPrefix = os.path.join(self.basePath, parameterPathPrefix) + success, items = self._get_params(_parametersPathPrefix, prefixToRemove=self.basePath, keyFilters=keyFilters) + if not success: + self.logger.error("Failed to bring variable by prefix %s", parameterPathPrefix) + return False, "" + + filtered = {} + for key, val in items.iteritems(): + skip = False + for filterName, filter in filters.iteritems(): + if not filter(key, val): + self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) + skip = True + break + if skip: + continue + filtered[key] = val + + return True, filtered + + + def setParameter(self, parameterPath, value): + parameterPath = parameterPath.lstrip("/") + _parameterPath = os.path.join(self.basePath, parameterPath) + success = self._set_param(_parameterPath, value) + if not success: + self.logger.error("Failed to set variable %s", parameterPath) + return False + return True + + def deleteParameter(self, parameterPath): + parameterPath = parameterPath.lstrip("/") + _parameterPath = os.path.join(self.basePath, parameterPath) + success = self._delete_param(_parameterPath) + if not success: + self.logger.error("Failed to delete variable %s", parameterPath) + return False + return True + + def _get_param(self, paramName): + self.logger.debug("Get parameter %s", paramName) + try: + with open(paramName) as fd: + value = fd.read() + except: + self.logger.exception("%s parameter not found.", paramName) + return False, None + + self.logger.debug("Get parameter %s succeed", paramName) + #self.logger.debug("Get parameter %s=%s", paramName, value) + return True, value + + def _get_params(self, paramNamePrefix, prefixToRemove = "", keyFilters={}): + self.logger.debug("Get parameters under path %s", paramNamePrefix) + prefixToRemoveLen = len(prefixToRemove) + parameters = {} + + fileNames = [] + for (dirpath, dirnames, filenames) in os.walk(paramNamePrefix): + fileNames += [os.path.join(dirpath, file) for file in filenames] + + for fileName in fileNames: + filteredOut = False + for filterName, filter in keyFilters.iteritems(): + if not filter(fileName): + self.logger.debug("Parameter %s dropped, not matching filter %s", fileName, filterName) + filteredOut = True + break + if filteredOut: + continue + + paramName = fileName[prefixToRemoveLen:] + success, value = self._get_param(fileName) + if not success: + self.logger.error("%s parameter not found.", paramName) + return False, None + + parameters[paramName] = value + + return True, parameters + + + def _set_param(self, paramName, value): + self.logger.debug("Set parameter %s", paramName) + #self.logger.debug("Set parameter %s=%s", paramName, value) + try: + dirname = os.path.dirname(paramName) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(paramName, "w") as fd: + fd.write(value) + except: + self.logger.exception("cloud not post parameter %s", paramName) + return False + self.logger.debug("Set parameter %s done", paramName) + return True + + def _delete_param(self, paramName): + self.logger.debug("Delete parameter %s", paramName) + try: + os.remove(paramName) + except: + self.logger.exception("cloud not delete parameter %s", paramName) + return False + self.logger.debug("Delete parameter %s done", paramName) + return True + diff --git a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.json b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.json new file mode 100644 index 0000000000..e32e5a8ba2 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.json @@ -0,0 +1,13 @@ +{ + "data": { + "ping-rel-path": "ping", + "db-path": "/opt/emulated-vault/db/", + "adaptor-type": "fs", + "listen-ip": "0.0.0.0", + "listen-port": 8088, + "use-ssl": true, + "ssl-key-path": "path/to/cert/key.pem", + "ssl-cert-path": "path/to/cert/cert.pem" + }, + "format-version": 100 +} diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-cmd b/experimental/emulated_vault/opt/emulated-vault/vault-cmd new file mode 100644 index 0000000000..4c3c6a53b3 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault-cmd @@ -0,0 +1,145 @@ +#! /usr/bin/python + +import json +import logging +import optparse +import os +import sys +import inspect + +import adaptor.fs + +if __name__ == '__main__': + progPath = inspect.stack()[-1][1] + progAbsPath = os.path.abspath( progPath ) + progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) + + confDir = os.path.join(progAbsPath, "conf") + confFile = os.path.join(confDir, "cfg.json") + with open(confFile) as fd: + cfg = json.load(fd) + + logDir = os.path.join(progAbsPath, "var/log") + debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") + mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") + if not os.path.exists(logDir): + os.makedirs(logDir) + + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + # create file handler which logs even debug messages + fhd = logging.FileHandler(debugLogFile) + fhd.setLevel(logging.DEBUG) + fhm = logging.FileHandler(mainLogFile) + fhm.setLevel(logging.INFO) + #TODO Set based on command line + verbose = logging.StreamHandler(sys.stdout) + verbose.setLevel(logging.INFO) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fhm.setFormatter(formatter) + fhd.setFormatter(formatter) + verbose.setFormatter(formatter) + # add the handlers to the logger + logger.addHandler(fhm) + logger.addHandler(fhd) + logger.addHandler(verbose) + + parser = optparse.OptionParser() + operationModeGroup = optparse.OptionGroup(parser, "Operation mode options", + "The vault script may work in one of the following modes.") + parser.add_option_group(operationModeGroup) + + operationModeGroup.add_option("--ping", + action="store_true", dest="ping", default=False, help="Ping") + + operationModeGroup.add_option("--get-parameter", + action="store_true", dest="getParameter", default=False, + help="Get parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--set-parameter", + action="store_true", dest="setParameter", default=False, + help="Set parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--delete-parameter", + action="store_true", dest="deleteParameter", default=False, + help="Delete parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--search-parameters", + action="store_true", dest="searchParameters", default=False, + help="Search parameter. Parameter relative path should be provided.") + + standardOptions = optparse.OptionGroup(parser, "Standard operation options", + "Standard operation options.") + + parser.add_option_group(standardOptions) + + standardOptions.add_option("--parameter-path", type="string", + action="store", dest="parameterPath", help="Parameter path") + + (options, args) = parser.parse_args() + + + basePath = cfg["data"]["db-path"] + pingRelPath = cfg["data"]["ping-rel-path"] + adaptorType = cfg["data"]["adaptor-type"] + + if adaptorType == "fs": + adaptor = adaptor.fs.Fs(logger=logger, basePath=basePath, pingRelPath=pingRelPath) + else: + logger.error("Invalid adaptor type '%s'", adaptorType) + sys.exit(1) + + if options.ping: + success, value = adaptor.ping() + if not success: + sys.exit(1) + print value + sys.exit(0) + + if options.getParameter: + if not options.parameterPath: + parser.error("Missing parameter-path.") + success, value = adaptor.getParameter(options.parameterPath) + if not success: + sys.exit(1) + if value is None: + sys.exit(1) + print value + sys.exit(0) + + if options.searchParameters: + if not options.parameterPath: + parser.error("Missing parameter-path.") + success, value = adaptor.searchParameters(options.parameterPath) + if not success: + sys.exit(1) + print json.dumps(value) + sys.exit(0) + + if options.setParameter: + if not options.parameterPath: + parser.error("Missing parameter-path.") + if len(args) != 1: + parser.error("Command should get a single arg - the to be set value.") + success = adaptor.setParameter(options.parameterPath, args[0]) + if not success: + sys.exit(1) + sys.exit(0) + + if options.deleteParameter: + if not options.parameterPath: + parser.error("Missing parameter-path.") + success = adaptor.deleteParameter(options.parameterPath) + if not success: + sys.exit(1) + sys.exit(0) + + logger.error("Operation is not set") + parser.print_help() + parser.error("No operation mode specified") + sys.exit(1) + + + diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server new file mode 100644 index 0000000000..c528ce10dc --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -0,0 +1,232 @@ +#! /usr/bin/python + +import json +import logging +import sys +import os +import inspect +from BaseHTTPServer import HTTPServer +from BaseHTTPServer import BaseHTTPRequestHandler +import ssl +import urlparse, json +import re, fnmatch + +import adaptor.fs + +class RequestHandler(BaseHTTPRequestHandler): + + def do_GET(self): + try: + self._do_GET() + except: + logger.exception("do_GET exception") + + do_HEAD = do_GET + + def do_POST(self): + try: + self._do_POST() + except: + logger.exception("do_POST exception") + + do_PUT = do_POST + + def do_DELETE(self): + try: + self._do_DELETE() + except: + logger.exception("do_DELETE exception") + + def _do_GET(self): + logger.debug("GET %s", self.path) + parsed_path = urlparse.urlparse(self.path) + + if parsed_path.path == "/ping": + logger.info("Ping") + success, value = adaptor.ping() + if not success: + self.send_response(503) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write("Failure") + return + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(value) + return + + if parsed_path.path == "/search/query/sslkeys": + logger.info("Search SSL: path=%s query=%s", parsed_path.path, parsed_path.query) + filters = {} + keyFilters = {} + cdnFind = re.search(".*q=cdn:([^&]*).*", parsed_path.query) + if cdnFind: + def cdnFilter(key,val): + try: + data = json.loads(val) + return data['cdn']==cdnFind.group(1) + except: + return False + filters['cdn'] = cdnFilter + + dsFind = re.search(".*q=deliveryservice:([^&]*).*", parsed_path.query) + if dsFind: + def dsFilter(key,val): + try: + data = json.loads(val) + return data['deliveryservice']==dsFind.group(1) + except: + return False + filters['ds'] = dsFilter + + keyFind = re.search(".*q=_yz_rk:([^&]*).*", parsed_path.query) + if keyFind: + def keyFilter(key): + return fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) + keyFilters['key-match'] = keyFilter + + success, parameters = adaptor.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}') + return + + docs = [] + for key,val in parameters.iteritems(): + docs.append(json.loads(val)) + toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(toReturn)) + return + + logger.info("GET %s", self.path) + success, value = adaptor.getParameter(parsed_path.path[1:]) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}') + return + + if value is None: + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Not found"}') + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(value) + + def _do_POST(self): + logger.info("POST %s", self.path) + parsed_path = urlparse.urlparse(self.path) + content_len = int(self.headers.getheader('content-length')) + post_body = self.rfile.read(content_len) + data = json.loads(post_body) + #mimic vault beahvior + if parsed_path.path.startswith('/riak/ssl/'): + for item in data['certificate']: + data['certificate.%s'%item]=data['certificate'][item] + success = adaptor.setParameter(parsed_path.path[1:], json.dumps(data)) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}') + return + + self.send_response(204) + self.send_header('Content-Type', 'application/json') + self.end_headers() + return + + + def _do_DELETE(self): + logger.info("DELETE %s", self.path) + parsed_path = urlparse.urlparse(self.path) + + success = adaptor.deleteParameter(parsed_path.path[1:]) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}') + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Deleted"}') + return + + + +if __name__ == '__main__': + progPath = inspect.stack()[-1][1] + progAbsPath = os.path.abspath( progPath ) + progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) + + confDir = os.path.join(progAbsPath, "conf") + confFile = os.path.join(confDir, "cfg.json") + with open(confFile) as fd: + cfg = json.load(fd) + + logDir = os.path.join(progAbsPath, "var/log") + debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") + mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") + if not os.path.exists(logDir): + os.makedirs(logDir) + + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + # create file handler which logs even debug messages + fhd = logging.FileHandler(debugLogFile) + fhd.setLevel(logging.DEBUG) + fhm = logging.FileHandler(mainLogFile) + fhm.setLevel(logging.INFO) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fhm.setFormatter(formatter) + fhd.setFormatter(formatter) + # add the handlers to the logger + logger.addHandler(fhm) + logger.addHandler(fhd) + + basePath = cfg["data"]["db-path"] + pingRelPath = cfg["data"]["ping-rel-path"] + listenIP = cfg["data"]["listen-ip"] + listenPort = cfg["data"]["listen-port"] + adaptorType = cfg["data"]["adaptor-type"] + use_ssl = cfg["data"]["use-ssl"] + + global adaptor + if adaptorType == "fs": + adaptor = adaptor.fs.Fs(logger=logger, basePath=basePath, pingRelPath=pingRelPath) + else: + logger.error("Invalid adaptor type '%s'", adaptorType) + sys.exit(1) + + if not adaptor.init_ping(): + logger.error("Failed initialization") + sys.exit(1) + + server = HTTPServer((listenIP, listenPort), RequestHandler) + if use_ssl: + key = cfg["data"]["ssl-key-path"] + cert = cfg["data"]["ssl-cert-path"] + server.socket = ssl.wrap_socket (server.socket, keyfile=key, certfile=cert, server_side=True) + msg = 'Starting server at http%s://%s:%d'%("s" if use_ssl else "", listenIP, listenPort) + print >> sys.stderr, msg + logger.info(msg) + server.serve_forever() From 66d6574f16bbd6e4a1f00022c20d989075d163d8 Mon Sep 17 00:00:00 2001 From: nirs Date: Mon, 11 Nov 2019 17:07:03 +0200 Subject: [PATCH 2/9] Emultaed vault PR fixes --- CHANGELOG.md | 1 + experimental/emulated_vault/README | 29 ++++++++++++------- .../etc/systemd/system/emulated-vault.service | 16 ++++++++++ .../opt/emulated-vault/adaptor/__init__.py | 17 ++++++++++- .../opt/emulated-vault/adaptor/fs.py | 17 +++++++++++ .../opt/emulated-vault/vault-cmd | 17 +++++++++++ .../opt/emulated-vault/vault-server | 17 +++++++++++ 7 files changed, 103 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27268c718d..eca735e288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added validation to prevent assigning servers to delivery services without required capabilities. - Added deep coverage zone routing percentage to the Traffic Portal dashboard. - Added a `traffic_ops/app/bin/osversions-convert.pl` script to convert the `osversions.cfg` file from Perl to JSON as part of the `/osversions` endpoint rewrite. +- Added [Experimental] - Emulated Vault suppling a HTTP server mimicking RIAK behavior for usage as traffic-control vault. ### Changed - Traffic Router: TR will now allow steering DSs and steering target DSs to have RGB enabled. (fixes #3910) diff --git a/experimental/emulated_vault/README b/experimental/emulated_vault/README index c3a71526bc..3bc01a16f8 100644 --- a/experimental/emulated_vault/README +++ b/experimental/emulated_vault/README @@ -1,17 +1,24 @@ -The emulated_vault module supplies a HTTP server mimicking RIAK behavior for the usage as traffic-control vault. -The server may use different type of persistent storage (e.g. file-system), using the proper adaptor. -Basic requirements: Centos ver >= 7; Python +# Emulated Vault - Background -In order to deploy the code on a server please: +The emulated_vault module supplies a HTTP server mimicking RIAK behavior for usage as traffic-control vault. +It may be used in order to replace RIAK traffic_vault, as it is much more simple to install. +The server may use different type of persistent storage (e.g. file-system), using the proper adaptor. +The resiliency of the stored keys is derived from the resiliency of the underlying storage. + +# Installation + +Basic requirements: Centos ver >= 7; Python >= 2.7 + +In order to install the module on a server please: 1. Copy the module files to the server's root -2. Add the certificate and key to you favorite path -3. Adjust opt/emulated_vault/conf/cfg.json - pointing at your certificate and key -4. chkconfig the service +2. Add the certificate and key to your favorite path +3. Adjust /opt/emulated_vault/conf/cfg.json - pointing at your certificate and key +4. "systemctl enable" the service Logs may be found under /opt/emulated_vault/var/log -Developers notes: ------------------------- +# Developer's Notes + If you just want to play around with the module, you may of course run the server script on its own. Before doing that, you would probably need to adjust the opt/emulated_vault/conf/cfg.json: 1. Changing the db-path to one you have access to @@ -20,4 +27,6 @@ Before doing that, you would probably need to adjust the opt/emulated_vault/conf Additionally, the vault-cmd script is also available to work against the DB with command line. It is mostly useful when developing a new adaptor. -For any question, please approach Nir B. Sopher (nir@apache.org) +# Contact + +For additional information, questions or assistance, please approach Nir B. Sopher (nir@apache.org) \ No newline at end of file diff --git a/experimental/emulated_vault/etc/systemd/system/emulated-vault.service b/experimental/emulated_vault/etc/systemd/system/emulated-vault.service index 8117ee6aaa..38b9991e00 100644 --- a/experimental/emulated_vault/etc/systemd/system/emulated-vault.service +++ b/experimental/emulated_vault/etc/systemd/system/emulated-vault.service @@ -1,3 +1,19 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# [Unit] Description=Emulated Vault Service After=network.target diff --git a/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py b/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py index 8b13789179..b887a3b472 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py +++ b/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py @@ -1 +1,16 @@ - +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# diff --git a/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py b/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py index 5dbfb10642..8bd78725b7 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py +++ b/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py @@ -1,3 +1,20 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + import os class Fs(object): diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-cmd b/experimental/emulated_vault/opt/emulated-vault/vault-cmd index 4c3c6a53b3..f11198e1a5 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault-cmd +++ b/experimental/emulated_vault/opt/emulated-vault/vault-cmd @@ -1,5 +1,22 @@ #! /usr/bin/python +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + import json import logging import optparse diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server index c528ce10dc..bf88607004 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -1,5 +1,22 @@ #! /usr/bin/python +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + import json import logging import sys From 04e8ba6b07db103e99e20c51e16d6c92f3db76d5 Mon Sep 17 00:00:00 2001 From: nirs Date: Sun, 17 Nov 2019 01:34:04 +0200 Subject: [PATCH 3/9] Emulated vault - code review fixes and further improvements. Supporting python 2 & 3 --- .../emulated_vault/{README => README.md} | 27 +- .../{adaptor => adapter}/__init__.py | 0 .../opt/emulated-vault/adapter/fs.py | 313 ++++++++++++++++++ .../opt/emulated-vault/adaptor/fs.py | 166 ---------- .../opt/emulated-vault/conf/cfg.ini | 13 + .../opt/emulated-vault/conf/cfg.json | 13 - .../emulated-vault/{vault-cmd => vault-debug} | 106 +++--- .../opt/emulated-vault/vault-server | 289 ++++++++++------ 8 files changed, 606 insertions(+), 321 deletions(-) rename experimental/emulated_vault/{README => README.md} (52%) rename experimental/emulated_vault/opt/emulated-vault/{adaptor => adapter}/__init__.py (100%) create mode 100644 experimental/emulated_vault/opt/emulated-vault/adapter/fs.py delete mode 100644 experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py create mode 100644 experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini delete mode 100644 experimental/emulated_vault/opt/emulated-vault/conf/cfg.json rename experimental/emulated_vault/opt/emulated-vault/{vault-cmd => vault-debug} (61%) diff --git a/experimental/emulated_vault/README b/experimental/emulated_vault/README.md similarity index 52% rename from experimental/emulated_vault/README rename to experimental/emulated_vault/README.md index 3bc01a16f8..c25ff5ba62 100644 --- a/experimental/emulated_vault/README +++ b/experimental/emulated_vault/README.md @@ -1,8 +1,27 @@ + + # Emulated Vault - Background The emulated_vault module supplies a HTTP server mimicking RIAK behavior for usage as traffic-control vault. It may be used in order to replace RIAK traffic_vault, as it is much more simple to install. -The server may use different type of persistent storage (e.g. file-system), using the proper adaptor. +The server may use different type of persistent storage (e.g. file-system), using the proper adapter. The resiliency of the stored keys is derived from the resiliency of the underlying storage. # Installation @@ -24,9 +43,9 @@ Before doing that, you would probably need to adjust the opt/emulated_vault/conf 1. Changing the db-path to one you have access to 2. Disable ssl (just to make it easier) -Additionally, the vault-cmd script is also available to work against the DB with command line. -It is mostly useful when developing a new adaptor. +Additionally, the vault-debug script is also available to work against the DB with command line. +It is mostly useful when developing a new adapter. # Contact -For additional information, questions or assistance, please approach Nir B. Sopher (nir@apache.org) \ No newline at end of file +For additional information, questions or assistance, please approach [Nir B. Sopher](mailto:nir@apache.org) diff --git a/experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py b/experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py similarity index 100% rename from experimental/emulated_vault/opt/emulated-vault/adaptor/__init__.py rename to experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py new file mode 100644 index 0000000000..33848971a5 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py @@ -0,0 +1,313 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +import os + +class Fs(object): + """ + Fs (file system) adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + This specific Adapter works using files upon a file-system. + Inputs configuration (held under "fs-adapter" section in the config file) include: + :param db-base-os-path: The path in which the DB files are stored + :param ping-os-path: The path of a variable mimicing the RIAK ping functionality + + Interface methods + :meth:`init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" boolean value + :meth:`init_ping` prepare the adapter for beign able to provide info to reply for + "ping" requests. Return "success" boolean value + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" boolean & "value" kept as ping variable + :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. + Return a tuple: "success" boolean & "value" kept in the parameter + :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding + variable key filters, and a key holding filters on the values as well. + Return "success" boolean indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`setParameter` given a parameter key (in url-path format) and a value string, + keep the parameter value. Return "success" boolean indicating a sucessful write + :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter + from the DB. Return "success" boolean indicating a sucessful deletion + """ + def __init__ (self, logger): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + """ + + self.logger = logger + + def init_cfg(self, fullConfig): + """ + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: Boolean + """ + myCfgData = dict(fullConfig.items("fs-adapter")) if fullConfig.has_section("fs-adapter") else {} + self.basePath = myCfgData.get("db-base-os-path") + if self.basePath is None: + self.logger.error("Missing %s/%s configuration", "fs-adapter", "db-base-os-path") + return False + self.pingOsPath = myCfgData.get("ping-os-path") + if self.pingOsPath is None: + self.logger.error("Missing %s/%s configuration", "fs-adapter", "ping-os-path") + return False + return True + + def init_ping(self): + """ + Initialize the class ability to answer for ping. Part of Adapter required API. + :return: 'True' for successful initialization + :rtype: Boolean + """ + value = "OK" + success = self._write_parameter_os_path(self.pingOsPath, value) + if not success: + self.logger.error("Failed to set parameter %s", self.pingOsPath) + return False + return True + + def ping(self): + """ + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + success, value = self._read_parameter_os_path(self.pingOsPath) + if not success or value is None: + self.logger.error("no ping response") + return (False, "") + self.logger.debug("ping response: %s", value) + return (True, value) + + def getParameter(self, parameterKeyUrlPath): + """ + Get value for the specified parameter. Part of Adapter required API. + :param parameterKeyUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyUrlPath: str + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + parameterOsPath = self._get_parameter_os_path(parameterKeyUrlPath) + success, value = self._read_parameter_os_path(parameterOsPath) + if not success: + self.logger.error("Failed to bring parameter %s", parameterKeyUrlPath) + return False, "" + if value is None: + self.logger.error("Could not find parameter %s", parameterKeyUrlPath) + return True, None + self.logger.debug("Parameter get response for %s ready", parameterKeyUrlPath) + return True, value + + def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): + """ + Get key/value dict of parameters by key pprefix + :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyPrefixUrlPath: str + :param keyFilters: a dictionary of filter-name/filter-functions - + each function gets a single variable - a parameter key - + and return "True" if the variable should be included in the response. + :type keyFilters: Dict[str, function[str]] + :param filters: a dictionary of filter-name/filter-functions - + each function gets 2 variable - a parameter key and val - + and return "True" if the variable should be included in the response. + :type filters: Dict[str, function[str,str]] + :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) + :rtype: Tuple[boolean, Dict[str, str]] + """ + self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) + parameterOsPathPrefix = self._get_parameter_os_path(parameterKeyPrefixUrlPath) + success, items = self._read_parameter_os_paths(parameterOsPathPrefix, keyFilters=keyFilters) + if not success: + self.logger.error("Failed to bring parameters by prefix %s", parameterOsPathPrefix) + return False, "" + + filtered = {} + for key, val in items.items():#items() - supporting python 2&3 + skip = False + for filterName, filterfunc in filters.items():#items() - supporting python 2&3 + if not filterfunc(key, val): + self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) + skip = True + break + if skip: + continue + filtered[key] = val + + return True, filtered + + def setParameter(self, parameterKeyUrlPath, value): + """ + Set value for the specified parameter. Part of Adapter required API. + :param parameterKeyUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyUrlPath: str + :param value: the value to be kept + :type value: str + :return: 'True' for successful settings + :rtype: Boolean + """ + self.logger.debug("Set parameter %s", parameterKeyUrlPath) + parameterOsPath = self._get_parameter_os_path(parameterKeyUrlPath) + success = self._write_parameter_os_path(parameterOsPath, value) + if not success: + self.logger.error("Failed to set parameter %s", parameterKeyUrlPath) + return False + return True + + def deleteParameter(self, parameterKeyUrlPath): + """ + Delete the specified parameter. Part of Adapter required API. + :param parameterKeyUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyUrlPath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + self.logger.debug("Delete parameter %s", parameterKeyUrlPath) + parameterOsPath = self._get_parameter_os_path(parameterKeyUrlPath) + success = self._remove_parameter_os_path(parameterOsPath) + if not success: + self.logger.error("Failed to delete parameter %s", parameterKeyUrlPath) + return False + return True + + def _get_parameter_os_path(self, parameterKeyUrlPath): + """ + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterKeyUrlPath: the "url-path" like key of the variable + :type parameterKeyUrlPath: str + :return: file path of where the value is be kept + :rtype: str + """ + return os.path.join(self.basePath, parameterKeyUrlPath.lstrip("/").replace("/", os.path.sep)) + + def _get_parameter_os_path_key_url_path(self, parameterOsPath): + """ + Conversion function - taking file path on the file system and translate to key's path + :param parameterOsPath: the file name holding a value + :type parameterKeyUrlPath: str + :return: the matching variable url-path like key + :rtype: str + """ + return "/"+os.path.relpath(parameterOsPath, self.basePath).replace(os.path.sep, "/") + + + def _read_parameter_os_path(self, parameterOsPath): + """ + Reading the value from the provided file name. + :param parameterOsPath: the file name + :type parameterKeyUrlPath: str + :return: 'True' for successful retrivaland the retrieved value + :rtype: Tuple[boolean, str] + """ + self.logger.debug("Get parameter by os path: %s", parameterOsPath) + try: + with open(parameterOsPath) as fd: + value = fd.read() + except: + self.logger.exception("%s parameter os path not found.", parameterOsPath) + return False, None + + self.logger.debug("Get parameter by os path %s succeed", parameterOsPath) + return True, value + + def _read_parameter_os_paths(self, parameterOsPathPrefix, keyFilters): + """ + Reading the values of the parameters the provided directory. + :param parameterOsPathPrefix: the directory to look into + :type parameterOsPathPrefix: str + :param keyFilters: filter-name/filter-func dict, holding functions that get a key as + input and retunn "true" if key should be included in the result + :type keyFilters: Dict[str,function[str]] + :return: 'True' for successful retrival and a dict for key-name/value + :rtype: Tuple[boolean, Dict[str, str]] + """ + self.logger.debug("Get parameters under os path %s", parameterOsPathPrefix) + parameters = {} + + fileNames = [] + for (dirpath, _, filenames) in os.walk(parameterOsPathPrefix): + fileNames += [os.path.join(dirpath, file) for file in filenames] + + for fileName in fileNames: + filteredOut = False + for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 + if not filterfunc(fileName): + self.logger.debug("Parameter os path %s dropped, not matching filter %s", self._get_parameter_os_path_key_url_path(fileName), filterName) + filteredOut = True + break + if filteredOut: + continue + + parameterKeyUrlPath = self._get_parameter_os_path_key_url_path(fileName) + success, value = self._read_parameter_os_path(fileName) + if not success: + self.logger.error("%s parameter os path not found.", fileName) + return False, None + + parameters[parameterKeyUrlPath] = value + + return True, parameters + + + def _write_parameter_os_path(self, parameterOsPath, value): + """ + Writing the value to the provided file name. + :param parameterOsPath: the file name + :type parameterKeyUrlPath: str + :param value: value to be writen + :type parameterKeyUrlPath: str + :return: 'True' for successful writing + :rtype: Boolean + """ + self.logger.debug("Set parameter by os path %s", parameterOsPath) + try: + dirname = os.path.dirname(parameterOsPath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + with open(parameterOsPath, "w") as fd: + fd.write(value) + except: + self.logger.exception("could not post parameter os path %s", parameterOsPath) + return False + self.logger.debug("Set parameter os path %s done", parameterOsPath) + return True + + def _remove_parameter_os_path(self, parameterOsPath): + """ + Deleting the the provided file. + :param parameterOsPath: the file name + :type parameterKeyUrlPath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + self.logger.debug("Delete parameter os path %s", parameterOsPath) + try: + os.remove(_parameterOsPath) + except: + self.logger.exception("could not delete parameter os path %s", parameterOsPath) + return False + self.logger.debug("Delete parameter os path %s done", parameterOsPath) + return True + diff --git a/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py b/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py deleted file mode 100644 index 8bd78725b7..0000000000 --- a/experimental/emulated_vault/opt/emulated-vault/adaptor/fs.py +++ /dev/null @@ -1,166 +0,0 @@ -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# - -import os - -class Fs(object): - def __init__ (self, logger, basePath, pingRelPath): - self.logger = logger - self.basePath = basePath - self.pingRelPath = pingRelPath - - def init_ping(self): - pingPath = os.path.join(self.basePath, self.pingRelPath) - value = "OK" - success = self._set_param(pingPath, value) - if not success: - self.logger.error("Failed to set variable %s", pingPath) - return False - return True - - def ping(self): - pingPath = os.path.join(self.basePath, self.pingRelPath) - success, value = self._get_param(pingPath) - if not success or value is None: - self.logger.error("no ping response") - return (False, "") - self.logger.debug("ping response: %s", value) - return (True, value) - - def getParameter(self, parameterPath): - parameterPath = parameterPath.lstrip("/") - _parameterPath = os.path.join(self.basePath, parameterPath) - success, value = self._get_param(_parameterPath) - if not success: - self.logger.error("Failed to bring variable %s", parameterPath) - return False, "" - if value is None: - self.logger.error("Could not find variable %s", parameterPath) - return True, None - self.logger.debug("Parameter get response for %s ready", parameterPath) - #self.logger.debug("Parameter get response: %s", value) - return True, value - - def searchParameters(self, parameterPathPrefix, keyFilters = {}, filters = {}): - parameterPathPrefix = parameterPathPrefix.lstrip("/") - _parametersPathPrefix = os.path.join(self.basePath, parameterPathPrefix) - success, items = self._get_params(_parametersPathPrefix, prefixToRemove=self.basePath, keyFilters=keyFilters) - if not success: - self.logger.error("Failed to bring variable by prefix %s", parameterPathPrefix) - return False, "" - - filtered = {} - for key, val in items.iteritems(): - skip = False - for filterName, filter in filters.iteritems(): - if not filter(key, val): - self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) - skip = True - break - if skip: - continue - filtered[key] = val - - return True, filtered - - - def setParameter(self, parameterPath, value): - parameterPath = parameterPath.lstrip("/") - _parameterPath = os.path.join(self.basePath, parameterPath) - success = self._set_param(_parameterPath, value) - if not success: - self.logger.error("Failed to set variable %s", parameterPath) - return False - return True - - def deleteParameter(self, parameterPath): - parameterPath = parameterPath.lstrip("/") - _parameterPath = os.path.join(self.basePath, parameterPath) - success = self._delete_param(_parameterPath) - if not success: - self.logger.error("Failed to delete variable %s", parameterPath) - return False - return True - - def _get_param(self, paramName): - self.logger.debug("Get parameter %s", paramName) - try: - with open(paramName) as fd: - value = fd.read() - except: - self.logger.exception("%s parameter not found.", paramName) - return False, None - - self.logger.debug("Get parameter %s succeed", paramName) - #self.logger.debug("Get parameter %s=%s", paramName, value) - return True, value - - def _get_params(self, paramNamePrefix, prefixToRemove = "", keyFilters={}): - self.logger.debug("Get parameters under path %s", paramNamePrefix) - prefixToRemoveLen = len(prefixToRemove) - parameters = {} - - fileNames = [] - for (dirpath, dirnames, filenames) in os.walk(paramNamePrefix): - fileNames += [os.path.join(dirpath, file) for file in filenames] - - for fileName in fileNames: - filteredOut = False - for filterName, filter in keyFilters.iteritems(): - if not filter(fileName): - self.logger.debug("Parameter %s dropped, not matching filter %s", fileName, filterName) - filteredOut = True - break - if filteredOut: - continue - - paramName = fileName[prefixToRemoveLen:] - success, value = self._get_param(fileName) - if not success: - self.logger.error("%s parameter not found.", paramName) - return False, None - - parameters[paramName] = value - - return True, parameters - - - def _set_param(self, paramName, value): - self.logger.debug("Set parameter %s", paramName) - #self.logger.debug("Set parameter %s=%s", paramName, value) - try: - dirname = os.path.dirname(paramName) - if not os.path.exists(dirname): - os.makedirs(dirname) - with open(paramName, "w") as fd: - fd.write(value) - except: - self.logger.exception("cloud not post parameter %s", paramName) - return False - self.logger.debug("Set parameter %s done", paramName) - return True - - def _delete_param(self, paramName): - self.logger.debug("Delete parameter %s", paramName) - try: - os.remove(paramName) - except: - self.logger.exception("cloud not delete parameter %s", paramName) - return False - self.logger.debug("Delete parameter %s done", paramName) - return True - diff --git a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini new file mode 100644 index 0000000000..de45397e39 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini @@ -0,0 +1,13 @@ +[general] +adapter-type = fs +# Optional: log-dir = /var/log/messages/emulated-vault +[http-server] +ssl-key-path = path/to/cert/key.pem +ssl-cert-path = path/to/cert/cert.pem +# Optional: use-ssl = False +# Optional: listen-ip = 1.2.3.4 +# Optional: listen-port = 12345 + +[fs-adapter] +db-base-os-path = /opt/emulated-vault/db +ping-os-path = /opt/emulated-vault/ping diff --git a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.json b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.json deleted file mode 100644 index e32e5a8ba2..0000000000 --- a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "data": { - "ping-rel-path": "ping", - "db-path": "/opt/emulated-vault/db/", - "adaptor-type": "fs", - "listen-ip": "0.0.0.0", - "listen-port": 8088, - "use-ssl": true, - "ssl-key-path": "path/to/cert/key.pem", - "ssl-cert-path": "path/to/cert/cert.pem" - }, - "format-version": 100 -} diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-cmd b/experimental/emulated_vault/opt/emulated-vault/vault-debug similarity index 61% rename from experimental/emulated_vault/opt/emulated-vault/vault-cmd rename to experimental/emulated_vault/opt/emulated-vault/vault-debug index f11198e1a5..75ade22a51 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault-cmd +++ b/experimental/emulated_vault/opt/emulated-vault/vault-debug @@ -17,14 +17,22 @@ # # +import inspect import json import logging import optparse import os import sys -import inspect -import adaptor.fs +import adapter.fs + +if sys.version_info >= (3, 0): + #python 3 + from configparser import ConfigParser +else: + #python 2 + from ConfigParser import ConfigParser + if __name__ == '__main__': progPath = inspect.stack()[-1][1] @@ -32,15 +40,30 @@ if __name__ == '__main__': progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) confDir = os.path.join(progAbsPath, "conf") - confFile = os.path.join(confDir, "cfg.json") - with open(confFile) as fd: - cfg = json.load(fd) + confFile = os.path.join(confDir, "cfg.ini") + try: + config = ConfigParser() + config.read(confFile) + except IOError as e: + sys.stderr.write("Failed to read configuration - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + sys.exit(1) + except Exception as e: + sys.stderr.write("Failed to read configuration: {0}\n".format(e)) + sys.exit(1) - logDir = os.path.join(progAbsPath, "var/log") + generalCfg = dict(config.items("general")) if config.has_section("general") else {} + logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") - if not os.path.exists(logDir): - os.makedirs(logDir) + try: + if not os.path.exists(logDir): + os.makedirs(logDir) + except IOError as e: + sys.stderr.write("Failed to create log dir - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + sys.exit(1) + except Exception as e: + sys.stderr.write("Failed to create log dir: {0}\n".format(e)) + sys.exit(1) global logger logger = logging.getLogger(__name__) @@ -87,68 +110,67 @@ if __name__ == '__main__': action="store_true", dest="searchParameters", default=False, help="Search parameter. Parameter relative path should be provided.") - standardOptions = optparse.OptionGroup(parser, "Standard operation options", - "Standard operation options.") - - parser.add_option_group(standardOptions) - - standardOptions.add_option("--parameter-path", type="string", - action="store", dest="parameterPath", help="Parameter path") - (options, args) = parser.parse_args() - - basePath = cfg["data"]["db-path"] - pingRelPath = cfg["data"]["ping-rel-path"] - adaptorType = cfg["data"]["adaptor-type"] - - if adaptorType == "fs": - adaptor = adaptor.fs.Fs(logger=logger, basePath=basePath, pingRelPath=pingRelPath) + global adapter + adapterType = generalCfg.get("adapter-type") + if not adapterType: + logger.error("Missing adapter type cfg") + sys.exit(1) + elif adapterType == "fs": + adapter = adapter.fs.Fs(logger=logger) else: - logger.error("Invalid adaptor type '%s'", adaptorType) + logger.error("Invalid adapter type '%s'", adapterType) sys.exit(1) + if not adapter.init_cfg(config): + logger.error("Failed adapter initialization") + sys.exit(1) + if not adapter.init_ping(): + logger.error("Failed adaper ping initialization") + sys.exit(1) + if options.ping: - success, value = adaptor.ping() + if len(args) != 0: + parser.error("Command should get no arguments.") + success, value = adapter.ping() if not success: sys.exit(1) - print value + print(value) sys.exit(0) if options.getParameter: - if not options.parameterPath: - parser.error("Missing parameter-path.") - success, value = adaptor.getParameter(options.parameterPath) + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success, value = adapter.getParameter(args[0]) if not success: sys.exit(1) if value is None: sys.exit(1) - print value + print(value) sys.exit(0) if options.searchParameters: - if not options.parameterPath: - parser.error("Missing parameter-path.") - success, value = adaptor.searchParameters(options.parameterPath) + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success, value = adapter.searchParameters(args[0], keyFilters={}, filters={}) if not success: sys.exit(1) - print json.dumps(value) + print(json.dumps(value)) sys.exit(0) if options.setParameter: - if not options.parameterPath: - parser.error("Missing parameter-path.") - if len(args) != 1: - parser.error("Command should get a single arg - the to be set value.") - success = adaptor.setParameter(options.parameterPath, args[0]) + if len(args) != 2: + parser.error("Command should get 2 arguments - parameter key and value.") + success = adapter.setParameter(args[0], args[1]) if not success: sys.exit(1) sys.exit(0) if options.deleteParameter: - if not options.parameterPath: - parser.error("Missing parameter-path.") - success = adaptor.deleteParameter(options.parameterPath) + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success = adapter.deleteParameter(args[0]) if not success: sys.exit(1) sys.exit(0) diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server index bf88607004..4b22c08b46 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -17,22 +17,42 @@ # # + +import fnmatch +import inspect import json import logging -import sys import os -import inspect -from BaseHTTPServer import HTTPServer -from BaseHTTPServer import BaseHTTPRequestHandler +import re import ssl -import urlparse, json -import re, fnmatch +import sys + +import adapter.fs + +if sys.version_info >= (3, 0): + #python 3 + from http.server import HTTPServer + from http.server import BaseHTTPRequestHandler + from configparser import ConfigParser + import urllib.parse as urlparse +else: + #python 2 + from BaseHTTPServer import HTTPServer + from BaseHTTPServer import BaseHTTPRequestHandler + from ConfigParser import ConfigParser + import urlparse -import adaptor.fs class RequestHandler(BaseHTTPRequestHandler): + """ + An HTTP server, emulating RIAK behavior on the calls done by traffic-ops. + The class implements BaseHTTPRequestHandler functions + """ def do_GET(self): + """ + Base class function implementation - to be called upon HTTP GET/HEAD requests + """ try: self._do_GET() except: @@ -41,6 +61,9 @@ class RequestHandler(BaseHTTPRequestHandler): do_HEAD = do_GET def do_POST(self): + """ + Base class function implementation - to be called upon HTTP POST/PUT requests + """ try: self._do_POST() except: @@ -49,102 +72,129 @@ class RequestHandler(BaseHTTPRequestHandler): do_PUT = do_POST def do_DELETE(self): + """ + Base class function implementation - to be called upon HTTP DEL request + """ try: self._do_DELETE() except: logger.exception("do_DELETE exception") def _do_GET(self): + """ + Actual GET logic + :raises: Exception + """ logger.debug("GET %s", self.path) parsed_path = urlparse.urlparse(self.path) if parsed_path.path == "/ping": logger.info("Ping") - success, value = adaptor.ping() - if not success: - self.send_response(503) - self.send_header('Content-type', 'text/html') - self.end_headers() - self.wfile.write("Failure") - return - - self.send_response(200) + self._do_GET_ping() + elif parsed_path.path == "/search/query/sslkeys": + logger.info("Search SSL: path=%s query=%s", parsed_path.path, parsed_path.query) + self._do_GET_sslkeys(parsed_path) + else: + self._do_GET_general(parsed_path) + + def _do_GET_ping(self): + """ + Actual GET logic for ping request + :raises: Exception + """ + success, value = adapter.ping() + if not success: + self.send_response(503) self.send_header('Content-type', 'text/html') self.end_headers() - self.wfile.write(value) + self.wfile.write("Failure".encode()) return - if parsed_path.path == "/search/query/sslkeys": - logger.info("Search SSL: path=%s query=%s", parsed_path.path, parsed_path.query) - filters = {} - keyFilters = {} - cdnFind = re.search(".*q=cdn:([^&]*).*", parsed_path.query) - if cdnFind: - def cdnFilter(key,val): - try: - data = json.loads(val) - return data['cdn']==cdnFind.group(1) - except: - return False - filters['cdn'] = cdnFilter - - dsFind = re.search(".*q=deliveryservice:([^&]*).*", parsed_path.query) - if dsFind: - def dsFilter(key,val): - try: - data = json.loads(val) - return data['deliveryservice']==dsFind.group(1) - except: - return False - filters['ds'] = dsFilter - - keyFind = re.search(".*q=_yz_rk:([^&]*).*", parsed_path.query) - if keyFind: - def keyFilter(key): - return fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) - keyFilters['key-match'] = keyFilter - - success, parameters = adaptor.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) - if not success: - self.send_response(503) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Failure"}') - return - - docs = [] - for key,val in parameters.iteritems(): - docs.append(json.loads(val)) - toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} - - self.send_response(200) + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(value.encode()) + + def _do_GET_sslkeys(self, parsed_path): + """ + Actual GET logic for ssl keys request + :param parsed_path: key's url path + :type parsed_path: str + :raises: Exception + """ + filters = {} + keyFilters = {} + cdnFind = re.search(".*q=cdn:([^&]*).*", parsed_path.query) + if cdnFind: + def cdnFilter(key,val): + try: + data = json.loads(val) + return data['cdn']==cdnFind.group(1) + except: + return False + filters['cdn'] = cdnFilter + + dsFind = re.search(".*q=deliveryservice:([^&]*).*", parsed_path.query) + if dsFind: + def dsFilter(key,val): + try: + data = json.loads(val) + return data['deliveryservice']==dsFind.group(1) + except: + return False + filters['ds'] = dsFilter + + keyFind = re.search(".*q=_yz_rk:([^&]*).*", parsed_path.query) + if keyFind: + keyFilters['key-match'] = lambda key: fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) + + success, parameters = adapter.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) + if not success: + self.send_response(503) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write(json.dumps(toReturn)) + self.wfile.write('{"Failure"}'.encode()) return - logger.info("GET %s", self.path) - success, value = adaptor.getParameter(parsed_path.path[1:]) + docs = [json.loads(val) for i in parameters.values()] + toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(toReturn).encode()) + + def _do_GET_general(self, parsed_path): + """ + Actual GET logic for general variable request + :param parsed_path: key's url path + :type parsed_path: str + :raises: Exception + """ + success, value = adapter.getParameter(parsed_path.path[1:]) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write('{"Failure"}') + self.wfile.write('{"Failure"}'.encode()) return if value is None: self.send_response(404) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write('{"Not found"}') + self.wfile.write('{"Not found"}'.encode()) return self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write(value) + self.wfile.write(value.encode()) def _do_POST(self): + """ + Actual POST request logic + """ logger.info("POST %s", self.path) parsed_path = urlparse.urlparse(self.path) content_len = int(self.headers.getheader('content-length')) @@ -154,12 +204,12 @@ class RequestHandler(BaseHTTPRequestHandler): if parsed_path.path.startswith('/riak/ssl/'): for item in data['certificate']: data['certificate.%s'%item]=data['certificate'][item] - success = adaptor.setParameter(parsed_path.path[1:], json.dumps(data)) + success = adapter.setParameter(parsed_path.path[1:], json.dumps(data)) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write('{"Failure"}') + self.wfile.write('{"Failure"}'.encode()) return self.send_response(204) @@ -169,21 +219,24 @@ class RequestHandler(BaseHTTPRequestHandler): def _do_DELETE(self): + """ + Actual DEL request logic + """ logger.info("DELETE %s", self.path) parsed_path = urlparse.urlparse(self.path) - success = adaptor.deleteParameter(parsed_path.path[1:]) + success = adapter.deleteParameter(parsed_path.path[1:]) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write('{"Failure"}') + self.wfile.write('{"Failure"}'.encode()) return self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() - self.wfile.write('{"Deleted"}') + self.wfile.write('{"Deleted"}'.encode()) return @@ -194,15 +247,31 @@ if __name__ == '__main__': progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) confDir = os.path.join(progAbsPath, "conf") - confFile = os.path.join(confDir, "cfg.json") - with open(confFile) as fd: - cfg = json.load(fd) + confFile = os.path.join(confDir, "cfg.ini") + try: + config = ConfigParser() + config.read(confFile) + except IOError as e: + sys.stderr.write("Failed to read configuration - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + sys.exit(1) + except Exception as e: + sys.stderr.write("Failed to read configuration: {0}\n".format(e)) + sys.exit(1) - logDir = os.path.join(progAbsPath, "var/log") + generalCfg = dict(config.items("general")) if config.has_section("general") else {} + logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") - if not os.path.exists(logDir): - os.makedirs(logDir) + try: + if not os.path.exists(logDir): + os.makedirs(logDir) + except IOError as e: + sys.stderr.write("Failed to create log dir - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + sys.exit(1) + except Exception as e: + sys.stderr.write("Failed to create log dir: {0}\n".format(e)) + sys.exit(1) + global logger logger = logging.getLogger(__name__) @@ -220,30 +289,58 @@ if __name__ == '__main__': logger.addHandler(fhm) logger.addHandler(fhd) - basePath = cfg["data"]["db-path"] - pingRelPath = cfg["data"]["ping-rel-path"] - listenIP = cfg["data"]["listen-ip"] - listenPort = cfg["data"]["listen-port"] - adaptorType = cfg["data"]["adaptor-type"] - use_ssl = cfg["data"]["use-ssl"] - - global adaptor - if adaptorType == "fs": - adaptor = adaptor.fs.Fs(logger=logger, basePath=basePath, pingRelPath=pingRelPath) + httpServerCfg = dict(config.items("http-server")) if config.has_section("http-server") else {} + listenIP = httpServerCfg.get("listen-ip", "0.0.0.0") + try: + listenPort = int(httpServerCfg.get("listen-port", "8088")) + except: + logger.exception("Failed %s integer conversion failed", "listen-port") + sys.exit(1) + use_ssl_val = httpServerCfg.get("use-ssl", "True") + if use_ssl_val in ["False", "false", "0"]: + use_ssl=False + elif use_ssl_val in ["True", "true", "1"]: + use_ssl=True else: - logger.error("Invalid adaptor type '%s'", adaptorType) + logger.error("Invalid %s value", "use-ssl") + sys.exit(1) + if use_ssl: + sslKey = httpServerCfg.get("ssl-key-path") + if not sslKey: + sys.stderr.write("Missing configuration: {0}/{1}\n".format("http-server", "ssl-key-path")) + sys.exit(1) + sslCert = httpServerCfg.get("ssl-cert-path") + if not sslCert: + sys.stderr.write("Missing configuration: {0}/{1}\n".format("http-server", "ssl-cert-path")) + sys.exit(1) + + global adapter + adapterType = generalCfg.get("adapter-type") + if not adapterType: + logger.error("Missing adapter type cfg") + sys.exit(1) + elif adapterType == "fs": + adapter = adapter.fs.Fs(logger=logger) + else: + logger.error("Invalid adapter type '%s'", adapterType) sys.exit(1) - if not adaptor.init_ping(): - logger.error("Failed initialization") + if not adapter.init_cfg(config): + logger.error("Failed adapter initialization") + sys.exit(1) + if not adapter.init_ping(): + logger.error("Failed adaper ping initialization") sys.exit(1) server = HTTPServer((listenIP, listenPort), RequestHandler) - if use_ssl: - key = cfg["data"]["ssl-key-path"] - cert = cfg["data"]["ssl-cert-path"] - server.socket = ssl.wrap_socket (server.socket, keyfile=key, certfile=cert, server_side=True) + if use_ssl: + try: + server.socket = ssl.wrap_socket (server.socket, keyfile=sslKey, certfile=sslCert, server_side=True) + except: + logger.exception("Failed on SSL init") + sys.exit(1) + msg = 'Starting server at http%s://%s:%d'%("s" if use_ssl else "", listenIP, listenPort) - print >> sys.stderr, msg + sys.stderr.write("%s\n"%msg) logger.info(msg) server.serve_forever() From e9c8bde3d00291fdec6475dfa7cfa4aa3b00f222 Mon Sep 17 00:00:00 2001 From: nirs Date: Mon, 18 Nov 2019 22:40:40 +0200 Subject: [PATCH 4/9] Emulated vault pr fixes --- .../opt/emulated-vault/adapter/fs.py | 6 +-- .../opt/emulated-vault/conf/cfg.ini | 17 ++++++++ .../opt/emulated-vault/vault-debug | 10 +++-- .../opt/emulated-vault/vault-server | 41 ++++++++++--------- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py index 33848971a5..56e52f1484 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py @@ -225,7 +225,7 @@ def _read_parameter_os_path(self, parameterOsPath): try: with open(parameterOsPath) as fd: value = fd.read() - except: + except OSError as e: self.logger.exception("%s parameter os path not found.", parameterOsPath) return False, None @@ -288,7 +288,7 @@ def _write_parameter_os_path(self, parameterOsPath, value): os.makedirs(dirname) with open(parameterOsPath, "w") as fd: fd.write(value) - except: + except OSError as e: self.logger.exception("could not post parameter os path %s", parameterOsPath) return False self.logger.debug("Set parameter os path %s done", parameterOsPath) @@ -305,7 +305,7 @@ def _remove_parameter_os_path(self, parameterOsPath): self.logger.debug("Delete parameter os path %s", parameterOsPath) try: os.remove(_parameterOsPath) - except: + except OSError as e: self.logger.exception("could not delete parameter os path %s", parameterOsPath) return False self.logger.debug("Delete parameter os path %s done", parameterOsPath) diff --git a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini index de45397e39..2f79fe0fb0 100644 --- a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini +++ b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini @@ -1,3 +1,20 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + [general] adapter-type = fs # Optional: log-dir = /var/log/messages/emulated-vault diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-debug b/experimental/emulated_vault/opt/emulated-vault/vault-debug index 75ade22a51..9db797f3a1 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault-debug +++ b/experimental/emulated_vault/opt/emulated-vault/vault-debug @@ -17,6 +17,8 @@ # # +from __future__ import print_function + import inspect import json import logging @@ -45,10 +47,10 @@ if __name__ == '__main__': config = ConfigParser() config.read(confFile) except IOError as e: - sys.stderr.write("Failed to read configuration - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) sys.exit(1) except Exception as e: - sys.stderr.write("Failed to read configuration: {0}\n".format(e)) + print("Failed to read configuration: {0}".format(e), file=sys.stderr) sys.exit(1) generalCfg = dict(config.items("general")) if config.has_section("general") else {} @@ -59,10 +61,10 @@ if __name__ == '__main__': if not os.path.exists(logDir): os.makedirs(logDir) except IOError as e: - sys.stderr.write("Failed to create log dir - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) sys.exit(1) except Exception as e: - sys.stderr.write("Failed to create log dir: {0}\n".format(e)) + print("Failed to create log dir: {0}".format(e), file=sys.stderr) sys.exit(1) global logger diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server index 4b22c08b46..0ecd9a1ade 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -17,7 +17,8 @@ # # - +from __future__ import print_function + import fnmatch import inspect import json @@ -55,7 +56,7 @@ class RequestHandler(BaseHTTPRequestHandler): """ try: self._do_GET() - except: + except Exception as e: logger.exception("do_GET exception") do_HEAD = do_GET @@ -66,7 +67,7 @@ class RequestHandler(BaseHTTPRequestHandler): """ try: self._do_POST() - except: + except Exception as e: logger.exception("do_POST exception") do_PUT = do_POST @@ -77,7 +78,7 @@ class RequestHandler(BaseHTTPRequestHandler): """ try: self._do_DELETE() - except: + except Exception as e: logger.exception("do_DELETE exception") def _do_GET(self): @@ -130,7 +131,7 @@ class RequestHandler(BaseHTTPRequestHandler): try: data = json.loads(val) return data['cdn']==cdnFind.group(1) - except: + except Exception as e: return False filters['cdn'] = cdnFilter @@ -140,7 +141,7 @@ class RequestHandler(BaseHTTPRequestHandler): try: data = json.loads(val) return data['deliveryservice']==dsFind.group(1) - except: + except Exception as e: return False filters['ds'] = dsFilter @@ -171,7 +172,7 @@ class RequestHandler(BaseHTTPRequestHandler): :type parsed_path: str :raises: Exception """ - success, value = adapter.getParameter(parsed_path.path[1:]) + success, value = adapter.getParameter(parsed_path.path) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -202,9 +203,9 @@ class RequestHandler(BaseHTTPRequestHandler): data = json.loads(post_body) #mimic vault beahvior if parsed_path.path.startswith('/riak/ssl/'): - for item in data['certificate']: - data['certificate.%s'%item]=data['certificate'][item] - success = adapter.setParameter(parsed_path.path[1:], json.dumps(data)) + certificate_data = data.get('certificate', {}) + data.update({"certificate.%s"%key: value for key, value in certificate_data.items()}) + success = adapter.setParameter(parsed_path.path, json.dumps(data)) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -225,7 +226,7 @@ class RequestHandler(BaseHTTPRequestHandler): logger.info("DELETE %s", self.path) parsed_path = urlparse.urlparse(self.path) - success = adapter.deleteParameter(parsed_path.path[1:]) + success = adapter.deleteParameter(parsed_path.path) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -252,10 +253,10 @@ if __name__ == '__main__': config = ConfigParser() config.read(confFile) except IOError as e: - sys.stderr.write("Failed to read configuration - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) sys.exit(1) except Exception as e: - sys.stderr.write("Failed to read configuration: {0}\n".format(e)) + print("Failed to read configuration: {0}".format(e), file=sys.stderr) sys.exit(1) generalCfg = dict(config.items("general")) if config.has_section("general") else {} @@ -266,10 +267,10 @@ if __name__ == '__main__': if not os.path.exists(logDir): os.makedirs(logDir) except IOError as e: - sys.stderr.write("Failed to create log dir - I/O error({0}): {1}\n".format(e.errno, e.strerror)) + print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) sys.exit(1) except Exception as e: - sys.stderr.write("Failed to create log dir: {0}\n".format(e)) + print("Failed to create log dir: {0}".format(e), file=sys.stderr) sys.exit(1) @@ -293,7 +294,7 @@ if __name__ == '__main__': listenIP = httpServerCfg.get("listen-ip", "0.0.0.0") try: listenPort = int(httpServerCfg.get("listen-port", "8088")) - except: + except Exception as e: logger.exception("Failed %s integer conversion failed", "listen-port") sys.exit(1) use_ssl_val = httpServerCfg.get("use-ssl", "True") @@ -307,11 +308,11 @@ if __name__ == '__main__': if use_ssl: sslKey = httpServerCfg.get("ssl-key-path") if not sslKey: - sys.stderr.write("Missing configuration: {0}/{1}\n".format("http-server", "ssl-key-path")) + print("Missing configuration: {0}/{1}".format("http-server", "ssl-key-path"), file=sys.stderr) sys.exit(1) sslCert = httpServerCfg.get("ssl-cert-path") if not sslCert: - sys.stderr.write("Missing configuration: {0}/{1}\n".format("http-server", "ssl-cert-path")) + print("Missing configuration: {0}/{1}".format("http-server", "ssl-cert-path"), file=sys.stderr) sys.exit(1) global adapter @@ -336,11 +337,11 @@ if __name__ == '__main__': if use_ssl: try: server.socket = ssl.wrap_socket (server.socket, keyfile=sslKey, certfile=sslCert, server_side=True) - except: + except Exception as e: logger.exception("Failed on SSL init") sys.exit(1) msg = 'Starting server at http%s://%s:%d'%("s" if use_ssl else "", listenIP, listenPort) - sys.stderr.write("%s\n"%msg) + print(msg, file=sys.stderr) logger.info(msg) server.serve_forever() From 79ade2862a05451b8485c1efb35e90d5bcd685c0 Mon Sep 17 00:00:00 2001 From: nirs Date: Tue, 19 Nov 2019 20:17:39 +0200 Subject: [PATCH 5/9] adaper base and derived --- .../opt/emulated-vault/adapter/base.py | 292 ++++++++++++++++++ .../opt/emulated-vault/adapter/fs.py | 246 +++++---------- 2 files changed, 369 insertions(+), 169 deletions(-) create mode 100644 experimental/emulated_vault/opt/emulated-vault/adapter/base.py diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/base.py b/experimental/emulated_vault/opt/emulated-vault/adapter/base.py new file mode 100644 index 0000000000..411cc0bc93 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/base.py @@ -0,0 +1,292 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +import os + +class Base(object): + """ + Base adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + + Implemented methods + :meth:`init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" boolean value + :meth:`init_ping` prepare the adapter for beign able to provide info to reply for + "ping" requests. Return "success" boolean value + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" boolean & "value" kept as ping variable + :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. + Return a tuple: "success" boolean & "value" kept in the parameter + :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding + variable key filters, and a key holding filters on the values as well. + Return "success" boolean indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`setParameter` given a parameter key (in url-path format) and a value string, + keep the parameter value. Return "success" boolean indicating a sucessful write + :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter + from the DB. Return "success" boolean indicating a sucessful deletion + + Methods to be implemented at derived classes: + :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path + :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter + retrun the url-key + :meth:`_init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" boolean value + :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for + "ping" requests. Return "success" boolean value + :meth:`_ping` test the 'ping' status with the adapter. + Return a tuple: "success" boolean & "value" kept as ping variable + :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. + Return a tuple: "success" boolean & "value" kept in the parameter + :meth:`_read_parameters_by_storage_path` given a storage and and a key holding + filters on the key and values. + Return "success" boolean indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`_write_parameter_by_storage_path` given a storage path and a value string, + keep the parameter value. Return "success" boolean indicating a sucessful write + :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter + from the DB. Return "success" boolean indicating a sucessful deletion + """ + def __init__ (self, logger): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + """ + + self.logger = logger + + def init_cfg(self, fullConfig): + """ + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: Boolean + """ + return self._init_cfg(fullConfig) + + def init_ping(self): + """ + Initialize the class ability to answer for ping. Part of Adapter required API. + :return: 'True' for successful initialization + :rtype: Boolean + """ + return self._init_ping() + + def ping(self): + """ + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + return self._ping() + + def getParameter(self, parameterUrlPath): + """ + Get value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) + success, value = self._read_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to bring parameter %s", parameterUrlPath) + return False, "" + if value is None: + self.logger.error("Could not find parameter %s", parameterUrlPath) + return True, None + self.logger.debug("Parameter get response for %s ready", parameterUrlPath) + return True, value + + def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): + """ + Get key/value dict of parameters by key pprefix + :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyPrefixUrlPath: str + :param keyFilters: a dictionary of filter-name/filter-functions - + each function gets a single variable - a parameter key - + and return "True" if the variable should be included in the response. + :type keyFilters: Dict[str, function[str]] + :param filters: a dictionary of filter-name/filter-functions - + each function gets 2 variable - a parameter key and val - + and return "True" if the variable should be included in the response. + :type filters: Dict[str, function[str,str]] + :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) + :rtype: Tuple[boolean, Dict[str, str]] + """ + self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) + parameterStoragePathPrefix = self._get_parameter_storage_path(parameterKeyPrefixUrlPath) + success, items = self._read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) + if not success: + self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) + return False, "" + + filtered = {} + for key, val in items.items():#items() - supporting python 2&3 + skip = False + for filterName, filterfunc in filters.items():#items() - supporting python 2&3 + if not filterfunc(key, val): + self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) + skip = True + break + if skip: + continue + filtered[key] = val + + return True, filtered + + def setParameter(self, parameterUrlPath, value): + """ + Set value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :param value: the value to be kept + :type value: str + :return: 'True' for successful settings + :rtype: Boolean + """ + self.logger.debug("Set parameter %s", parameterUrlPath) + parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) + success = self._write_parameter_by_storage_path(parameterStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", parameterUrlPath) + return False + return True + + def deleteParameter(self, parameterUrlPath): + """ + Delete the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + self.logger.debug("Delete parameter %s", parameterUrlPath) + parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) + success = self._remove_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to delete parameter %s", parameterUrlPath) + return False + return True + + def _init_cfg(self, fullConfig): + """ + Method to be implemented at derived classes + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: Boolean + """ + raise NotImplementedError() + + def _get_parameter_storage_path(self, parameterUrlPath): + """ + Method to be implemented at derived classes + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterUrlPath: the "url-path" like key of the variable + :type parameterUrlPath: str + :return: file path of where the value is be kept + :rtype: str + """ + raise NotImplementedError() + + def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): + """ + Method to be implemented at derived classes + Conversion function - taking file path on the file system and translate to key's path + :param parameterStoragePath: the file name holding a value + :type parameterUrlPath: str + :return: the matching variable url-path like key + :rtype: str + """ + raise NotImplementedError() + + def _init_ping(self): + """ + Method to be implemented at derived classes + Initialize the class ability to answer for ping. Part of Adapter required API. + :return: 'True' for successful initialization + :rtype: Boolean + """ + raise NotImplementedError() + + def _ping(self): + """ + Method to be implemented at derived classes + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + raise NotImplementedError() + + def _read_parameter_by_storage_path(self, parameterStoragePath): + """ + Method to be implemented at derived classes + Reading the value from the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful retrivaland the retrieved value + :rtype: Tuple[boolean, str] + """ + raise NotImplementedError() + + def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): + """ + Method to be implemented at derived classes + Reading the values of the parameters the provided directory. + :param parameterStoragePathPrefix: the directory to look into + :type parameterStoragePathPrefix: str + :param keyFilters: filter-name/filter-func dict, holding functions that get a key as + input and retunn "true" if key should be included in the result + :type keyFilters: Dict[str,function[str]] + :return: 'True' for successful retrival and a dict for key-name/value + :rtype: Tuple[boolean, Dict[str, str]] + """ + raise NotImplementedError() + + + def _write_parameter_by_storage_path(self, parameterStoragePath, value): + """ + Method to be implemented at derived classes + Writing the value to the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :param value: value to be writen + :type parameterUrlPath: str + :return: 'True' for successful writing + :rtype: Boolean + """ + raise NotImplementedError() + + def _remove_parameter_by_storage_path(self, parameterStoragePath): + """ + Method to be implemented at derived classes + Deleting the the provided file. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + raise NotImplementedError() \ No newline at end of file diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py index 56e52f1484..b941416be6 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py @@ -16,8 +16,9 @@ # import os +from . base import Base -class Fs(object): +class Fs(Base): """ Fs (file system) adapter class. This class implements the API required for storing and retriving the content kept in the vault. @@ -27,21 +28,24 @@ class Fs(object): :param ping-os-path: The path of a variable mimicing the RIAK ping functionality Interface methods - :meth:`init_cfg` given a config-parser object, read the parameters required for + :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path + :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter + retrun the url-key + :meth:`_init_cfg` given a config-parser object, read the parameters required for adapter's operation. Return "success" boolean value - :meth:`init_ping` prepare the adapter for beign able to provide info to reply for + :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for "ping" requests. Return "success" boolean value - :meth:`ping` test the 'ping' status with the adapter. + :meth:`_ping` test the 'ping' status with the adapter. Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. + :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding - variable key filters, and a key holding filters on the values as well. + :meth:`_read_parameters_by_storage_path` given a storage and and a key holding + filters on the key and values. Return "success" boolean indicating a sucessful write, and a key->value dictionary for the relevant parameters - :meth:`setParameter` given a parameter key (in url-path format) and a value string, + :meth:`_write_parameter_by_storage_path` given a storage path and a value string, keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter + :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter from the DB. Return "success" boolean indicating a sucessful deletion """ def __init__ (self, logger): @@ -50,10 +54,9 @@ def __init__ (self, logger): :param logger: logger to send log messages to :type logger: a python logging logger class """ - - self.logger = logger + Base.__init__(self, logger) - def init_cfg(self, fullConfig): + def _init_cfg(self, fullConfig): """ Initialize the class basic parameters. Part of Adapter required API. :param fullConfig: configuration to operate upon. @@ -66,248 +69,153 @@ def init_cfg(self, fullConfig): if self.basePath is None: self.logger.error("Missing %s/%s configuration", "fs-adapter", "db-base-os-path") return False - self.pingOsPath = myCfgData.get("ping-os-path") - if self.pingOsPath is None: + self.pingStoragePath = myCfgData.get("ping-os-path") + if self.pingStoragePath is None: self.logger.error("Missing %s/%s configuration", "fs-adapter", "ping-os-path") return False return True - def init_ping(self): + def _get_parameter_storage_path(self, parameterUrlPath): + """ + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterUrlPath: the "url-path" like key of the variable + :type parameterUrlPath: str + :return: file path of where the value is be kept + :rtype: str + """ + return os.path.join(self.basePath, parameterUrlPath.lstrip("/").replace("/", os.path.sep)) + + def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): + """ + Conversion function - taking file path on the file system and translate to key's path + :param parameterStoragePath: the file name holding a value + :type parameterUrlPath: str + :return: the matching variable url-path like key + :rtype: str + """ + return "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") + + def _init_ping(self): """ Initialize the class ability to answer for ping. Part of Adapter required API. :return: 'True' for successful initialization :rtype: Boolean """ value = "OK" - success = self._write_parameter_os_path(self.pingOsPath, value) + success = self._write_parameter_by_storage_path(self.pingStoragePath, value) if not success: - self.logger.error("Failed to set parameter %s", self.pingOsPath) + self.logger.error("Failed to set parameter %s", self.pingStoragePath) return False return True - def ping(self): + def _ping(self): """ get value for the ping request. Part of Adapter required API. :return: A tuple - 'True' for successful retrival and the retrieved value :rtype: Tuple[Boolean, str] """ - success, value = self._read_parameter_os_path(self.pingOsPath) + success, value = self._read_parameter_by_storage_path(self.pingStoragePath) if not success or value is None: self.logger.error("no ping response") return (False, "") self.logger.debug("ping response: %s", value) return (True, value) - def getParameter(self, parameterKeyUrlPath): - """ - Get value for the specified parameter. Part of Adapter required API. - :param parameterKeyUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterKeyUrlPath: str - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - parameterOsPath = self._get_parameter_os_path(parameterKeyUrlPath) - success, value = self._read_parameter_os_path(parameterOsPath) - if not success: - self.logger.error("Failed to bring parameter %s", parameterKeyUrlPath) - return False, "" - if value is None: - self.logger.error("Could not find parameter %s", parameterKeyUrlPath) - return True, None - self.logger.debug("Parameter get response for %s ready", parameterKeyUrlPath) - return True, value - - def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): - """ - Get key/value dict of parameters by key pprefix - :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterKeyPrefixUrlPath: str - :param keyFilters: a dictionary of filter-name/filter-functions - - each function gets a single variable - a parameter key - - and return "True" if the variable should be included in the response. - :type keyFilters: Dict[str, function[str]] - :param filters: a dictionary of filter-name/filter-functions - - each function gets 2 variable - a parameter key and val - - and return "True" if the variable should be included in the response. - :type filters: Dict[str, function[str,str]] - :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) - :rtype: Tuple[boolean, Dict[str, str]] - """ - self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) - parameterOsPathPrefix = self._get_parameter_os_path(parameterKeyPrefixUrlPath) - success, items = self._read_parameter_os_paths(parameterOsPathPrefix, keyFilters=keyFilters) - if not success: - self.logger.error("Failed to bring parameters by prefix %s", parameterOsPathPrefix) - return False, "" - - filtered = {} - for key, val in items.items():#items() - supporting python 2&3 - skip = False - for filterName, filterfunc in filters.items():#items() - supporting python 2&3 - if not filterfunc(key, val): - self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) - skip = True - break - if skip: - continue - filtered[key] = val - - return True, filtered - - def setParameter(self, parameterKeyUrlPath, value): - """ - Set value for the specified parameter. Part of Adapter required API. - :param parameterKeyUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterKeyUrlPath: str - :param value: the value to be kept - :type value: str - :return: 'True' for successful settings - :rtype: Boolean - """ - self.logger.debug("Set parameter %s", parameterKeyUrlPath) - parameterOsPath = self._get_parameter_os_path(parameterKeyUrlPath) - success = self._write_parameter_os_path(parameterOsPath, value) - if not success: - self.logger.error("Failed to set parameter %s", parameterKeyUrlPath) - return False - return True - - def deleteParameter(self, parameterKeyUrlPath): - """ - Delete the specified parameter. Part of Adapter required API. - :param parameterKeyUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterKeyUrlPath: str - :return: 'True' for successful deletion - :rtype: Boolean - """ - self.logger.debug("Delete parameter %s", parameterKeyUrlPath) - parameterOsPath = self._get_parameter_os_path(parameterKeyUrlPath) - success = self._remove_parameter_os_path(parameterOsPath) - if not success: - self.logger.error("Failed to delete parameter %s", parameterKeyUrlPath) - return False - return True - - def _get_parameter_os_path(self, parameterKeyUrlPath): - """ - Conversion function - taking a key's path and translate to a file path on the file system - :param parameterKeyUrlPath: the "url-path" like key of the variable - :type parameterKeyUrlPath: str - :return: file path of where the value is be kept - :rtype: str - """ - return os.path.join(self.basePath, parameterKeyUrlPath.lstrip("/").replace("/", os.path.sep)) - - def _get_parameter_os_path_key_url_path(self, parameterOsPath): - """ - Conversion function - taking file path on the file system and translate to key's path - :param parameterOsPath: the file name holding a value - :type parameterKeyUrlPath: str - :return: the matching variable url-path like key - :rtype: str - """ - return "/"+os.path.relpath(parameterOsPath, self.basePath).replace(os.path.sep, "/") - - - def _read_parameter_os_path(self, parameterOsPath): + def _read_parameter_by_storage_path(self, parameterStoragePath): """ Reading the value from the provided file name. - :param parameterOsPath: the file name - :type parameterKeyUrlPath: str + :param parameterStoragePath: the file name + :type parameterStoragePath: str :return: 'True' for successful retrivaland the retrieved value :rtype: Tuple[boolean, str] """ - self.logger.debug("Get parameter by os path: %s", parameterOsPath) + self.logger.debug("Get parameter by os path: %s", parameterStoragePath) try: - with open(parameterOsPath) as fd: + with open(parameterStoragePath) as fd: value = fd.read() - except OSError as e: - self.logger.exception("%s parameter os path not found.", parameterOsPath) + except Exception as e: + self.logger.exception("%s parameter os path not found.", parameterStoragePath) return False, None - self.logger.debug("Get parameter by os path %s succeed", parameterOsPath) + self.logger.debug("Get parameter by os path %s succeed", parameterStoragePath) return True, value - def _read_parameter_os_paths(self, parameterOsPathPrefix, keyFilters): + def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): """ Reading the values of the parameters the provided directory. - :param parameterOsPathPrefix: the directory to look into - :type parameterOsPathPrefix: str + :param parameterStoragePathPrefix: the directory to look into + :type parameterStoragePathPrefix: str :param keyFilters: filter-name/filter-func dict, holding functions that get a key as input and retunn "true" if key should be included in the result :type keyFilters: Dict[str,function[str]] :return: 'True' for successful retrival and a dict for key-name/value :rtype: Tuple[boolean, Dict[str, str]] """ - self.logger.debug("Get parameters under os path %s", parameterOsPathPrefix) + self.logger.debug("Get parameters under os path %s", parameterStoragePathPrefix) parameters = {} fileNames = [] - for (dirpath, _, filenames) in os.walk(parameterOsPathPrefix): + for (dirpath, _, filenames) in os.walk(parameterStoragePathPrefix): fileNames += [os.path.join(dirpath, file) for file in filenames] for fileName in fileNames: filteredOut = False for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 if not filterfunc(fileName): - self.logger.debug("Parameter os path %s dropped, not matching filter %s", self._get_parameter_os_path_key_url_path(fileName), filterName) + self.logger.debug("Parameter os path %s dropped, not matching filter %s", self._get_parameter_storage_path_from_url_path(fileName), filterName) filteredOut = True break if filteredOut: continue - parameterKeyUrlPath = self._get_parameter_os_path_key_url_path(fileName) - success, value = self._read_parameter_os_path(fileName) + parameterUrlPath = self._get_parameter_storage_path_from_url_path(fileName) + success, value = self._read_parameter_by_storage_path(fileName) if not success: self.logger.error("%s parameter os path not found.", fileName) return False, None - parameters[parameterKeyUrlPath] = value + parameters[parameterUrlPath] = value return True, parameters - def _write_parameter_os_path(self, parameterOsPath, value): + def _write_parameter_by_storage_path(self, parameterStoragePath, value): """ Writing the value to the provided file name. - :param parameterOsPath: the file name - :type parameterKeyUrlPath: str + :param parameterStoragePath: the file name + :type parameterStoragePath: str :param value: value to be writen - :type parameterKeyUrlPath: str + :type value: str :return: 'True' for successful writing :rtype: Boolean """ - self.logger.debug("Set parameter by os path %s", parameterOsPath) + self.logger.debug("Set parameter by os path %s", parameterStoragePath) try: - dirname = os.path.dirname(parameterOsPath) + dirname = os.path.dirname(parameterStoragePath) if dirname and not os.path.exists(dirname): os.makedirs(dirname) - with open(parameterOsPath, "w") as fd: + with open(parameterStoragePath, "w") as fd: fd.write(value) - except OSError as e: - self.logger.exception("could not post parameter os path %s", parameterOsPath) + except Exception as e: + self.logger.exception("could not post parameter os path %s", parameterStoragePath) return False - self.logger.debug("Set parameter os path %s done", parameterOsPath) + self.logger.debug("Set parameter os path %s done", parameterStoragePath) return True - def _remove_parameter_os_path(self, parameterOsPath): + def _remove_parameter_by_storage_path(self, parameterStoragePath): """ Deleting the the provided file. - :param parameterOsPath: the file name - :type parameterKeyUrlPath: str + :param parameterStoragePath: the file name + :type parameterStoragePath: str :return: 'True' for successful deletion :rtype: Boolean """ - self.logger.debug("Delete parameter os path %s", parameterOsPath) + self.logger.debug("Delete parameter os path %s", parameterStoragePath) try: - os.remove(_parameterOsPath) - except OSError as e: - self.logger.exception("could not delete parameter os path %s", parameterOsPath) + os.remove(_parameterStoragePath) + except Exception as e: + self.logger.exception("could not delete parameter os path %s", parameterStoragePath) return False - self.logger.debug("Delete parameter os path %s done", parameterOsPath) + self.logger.debug("Delete parameter os path %s done", parameterStoragePath) return True From 3eb1beff2fc1093dd94d4c13f9825a20037c3b9f Mon Sep 17 00:00:00 2001 From: nirs Date: Tue, 19 Nov 2019 22:41:31 +0200 Subject: [PATCH 6/9] Emulated vault - happier linter --- .../opt/emulated-vault/adapter/__init__.py | 2 +- .../opt/emulated-vault/adapter/base.py | 506 +++++++------- .../opt/emulated-vault/adapter/fs.py | 400 +++++------ .../opt/emulated-vault/vault-debug | 308 ++++----- .../opt/emulated-vault/vault-server | 624 +++++++++--------- 5 files changed, 922 insertions(+), 918 deletions(-) mode change 100644 => 100755 experimental/emulated_vault/opt/emulated-vault/vault-debug mode change 100644 => 100755 experimental/emulated_vault/opt/emulated-vault/vault-server diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py b/experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py index b887a3b472..dc9911a6d6 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/base.py b/experimental/emulated_vault/opt/emulated-vault/adapter/base.py index 411cc0bc93..af6cf80062 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/base.py +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/base.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -18,275 +18,275 @@ import os class Base(object): - """ - Base adapter class. - This class implements the API required for storing and retriving the content kept in the vault. - - Implemented methods - :meth:`init_cfg` given a config-parser object, read the parameters required for - adapter's operation. Return "success" boolean value - :meth:`init_ping` prepare the adapter for beign able to provide info to reply for - "ping" requests. Return "success" boolean value - :meth:`ping` test the 'ping' status with the adapter. - Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. - Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding - variable key filters, and a key holding filters on the values as well. - Return "success" boolean indicating a sucessful write, and a key->value dictionary - for the relevant parameters - :meth:`setParameter` given a parameter key (in url-path format) and a value string, - keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter - from the DB. Return "success" boolean indicating a sucessful deletion + """ + Base adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + + Implemented methods + :meth:`init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" boolean value + :meth:`init_ping` prepare the adapter for beign able to provide info to reply for + "ping" requests. Return "success" boolean value + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" boolean & "value" kept as ping variable + :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. + Return a tuple: "success" boolean & "value" kept in the parameter + :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding + variable key filters, and a key holding filters on the values as well. + Return "success" boolean indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`setParameter` given a parameter key (in url-path format) and a value string, + keep the parameter value. Return "success" boolean indicating a sucessful write + :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter + from the DB. Return "success" boolean indicating a sucessful deletion - Methods to be implemented at derived classes: - :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path - :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter - retrun the url-key - :meth:`_init_cfg` given a config-parser object, read the parameters required for - adapter's operation. Return "success" boolean value - :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for - "ping" requests. Return "success" boolean value - :meth:`_ping` test the 'ping' status with the adapter. - Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. - Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`_read_parameters_by_storage_path` given a storage and and a key holding - filters on the key and values. - Return "success" boolean indicating a sucessful write, and a key->value dictionary - for the relevant parameters - :meth:`_write_parameter_by_storage_path` given a storage path and a value string, - keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter - from the DB. Return "success" boolean indicating a sucessful deletion - """ - def __init__ (self, logger): - """ - The class constructor. - :param logger: logger to send log messages to - :type logger: a python logging logger class - """ - - self.logger = logger + Methods to be implemented at derived classes: + :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path + :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter + retrun the url-key + :meth:`_init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" boolean value + :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for + "ping" requests. Return "success" boolean value + :meth:`_ping` test the 'ping' status with the adapter. + Return a tuple: "success" boolean & "value" kept as ping variable + :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. + Return a tuple: "success" boolean & "value" kept in the parameter + :meth:`_read_parameters_by_storage_path` given a storage and and a key holding + filters on the key and values. + Return "success" boolean indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`_write_parameter_by_storage_path` given a storage path and a value string, + keep the parameter value. Return "success" boolean indicating a sucessful write + :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter + from the DB. Return "success" boolean indicating a sucessful deletion + """ + def __init__ (self, logger): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + """ + + self.logger = logger - def init_cfg(self, fullConfig): - """ - Initialize the class basic parameters. Part of Adapter required API. - :param fullConfig: configuration to operate upon. - :type fullConfig: configparser.ConfigParser class - :return: 'True' for successful initialization - :rtype: Boolean - """ - return self._init_cfg(fullConfig) + def init_cfg(self, fullConfig): + """ + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: Boolean + """ + return self._init_cfg(fullConfig) - def init_ping(self): - """ - Initialize the class ability to answer for ping. Part of Adapter required API. - :return: 'True' for successful initialization - :rtype: Boolean - """ - return self._init_ping() + def init_ping(self): + """ + Initialize the class ability to answer for ping. Part of Adapter required API. + :return: 'True' for successful initialization + :rtype: Boolean + """ + return self._init_ping() - def ping(self): - """ - get value for the ping request. Part of Adapter required API. - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - return self._ping() + def ping(self): + """ + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + return self._ping() - def getParameter(self, parameterUrlPath): - """ - Get value for the specified parameter. Part of Adapter required API. - :param parameterUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterUrlPath: str - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) - success, value = self._read_parameter_by_storage_path(parameterStoragePath) - if not success: - self.logger.error("Failed to bring parameter %s", parameterUrlPath) - return False, "" - if value is None: - self.logger.error("Could not find parameter %s", parameterUrlPath) - return True, None - self.logger.debug("Parameter get response for %s ready", parameterUrlPath) - return True, value + def getParameter(self, parameterUrlPath): + """ + Get value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) + success, value = self._read_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to bring parameter %s", parameterUrlPath) + return False, "" + if value is None: + self.logger.error("Could not find parameter %s", parameterUrlPath) + return True, None + self.logger.debug("Parameter get response for %s ready", parameterUrlPath) + return True, value - def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): - """ - Get key/value dict of parameters by key pprefix - :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterKeyPrefixUrlPath: str - :param keyFilters: a dictionary of filter-name/filter-functions - - each function gets a single variable - a parameter key - - and return "True" if the variable should be included in the response. - :type keyFilters: Dict[str, function[str]] - :param filters: a dictionary of filter-name/filter-functions - - each function gets 2 variable - a parameter key and val - - and return "True" if the variable should be included in the response. - :type filters: Dict[str, function[str,str]] - :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) - :rtype: Tuple[boolean, Dict[str, str]] - """ - self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) - parameterStoragePathPrefix = self._get_parameter_storage_path(parameterKeyPrefixUrlPath) - success, items = self._read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) - if not success: - self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) - return False, "" + def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): + """ + Get key/value dict of parameters by key pprefix + :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyPrefixUrlPath: str + :param keyFilters: a dictionary of filter-name/filter-functions - + each function gets a single variable - a parameter key - + and return "True" if the variable should be included in the response. + :type keyFilters: Dict[str, function[str]] + :param filters: a dictionary of filter-name/filter-functions - + each function gets 2 variable - a parameter key and val - + and return "True" if the variable should be included in the response. + :type filters: Dict[str, function[str,str]] + :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) + :rtype: Tuple[boolean, Dict[str, str]] + """ + self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) + parameterStoragePathPrefix = self._get_parameter_storage_path(parameterKeyPrefixUrlPath) + success, items = self._read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) + if not success: + self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) + return False, "" - filtered = {} - for key, val in items.items():#items() - supporting python 2&3 - skip = False - for filterName, filterfunc in filters.items():#items() - supporting python 2&3 - if not filterfunc(key, val): - self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) - skip = True - break - if skip: - continue - filtered[key] = val + filtered = {} + for key, val in items.items():#items() - supporting python 2&3 + skip = False + for filterName, filterfunc in filters.items():#items() - supporting python 2&3 + if not filterfunc(key, val): + self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) + skip = True + break + if skip: + continue + filtered[key] = val - return True, filtered + return True, filtered - def setParameter(self, parameterUrlPath, value): - """ - Set value for the specified parameter. Part of Adapter required API. - :param parameterUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterUrlPath: str - :param value: the value to be kept - :type value: str - :return: 'True' for successful settings - :rtype: Boolean - """ - self.logger.debug("Set parameter %s", parameterUrlPath) - parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) - success = self._write_parameter_by_storage_path(parameterStoragePath, value) - if not success: - self.logger.error("Failed to set parameter %s", parameterUrlPath) - return False - return True + def setParameter(self, parameterUrlPath, value): + """ + Set value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :param value: the value to be kept + :type value: str + :return: 'True' for successful settings + :rtype: Boolean + """ + self.logger.debug("Set parameter %s", parameterUrlPath) + parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) + success = self._write_parameter_by_storage_path(parameterStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", parameterUrlPath) + return False + return True - def deleteParameter(self, parameterUrlPath): - """ - Delete the specified parameter. Part of Adapter required API. - :param parameterUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterUrlPath: str - :return: 'True' for successful deletion - :rtype: Boolean - """ - self.logger.debug("Delete parameter %s", parameterUrlPath) - parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) - success = self._remove_parameter_by_storage_path(parameterStoragePath) - if not success: - self.logger.error("Failed to delete parameter %s", parameterUrlPath) - return False - return True + def deleteParameter(self, parameterUrlPath): + """ + Delete the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + self.logger.debug("Delete parameter %s", parameterUrlPath) + parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) + success = self._remove_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to delete parameter %s", parameterUrlPath) + return False + return True - def _init_cfg(self, fullConfig): - """ - Method to be implemented at derived classes - Initialize the class basic parameters. Part of Adapter required API. - :param fullConfig: configuration to operate upon. - :type fullConfig: configparser.ConfigParser class - :return: 'True' for successful initialization - :rtype: Boolean - """ - raise NotImplementedError() + def _init_cfg(self, fullConfig): + """ + Method to be implemented at derived classes + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: Boolean + """ + raise NotImplementedError() - def _get_parameter_storage_path(self, parameterUrlPath): - """ - Method to be implemented at derived classes - Conversion function - taking a key's path and translate to a file path on the file system - :param parameterUrlPath: the "url-path" like key of the variable - :type parameterUrlPath: str - :return: file path of where the value is be kept - :rtype: str - """ - raise NotImplementedError() + def _get_parameter_storage_path(self, parameterUrlPath): + """ + Method to be implemented at derived classes + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterUrlPath: the "url-path" like key of the variable + :type parameterUrlPath: str + :return: file path of where the value is be kept + :rtype: str + """ + raise NotImplementedError() - def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): - """ - Method to be implemented at derived classes - Conversion function - taking file path on the file system and translate to key's path - :param parameterStoragePath: the file name holding a value - :type parameterUrlPath: str - :return: the matching variable url-path like key - :rtype: str - """ - raise NotImplementedError() + def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): + """ + Method to be implemented at derived classes + Conversion function - taking file path on the file system and translate to key's path + :param parameterStoragePath: the file name holding a value + :type parameterUrlPath: str + :return: the matching variable url-path like key + :rtype: str + """ + raise NotImplementedError() - def _init_ping(self): - """ - Method to be implemented at derived classes - Initialize the class ability to answer for ping. Part of Adapter required API. - :return: 'True' for successful initialization - :rtype: Boolean - """ - raise NotImplementedError() + def _init_ping(self): + """ + Method to be implemented at derived classes + Initialize the class ability to answer for ping. Part of Adapter required API. + :return: 'True' for successful initialization + :rtype: Boolean + """ + raise NotImplementedError() - def _ping(self): - """ - Method to be implemented at derived classes - get value for the ping request. Part of Adapter required API. - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - raise NotImplementedError() + def _ping(self): + """ + Method to be implemented at derived classes + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + raise NotImplementedError() - def _read_parameter_by_storage_path(self, parameterStoragePath): - """ - Method to be implemented at derived classes - Reading the value from the provided file name. - :param parameterStoragePath: the file name - :type parameterUrlPath: str - :return: 'True' for successful retrivaland the retrieved value - :rtype: Tuple[boolean, str] - """ - raise NotImplementedError() + def _read_parameter_by_storage_path(self, parameterStoragePath): + """ + Method to be implemented at derived classes + Reading the value from the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful retrivaland the retrieved value + :rtype: Tuple[boolean, str] + """ + raise NotImplementedError() - def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): - """ - Method to be implemented at derived classes - Reading the values of the parameters the provided directory. - :param parameterStoragePathPrefix: the directory to look into - :type parameterStoragePathPrefix: str - :param keyFilters: filter-name/filter-func dict, holding functions that get a key as - input and retunn "true" if key should be included in the result - :type keyFilters: Dict[str,function[str]] - :return: 'True' for successful retrival and a dict for key-name/value - :rtype: Tuple[boolean, Dict[str, str]] - """ - raise NotImplementedError() + def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): + """ + Method to be implemented at derived classes + Reading the values of the parameters the provided directory. + :param parameterStoragePathPrefix: the directory to look into + :type parameterStoragePathPrefix: str + :param keyFilters: filter-name/filter-func dict, holding functions that get a key as + input and retunn "true" if key should be included in the result + :type keyFilters: Dict[str,function[str]] + :return: 'True' for successful retrival and a dict for key-name/value + :rtype: Tuple[boolean, Dict[str, str]] + """ + raise NotImplementedError() - def _write_parameter_by_storage_path(self, parameterStoragePath, value): - """ - Method to be implemented at derived classes - Writing the value to the provided file name. - :param parameterStoragePath: the file name - :type parameterUrlPath: str - :param value: value to be writen - :type parameterUrlPath: str - :return: 'True' for successful writing - :rtype: Boolean - """ - raise NotImplementedError() + def _write_parameter_by_storage_path(self, parameterStoragePath, value): + """ + Method to be implemented at derived classes + Writing the value to the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :param value: value to be writen + :type parameterUrlPath: str + :return: 'True' for successful writing + :rtype: Boolean + """ + raise NotImplementedError() - def _remove_parameter_by_storage_path(self, parameterStoragePath): - """ - Method to be implemented at derived classes - Deleting the the provided file. - :param parameterStoragePath: the file name - :type parameterUrlPath: str - :return: 'True' for successful deletion - :rtype: Boolean - """ - raise NotImplementedError() \ No newline at end of file + def _remove_parameter_by_storage_path(self, parameterStoragePath): + """ + Method to be implemented at derived classes + Deleting the the provided file. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + raise NotImplementedError() \ No newline at end of file diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py index b941416be6..295ead4e29 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py +++ b/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -19,203 +19,203 @@ from . base import Base class Fs(Base): - """ - Fs (file system) adapter class. - This class implements the API required for storing and retriving the content kept in the vault. - This specific Adapter works using files upon a file-system. - Inputs configuration (held under "fs-adapter" section in the config file) include: - :param db-base-os-path: The path in which the DB files are stored - :param ping-os-path: The path of a variable mimicing the RIAK ping functionality - - Interface methods - :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path - :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter - retrun the url-key - :meth:`_init_cfg` given a config-parser object, read the parameters required for - adapter's operation. Return "success" boolean value - :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for - "ping" requests. Return "success" boolean value - :meth:`_ping` test the 'ping' status with the adapter. - Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. - Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`_read_parameters_by_storage_path` given a storage and and a key holding - filters on the key and values. - Return "success" boolean indicating a sucessful write, and a key->value dictionary - for the relevant parameters - :meth:`_write_parameter_by_storage_path` given a storage path and a value string, - keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter - from the DB. Return "success" boolean indicating a sucessful deletion - """ - def __init__ (self, logger): - """ - The class constructor. - :param logger: logger to send log messages to - :type logger: a python logging logger class - """ - Base.__init__(self, logger) - - def _init_cfg(self, fullConfig): - """ - Initialize the class basic parameters. Part of Adapter required API. - :param fullConfig: configuration to operate upon. - :type fullConfig: configparser.ConfigParser class - :return: 'True' for successful initialization - :rtype: Boolean - """ - myCfgData = dict(fullConfig.items("fs-adapter")) if fullConfig.has_section("fs-adapter") else {} - self.basePath = myCfgData.get("db-base-os-path") - if self.basePath is None: - self.logger.error("Missing %s/%s configuration", "fs-adapter", "db-base-os-path") - return False - self.pingStoragePath = myCfgData.get("ping-os-path") - if self.pingStoragePath is None: - self.logger.error("Missing %s/%s configuration", "fs-adapter", "ping-os-path") - return False - return True - - def _get_parameter_storage_path(self, parameterUrlPath): - """ - Conversion function - taking a key's path and translate to a file path on the file system - :param parameterUrlPath: the "url-path" like key of the variable - :type parameterUrlPath: str - :return: file path of where the value is be kept - :rtype: str - """ - return os.path.join(self.basePath, parameterUrlPath.lstrip("/").replace("/", os.path.sep)) - - def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): - """ - Conversion function - taking file path on the file system and translate to key's path - :param parameterStoragePath: the file name holding a value - :type parameterUrlPath: str - :return: the matching variable url-path like key - :rtype: str - """ - return "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") - - def _init_ping(self): - """ - Initialize the class ability to answer for ping. Part of Adapter required API. - :return: 'True' for successful initialization - :rtype: Boolean - """ - value = "OK" - success = self._write_parameter_by_storage_path(self.pingStoragePath, value) - if not success: - self.logger.error("Failed to set parameter %s", self.pingStoragePath) - return False - return True - - def _ping(self): - """ - get value for the ping request. Part of Adapter required API. - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - success, value = self._read_parameter_by_storage_path(self.pingStoragePath) - if not success or value is None: - self.logger.error("no ping response") - return (False, "") - self.logger.debug("ping response: %s", value) - return (True, value) - - def _read_parameter_by_storage_path(self, parameterStoragePath): - """ - Reading the value from the provided file name. - :param parameterStoragePath: the file name - :type parameterStoragePath: str - :return: 'True' for successful retrivaland the retrieved value - :rtype: Tuple[boolean, str] - """ - self.logger.debug("Get parameter by os path: %s", parameterStoragePath) - try: - with open(parameterStoragePath) as fd: - value = fd.read() - except Exception as e: - self.logger.exception("%s parameter os path not found.", parameterStoragePath) - return False, None - - self.logger.debug("Get parameter by os path %s succeed", parameterStoragePath) - return True, value - - def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): - """ - Reading the values of the parameters the provided directory. - :param parameterStoragePathPrefix: the directory to look into - :type parameterStoragePathPrefix: str - :param keyFilters: filter-name/filter-func dict, holding functions that get a key as - input and retunn "true" if key should be included in the result - :type keyFilters: Dict[str,function[str]] - :return: 'True' for successful retrival and a dict for key-name/value - :rtype: Tuple[boolean, Dict[str, str]] - """ - self.logger.debug("Get parameters under os path %s", parameterStoragePathPrefix) - parameters = {} - - fileNames = [] - for (dirpath, _, filenames) in os.walk(parameterStoragePathPrefix): - fileNames += [os.path.join(dirpath, file) for file in filenames] - - for fileName in fileNames: - filteredOut = False - for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 - if not filterfunc(fileName): - self.logger.debug("Parameter os path %s dropped, not matching filter %s", self._get_parameter_storage_path_from_url_path(fileName), filterName) - filteredOut = True - break - if filteredOut: - continue - - parameterUrlPath = self._get_parameter_storage_path_from_url_path(fileName) - success, value = self._read_parameter_by_storage_path(fileName) - if not success: - self.logger.error("%s parameter os path not found.", fileName) - return False, None - - parameters[parameterUrlPath] = value - - return True, parameters - - - def _write_parameter_by_storage_path(self, parameterStoragePath, value): - """ - Writing the value to the provided file name. - :param parameterStoragePath: the file name - :type parameterStoragePath: str - :param value: value to be writen - :type value: str - :return: 'True' for successful writing - :rtype: Boolean - """ - self.logger.debug("Set parameter by os path %s", parameterStoragePath) - try: - dirname = os.path.dirname(parameterStoragePath) - if dirname and not os.path.exists(dirname): - os.makedirs(dirname) - with open(parameterStoragePath, "w") as fd: - fd.write(value) - except Exception as e: - self.logger.exception("could not post parameter os path %s", parameterStoragePath) - return False - self.logger.debug("Set parameter os path %s done", parameterStoragePath) - return True - - def _remove_parameter_by_storage_path(self, parameterStoragePath): - """ - Deleting the the provided file. - :param parameterStoragePath: the file name - :type parameterStoragePath: str - :return: 'True' for successful deletion - :rtype: Boolean - """ - self.logger.debug("Delete parameter os path %s", parameterStoragePath) - try: - os.remove(_parameterStoragePath) - except Exception as e: - self.logger.exception("could not delete parameter os path %s", parameterStoragePath) - return False - self.logger.debug("Delete parameter os path %s done", parameterStoragePath) - return True + """ + Fs (file system) adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + This specific Adapter works using files upon a file-system. + Inputs configuration (held under "fs-adapter" section in the config file) include: + :param db-base-os-path: The path in which the DB files are stored + :param ping-os-path: The path of a variable mimicing the RIAK ping functionality + + Interface methods + :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path + :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter + retrun the url-key + :meth:`_init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" boolean value + :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for + "ping" requests. Return "success" boolean value + :meth:`_ping` test the 'ping' status with the adapter. + Return a tuple: "success" boolean & "value" kept as ping variable + :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. + Return a tuple: "success" boolean & "value" kept in the parameter + :meth:`_read_parameters_by_storage_path` given a storage and and a key holding + filters on the key and values. + Return "success" boolean indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`_write_parameter_by_storage_path` given a storage path and a value string, + keep the parameter value. Return "success" boolean indicating a sucessful write + :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter + from the DB. Return "success" boolean indicating a sucessful deletion + """ + def __init__ (self, logger): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + """ + Base.__init__(self, logger) + + def _init_cfg(self, fullConfig): + """ + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: Boolean + """ + myCfgData = dict(fullConfig.items("fs-adapter")) if fullConfig.has_section("fs-adapter") else {} + self.basePath = myCfgData.get("db-base-os-path") + if self.basePath is None: + self.logger.error("Missing %s/%s configuration", "fs-adapter", "db-base-os-path") + return False + self.pingStoragePath = myCfgData.get("ping-os-path") + if self.pingStoragePath is None: + self.logger.error("Missing %s/%s configuration", "fs-adapter", "ping-os-path") + return False + return True + + def _get_parameter_storage_path(self, parameterUrlPath): + """ + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterUrlPath: the "url-path" like key of the variable + :type parameterUrlPath: str + :return: file path of where the value is be kept + :rtype: str + """ + return os.path.join(self.basePath, parameterUrlPath.lstrip("/").replace("/", os.path.sep)) + + def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): + """ + Conversion function - taking file path on the file system and translate to key's path + :param parameterStoragePath: the file name holding a value + :type parameterUrlPath: str + :return: the matching variable url-path like key + :rtype: str + """ + return "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") + + def _init_ping(self): + """ + Initialize the class ability to answer for ping. Part of Adapter required API. + :return: 'True' for successful initialization + :rtype: Boolean + """ + value = "OK" + success = self._write_parameter_by_storage_path(self.pingStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", self.pingStoragePath) + return False + return True + + def _ping(self): + """ + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[Boolean, str] + """ + success, value = self._read_parameter_by_storage_path(self.pingStoragePath) + if not success or value is None: + self.logger.error("no ping response") + return (False, "") + self.logger.debug("ping response: %s", value) + return (True, value) + + def _read_parameter_by_storage_path(self, parameterStoragePath): + """ + Reading the value from the provided file name. + :param parameterStoragePath: the file name + :type parameterStoragePath: str + :return: 'True' for successful retrivaland the retrieved value + :rtype: Tuple[boolean, str] + """ + self.logger.debug("Get parameter by os path: %s", parameterStoragePath) + try: + with open(parameterStoragePath) as fd: + value = fd.read() + except Exception as e: + self.logger.exception("%s parameter os path not found.", parameterStoragePath) + return False, None + + self.logger.debug("Get parameter by os path %s succeed", parameterStoragePath) + return True, value + + def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): + """ + Reading the values of the parameters the provided directory. + :param parameterStoragePathPrefix: the directory to look into + :type parameterStoragePathPrefix: str + :param keyFilters: filter-name/filter-func dict, holding functions that get a key as + input and retunn "true" if key should be included in the result + :type keyFilters: Dict[str,function[str]] + :return: 'True' for successful retrival and a dict for key-name/value + :rtype: Tuple[boolean, Dict[str, str]] + """ + self.logger.debug("Get parameters under os path %s", parameterStoragePathPrefix) + parameters = {} + + fileNames = [] + for (dirpath, _, filenames) in os.walk(parameterStoragePathPrefix): + fileNames += [os.path.join(dirpath, file) for file in filenames] + + for fileName in fileNames: + filteredOut = False + for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 + if not filterfunc(fileName): + self.logger.debug("Parameter os path %s dropped, not matching filter %s", self._get_parameter_storage_path_from_url_path(fileName), filterName) + filteredOut = True + break + if filteredOut: + continue + + parameterUrlPath = self._get_parameter_storage_path_from_url_path(fileName) + success, value = self._read_parameter_by_storage_path(fileName) + if not success: + self.logger.error("%s parameter os path not found.", fileName) + return False, None + + parameters[parameterUrlPath] = value + + return True, parameters + + + def _write_parameter_by_storage_path(self, parameterStoragePath, value): + """ + Writing the value to the provided file name. + :param parameterStoragePath: the file name + :type parameterStoragePath: str + :param value: value to be writen + :type value: str + :return: 'True' for successful writing + :rtype: Boolean + """ + self.logger.debug("Set parameter by os path %s", parameterStoragePath) + try: + dirname = os.path.dirname(parameterStoragePath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + with open(parameterStoragePath, "w") as fd: + fd.write(value) + except Exception as e: + self.logger.exception("could not post parameter os path %s", parameterStoragePath) + return False + self.logger.debug("Set parameter os path %s done", parameterStoragePath) + return True + + def _remove_parameter_by_storage_path(self, parameterStoragePath): + """ + Deleting the the provided file. + :param parameterStoragePath: the file name + :type parameterStoragePath: str + :return: 'True' for successful deletion + :rtype: Boolean + """ + self.logger.debug("Delete parameter os path %s", parameterStoragePath) + try: + os.remove(_parameterStoragePath) + except Exception as e: + self.logger.exception("could not delete parameter os path %s", parameterStoragePath) + return False + self.logger.debug("Delete parameter os path %s done", parameterStoragePath) + return True diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-debug b/experimental/emulated_vault/opt/emulated-vault/vault-debug old mode 100644 new mode 100755 index 9db797f3a1..691d34b2b3 --- a/experimental/emulated_vault/opt/emulated-vault/vault-debug +++ b/experimental/emulated_vault/opt/emulated-vault/vault-debug @@ -6,7 +6,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -29,158 +29,158 @@ import sys import adapter.fs if sys.version_info >= (3, 0): - #python 3 - from configparser import ConfigParser -else: - #python 2 - from ConfigParser import ConfigParser - + #python 3 + from configparser import ConfigParser +else: + #python 2 + from ConfigParser import ConfigParser + + +def main(): + progPath = inspect.stack()[-1][1] + progAbsPath = os.path.abspath( progPath ) + progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) + + confDir = os.path.join(progAbsPath, "conf") + confFile = os.path.join(confDir, "cfg.ini") + try: + config = ConfigParser() + config.read(confFile) + except IOError as e: + print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to read configuration: {0}".format(e), file=sys.stderr) + return 1 + + generalCfg = dict(config.items("general")) if config.has_section("general") else {} + logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) + debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") + mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") + try: + if not os.path.exists(logDir): + os.makedirs(logDir) + except IOError as e: + print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to create log dir: {0}".format(e), file=sys.stderr) + return 1 + + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + # create file handler which logs even debug messages + fhd = logging.FileHandler(debugLogFile) + fhd.setLevel(logging.DEBUG) + fhm = logging.FileHandler(mainLogFile) + fhm.setLevel(logging.INFO) + #TODO Set based on command line + verbose = logging.StreamHandler(sys.stdout) + verbose.setLevel(logging.INFO) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fhm.setFormatter(formatter) + fhd.setFormatter(formatter) + verbose.setFormatter(formatter) + # add the handlers to the logger + logger.addHandler(fhm) + logger.addHandler(fhd) + logger.addHandler(verbose) + + parser = optparse.OptionParser() + operationModeGroup = optparse.OptionGroup(parser, "Operation mode options", + "The vault script may work in one of the following modes.") + parser.add_option_group(operationModeGroup) + + operationModeGroup.add_option("--ping", + action="store_true", dest="ping", default=False, help="Ping") + + operationModeGroup.add_option("--get-parameter", + action="store_true", dest="getParameter", default=False, + help="Get parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--set-parameter", + action="store_true", dest="setParameter", default=False, + help="Set parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--delete-parameter", + action="store_true", dest="deleteParameter", default=False, + help="Delete parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--search-parameters", + action="store_true", dest="searchParameters", default=False, + help="Search parameter. Parameter relative path should be provided.") + + (options, args) = parser.parse_args() + + global adapter + adapterType = generalCfg.get("adapter-type") + if not adapterType: + logger.error("Missing adapter type cfg") + return 1 + elif adapterType == "fs": + adapter = adapter.fs.Fs(logger=logger) + else: + logger.error("Invalid adapter type '%s'", adapterType) + return 1 + + if not adapter.init_cfg(config): + logger.error("Failed adapter initialization") + return 1 + if not adapter.init_ping(): + logger.error("Failed adaper ping initialization") + return 1 + + if options.ping: + if len(args) != 0: + parser.error("Command should get no arguments.") + success, value = adapter.ping() + if not success: + return 1 + print(value) + return 0 + + if options.getParameter: + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success, value = adapter.getParameter(args[0]) + if not success: + return 1 + if value is None: + return 1 + print(value) + return 0 + + if options.searchParameters: + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success, value = adapter.searchParameters(args[0], keyFilters={}, filters={}) + if not success: + return 1 + print(json.dumps(value)) + return 0 + + if options.setParameter: + if len(args) != 2: + parser.error("Command should get 2 arguments - parameter key and value.") + success = adapter.setParameter(args[0], args[1]) + if not success: + return 1 + return 0 + + if options.deleteParameter: + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success = adapter.deleteParameter(args[0]) + if not success: + return 1 + return 0 + + logger.error("Operation is not set") + parser.print_help() + parser.error("No operation mode specified") + return 1 if __name__ == '__main__': - progPath = inspect.stack()[-1][1] - progAbsPath = os.path.abspath( progPath ) - progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) - - confDir = os.path.join(progAbsPath, "conf") - confFile = os.path.join(confDir, "cfg.ini") - try: - config = ConfigParser() - config.read(confFile) - except IOError as e: - print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) - sys.exit(1) - except Exception as e: - print("Failed to read configuration: {0}".format(e), file=sys.stderr) - sys.exit(1) - - generalCfg = dict(config.items("general")) if config.has_section("general") else {} - logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) - debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") - mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") - try: - if not os.path.exists(logDir): - os.makedirs(logDir) - except IOError as e: - print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) - sys.exit(1) - except Exception as e: - print("Failed to create log dir: {0}".format(e), file=sys.stderr) - sys.exit(1) - - global logger - logger = logging.getLogger(__name__) - logger.setLevel(logging.INFO) - # create file handler which logs even debug messages - fhd = logging.FileHandler(debugLogFile) - fhd.setLevel(logging.DEBUG) - fhm = logging.FileHandler(mainLogFile) - fhm.setLevel(logging.INFO) - #TODO Set based on command line - verbose = logging.StreamHandler(sys.stdout) - verbose.setLevel(logging.INFO) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - fhm.setFormatter(formatter) - fhd.setFormatter(formatter) - verbose.setFormatter(formatter) - # add the handlers to the logger - logger.addHandler(fhm) - logger.addHandler(fhd) - logger.addHandler(verbose) - - parser = optparse.OptionParser() - operationModeGroup = optparse.OptionGroup(parser, "Operation mode options", - "The vault script may work in one of the following modes.") - parser.add_option_group(operationModeGroup) - - operationModeGroup.add_option("--ping", - action="store_true", dest="ping", default=False, help="Ping") - - operationModeGroup.add_option("--get-parameter", - action="store_true", dest="getParameter", default=False, - help="Get parameter. Parameter relative path should be provided.") - - operationModeGroup.add_option("--set-parameter", - action="store_true", dest="setParameter", default=False, - help="Set parameter. Parameter relative path should be provided.") - - operationModeGroup.add_option("--delete-parameter", - action="store_true", dest="deleteParameter", default=False, - help="Delete parameter. Parameter relative path should be provided.") - - operationModeGroup.add_option("--search-parameters", - action="store_true", dest="searchParameters", default=False, - help="Search parameter. Parameter relative path should be provided.") - - (options, args) = parser.parse_args() - - global adapter - adapterType = generalCfg.get("adapter-type") - if not adapterType: - logger.error("Missing adapter type cfg") - sys.exit(1) - elif adapterType == "fs": - adapter = adapter.fs.Fs(logger=logger) - else: - logger.error("Invalid adapter type '%s'", adapterType) - sys.exit(1) - - if not adapter.init_cfg(config): - logger.error("Failed adapter initialization") - sys.exit(1) - if not adapter.init_ping(): - logger.error("Failed adaper ping initialization") - sys.exit(1) - - if options.ping: - if len(args) != 0: - parser.error("Command should get no arguments.") - success, value = adapter.ping() - if not success: - sys.exit(1) - print(value) - sys.exit(0) - - if options.getParameter: - if len(args) != 1: - parser.error("Command should get a single argument - parameter key.") - success, value = adapter.getParameter(args[0]) - if not success: - sys.exit(1) - if value is None: - sys.exit(1) - print(value) - sys.exit(0) - - if options.searchParameters: - if len(args) != 1: - parser.error("Command should get a single argument - parameter key.") - success, value = adapter.searchParameters(args[0], keyFilters={}, filters={}) - if not success: - sys.exit(1) - print(json.dumps(value)) - sys.exit(0) - - if options.setParameter: - if len(args) != 2: - parser.error("Command should get 2 arguments - parameter key and value.") - success = adapter.setParameter(args[0], args[1]) - if not success: - sys.exit(1) - sys.exit(0) - - if options.deleteParameter: - if len(args) != 1: - parser.error("Command should get a single argument - parameter key.") - success = adapter.deleteParameter(args[0]) - if not success: - sys.exit(1) - sys.exit(0) - - logger.error("Operation is not set") - parser.print_help() - parser.error("No operation mode specified") - sys.exit(1) - - - + sys.exit(main()) diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server old mode 100644 new mode 100755 index 0ecd9a1ade..bedc66acf6 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -6,7 +6,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -31,317 +31,321 @@ import sys import adapter.fs if sys.version_info >= (3, 0): - #python 3 - from http.server import HTTPServer - from http.server import BaseHTTPRequestHandler - from configparser import ConfigParser - import urllib.parse as urlparse -else: - #python 2 - from BaseHTTPServer import HTTPServer - from BaseHTTPServer import BaseHTTPRequestHandler - from ConfigParser import ConfigParser - import urlparse + #python 3 + from http.server import HTTPServer + from http.server import BaseHTTPRequestHandler + from configparser import ConfigParser + import urllib.parse as urlparse +else: + #python 2 + from BaseHTTPServer import HTTPServer + from BaseHTTPServer import BaseHTTPRequestHandler + from ConfigParser import ConfigParser + import urlparse class RequestHandler(BaseHTTPRequestHandler): - """ - An HTTP server, emulating RIAK behavior on the calls done by traffic-ops. - The class implements BaseHTTPRequestHandler functions - """ - - def do_GET(self): - """ - Base class function implementation - to be called upon HTTP GET/HEAD requests - """ - try: - self._do_GET() - except Exception as e: - logger.exception("do_GET exception") - - do_HEAD = do_GET - - def do_POST(self): - """ - Base class function implementation - to be called upon HTTP POST/PUT requests - """ - try: - self._do_POST() - except Exception as e: - logger.exception("do_POST exception") - - do_PUT = do_POST - - def do_DELETE(self): - """ - Base class function implementation - to be called upon HTTP DEL request - """ - try: - self._do_DELETE() - except Exception as e: - logger.exception("do_DELETE exception") - - def _do_GET(self): - """ - Actual GET logic - :raises: Exception - """ - logger.debug("GET %s", self.path) - parsed_path = urlparse.urlparse(self.path) - - if parsed_path.path == "/ping": - logger.info("Ping") - self._do_GET_ping() - elif parsed_path.path == "/search/query/sslkeys": - logger.info("Search SSL: path=%s query=%s", parsed_path.path, parsed_path.query) - self._do_GET_sslkeys(parsed_path) - else: - self._do_GET_general(parsed_path) - - def _do_GET_ping(self): - """ - Actual GET logic for ping request - :raises: Exception - """ - success, value = adapter.ping() - if not success: - self.send_response(503) - self.send_header('Content-type', 'text/html') - self.end_headers() - self.wfile.write("Failure".encode()) - return - - self.send_response(200) - self.send_header('Content-type', 'text/html') - self.end_headers() - self.wfile.write(value.encode()) - - def _do_GET_sslkeys(self, parsed_path): - """ - Actual GET logic for ssl keys request - :param parsed_path: key's url path - :type parsed_path: str - :raises: Exception - """ - filters = {} - keyFilters = {} - cdnFind = re.search(".*q=cdn:([^&]*).*", parsed_path.query) - if cdnFind: - def cdnFilter(key,val): - try: - data = json.loads(val) - return data['cdn']==cdnFind.group(1) - except Exception as e: - return False - filters['cdn'] = cdnFilter - - dsFind = re.search(".*q=deliveryservice:([^&]*).*", parsed_path.query) - if dsFind: - def dsFilter(key,val): - try: - data = json.loads(val) - return data['deliveryservice']==dsFind.group(1) - except Exception as e: - return False - filters['ds'] = dsFilter - - keyFind = re.search(".*q=_yz_rk:([^&]*).*", parsed_path.query) - if keyFind: - keyFilters['key-match'] = lambda key: fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) - - success, parameters = adapter.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) - if not success: - self.send_response(503) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Failure"}'.encode()) - return - - docs = [json.loads(val) for i in parameters.values()] - toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} - - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write(json.dumps(toReturn).encode()) - - def _do_GET_general(self, parsed_path): - """ - Actual GET logic for general variable request - :param parsed_path: key's url path - :type parsed_path: str - :raises: Exception - """ - success, value = adapter.getParameter(parsed_path.path) - if not success: - self.send_response(503) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Failure"}'.encode()) - return - - if value is None: - self.send_response(404) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Not found"}'.encode()) - return - - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write(value.encode()) - - def _do_POST(self): - """ - Actual POST request logic - """ - logger.info("POST %s", self.path) - parsed_path = urlparse.urlparse(self.path) - content_len = int(self.headers.getheader('content-length')) - post_body = self.rfile.read(content_len) - data = json.loads(post_body) - #mimic vault beahvior - if parsed_path.path.startswith('/riak/ssl/'): - certificate_data = data.get('certificate', {}) - data.update({"certificate.%s"%key: value for key, value in certificate_data.items()}) - success = adapter.setParameter(parsed_path.path, json.dumps(data)) - if not success: - self.send_response(503) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Failure"}'.encode()) - return - - self.send_response(204) - self.send_header('Content-Type', 'application/json') - self.end_headers() - return - - - def _do_DELETE(self): - """ - Actual DEL request logic - """ - logger.info("DELETE %s", self.path) - parsed_path = urlparse.urlparse(self.path) - - success = adapter.deleteParameter(parsed_path.path) - if not success: - self.send_response(503) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Failure"}'.encode()) - return - - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write('{"Deleted"}'.encode()) - return - - + """ + An HTTP server, emulating RIAK behavior on the calls done by traffic-ops. + The class implements BaseHTTPRequestHandler functions + """ + + def do_GET(self): + """ + Base class function implementation - to be called upon HTTP GET/HEAD requests + """ + try: + self._do_GET() + except Exception as e: + logger.exception("do_GET exception") + + do_HEAD = do_GET + + def do_POST(self): + """ + Base class function implementation - to be called upon HTTP POST/PUT requests + """ + try: + self._do_POST() + except Exception as e: + logger.exception("do_POST exception") + + do_PUT = do_POST + + def do_DELETE(self): + """ + Base class function implementation - to be called upon HTTP DEL request + """ + try: + self._do_DELETE() + except Exception as e: + logger.exception("do_DELETE exception") + + def _do_GET(self): + """ + Actual GET logic + :raises: Exception + """ + logger.debug("GET %s", self.path) + parsed_path = urlparse.urlparse(self.path) + + if parsed_path.path == "/ping": + logger.info("Ping") + self._do_GET_ping() + elif parsed_path.path == "/search/query/sslkeys": + logger.info("Search SSL: path=%s query=%s", parsed_path.path, parsed_path.query) + self._do_GET_sslkeys(parsed_path) + else: + self._do_GET_general(parsed_path) + + def _do_GET_ping(self): + """ + Actual GET logic for ping request + :raises: Exception + """ + success, value = adapter.ping() + if not success: + self.send_response(503) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write("Failure".encode()) + return + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(value.encode()) + + def _do_GET_sslkeys(self, parsed_path): + """ + Actual GET logic for ssl keys request + :param parsed_path: key's url path + :type parsed_path: str + :raises: Exception + """ + filters = {} + keyFilters = {} + cdnFind = re.search(".*q=cdn:([^&]*).*", parsed_path.query) + if cdnFind: + def cdnFilter(key,val): + try: + data = json.loads(val) + return data['cdn']==cdnFind.group(1) + except Exception as e: + return False + filters['cdn'] = cdnFilter + + dsFind = re.search(".*q=deliveryservice:([^&]*).*", parsed_path.query) + if dsFind: + def dsFilter(key,val): + try: + data = json.loads(val) + return data['deliveryservice']==dsFind.group(1) + except Exception as e: + return False + filters['ds'] = dsFilter + + keyFind = re.search(".*q=_yz_rk:([^&]*).*", parsed_path.query) + if keyFind: + keyFilters['key-match'] = lambda key: fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) + + success, parameters = adapter.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + docs = [json.loads(val) for i in parameters.values()] + toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(toReturn).encode()) + + def _do_GET_general(self, parsed_path): + """ + Actual GET logic for general variable request + :param parsed_path: key's url path + :type parsed_path: str + :raises: Exception + """ + success, value = adapter.getParameter(parsed_path.path) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + if value is None: + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Not found"}'.encode()) + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(value.encode()) + + def _do_POST(self): + """ + Actual POST request logic + """ + logger.info("POST %s", self.path) + parsed_path = urlparse.urlparse(self.path) + content_len = int(self.headers.getheader('content-length')) + post_body = self.rfile.read(content_len) + data = json.loads(post_body) + #mimic vault beahvior + if parsed_path.path.startswith('/riak/ssl/'): + certificate_data = data.get('certificate', {}) + data.update({"certificate.%s"%key: value for key, value in certificate_data.items()}) + success = adapter.setParameter(parsed_path.path, json.dumps(data)) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + self.send_response(204) + self.send_header('Content-Type', 'application/json') + self.end_headers() + return + + + def _do_DELETE(self): + """ + Actual DEL request logic + """ + logger.info("DELETE %s", self.path) + parsed_path = urlparse.urlparse(self.path) + + success = adapter.deleteParameter(parsed_path.path) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Deleted"}'.encode()) + return + + + +def main(): + progPath = inspect.stack()[-1][1] + progAbsPath = os.path.abspath( progPath ) + progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) + + confDir = os.path.join(progAbsPath, "conf") + confFile = os.path.join(confDir, "cfg.ini") + try: + config = ConfigParser() + config.read(confFile) + except IOError as e: + print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to read configuration: {0}".format(e), file=sys.stderr) + return 1 + + generalCfg = dict(config.items("general")) if config.has_section("general") else {} + logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) + debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") + mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") + try: + if not os.path.exists(logDir): + os.makedirs(logDir) + except IOError as e: + print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to create log dir: {0}".format(e), file=sys.stderr) + return 1 + + + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + # create file handler which logs even debug messages + fhd = logging.FileHandler(debugLogFile) + fhd.setLevel(logging.DEBUG) + fhm = logging.FileHandler(mainLogFile) + fhm.setLevel(logging.INFO) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fhm.setFormatter(formatter) + fhd.setFormatter(formatter) + # add the handlers to the logger + logger.addHandler(fhm) + logger.addHandler(fhd) + + httpServerCfg = dict(config.items("http-server")) if config.has_section("http-server") else {} + listenIP = httpServerCfg.get("listen-ip", "0.0.0.0") + try: + listenPort = int(httpServerCfg.get("listen-port", "8088")) + except Exception as e: + logger.exception("Failed %s integer conversion failed", "listen-port") + return 1 + use_ssl_val = httpServerCfg.get("use-ssl", "True") + if use_ssl_val in ["False", "false", "0"]: + use_ssl=False + elif use_ssl_val in ["True", "true", "1"]: + use_ssl=True + else: + logger.error("Invalid %s value", "use-ssl") + return 1 + if use_ssl: + sslKey = httpServerCfg.get("ssl-key-path") + if not sslKey: + print("Missing configuration: {0}/{1}".format("http-server", "ssl-key-path"), file=sys.stderr) + return 1 + sslCert = httpServerCfg.get("ssl-cert-path") + if not sslCert: + print("Missing configuration: {0}/{1}".format("http-server", "ssl-cert-path"), file=sys.stderr) + return 1 + + global adapter + adapterType = generalCfg.get("adapter-type") + if not adapterType: + logger.error("Missing adapter type cfg") + return 1 + elif adapterType == "fs": + adapter = adapter.fs.Fs(logger=logger) + else: + logger.error("Invalid adapter type '%s'", adapterType) + return 1 + + if not adapter.init_cfg(config): + logger.error("Failed adapter initialization") + return 1 + if not adapter.init_ping(): + logger.error("Failed adaper ping initialization") + return 1 + + server = HTTPServer((listenIP, listenPort), RequestHandler) + if use_ssl: + try: + server.socket = ssl.wrap_socket (server.socket, keyfile=sslKey, certfile=sslCert, server_side=True) + except Exception as e: + logger.exception("Failed on SSL init") + return 1 + + msg = 'Starting server at http%s://%s:%d'%("s" if use_ssl else "", listenIP, listenPort) + print(msg, file=sys.stderr) + logger.info(msg) + server.serve_forever() + return 0 if __name__ == '__main__': - progPath = inspect.stack()[-1][1] - progAbsPath = os.path.abspath( progPath ) - progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) - - confDir = os.path.join(progAbsPath, "conf") - confFile = os.path.join(confDir, "cfg.ini") - try: - config = ConfigParser() - config.read(confFile) - except IOError as e: - print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) - sys.exit(1) - except Exception as e: - print("Failed to read configuration: {0}".format(e), file=sys.stderr) - sys.exit(1) - - generalCfg = dict(config.items("general")) if config.has_section("general") else {} - logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) - debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") - mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") - try: - if not os.path.exists(logDir): - os.makedirs(logDir) - except IOError as e: - print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) - sys.exit(1) - except Exception as e: - print("Failed to create log dir: {0}".format(e), file=sys.stderr) - sys.exit(1) - - - global logger - logger = logging.getLogger(__name__) - logger.setLevel(logging.INFO) - # create file handler which logs even debug messages - fhd = logging.FileHandler(debugLogFile) - fhd.setLevel(logging.DEBUG) - fhm = logging.FileHandler(mainLogFile) - fhm.setLevel(logging.INFO) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - fhm.setFormatter(formatter) - fhd.setFormatter(formatter) - # add the handlers to the logger - logger.addHandler(fhm) - logger.addHandler(fhd) - - httpServerCfg = dict(config.items("http-server")) if config.has_section("http-server") else {} - listenIP = httpServerCfg.get("listen-ip", "0.0.0.0") - try: - listenPort = int(httpServerCfg.get("listen-port", "8088")) - except Exception as e: - logger.exception("Failed %s integer conversion failed", "listen-port") - sys.exit(1) - use_ssl_val = httpServerCfg.get("use-ssl", "True") - if use_ssl_val in ["False", "false", "0"]: - use_ssl=False - elif use_ssl_val in ["True", "true", "1"]: - use_ssl=True - else: - logger.error("Invalid %s value", "use-ssl") - sys.exit(1) - if use_ssl: - sslKey = httpServerCfg.get("ssl-key-path") - if not sslKey: - print("Missing configuration: {0}/{1}".format("http-server", "ssl-key-path"), file=sys.stderr) - sys.exit(1) - sslCert = httpServerCfg.get("ssl-cert-path") - if not sslCert: - print("Missing configuration: {0}/{1}".format("http-server", "ssl-cert-path"), file=sys.stderr) - sys.exit(1) - - global adapter - adapterType = generalCfg.get("adapter-type") - if not adapterType: - logger.error("Missing adapter type cfg") - sys.exit(1) - elif adapterType == "fs": - adapter = adapter.fs.Fs(logger=logger) - else: - logger.error("Invalid adapter type '%s'", adapterType) - sys.exit(1) - - if not adapter.init_cfg(config): - logger.error("Failed adapter initialization") - sys.exit(1) - if not adapter.init_ping(): - logger.error("Failed adaper ping initialization") - sys.exit(1) - - server = HTTPServer((listenIP, listenPort), RequestHandler) - if use_ssl: - try: - server.socket = ssl.wrap_socket (server.socket, keyfile=sslKey, certfile=sslCert, server_side=True) - except Exception as e: - logger.exception("Failed on SSL init") - sys.exit(1) - - msg = 'Starting server at http%s://%s:%d'%("s" if use_ssl else "", listenIP, listenPort) - print(msg, file=sys.stderr) - logger.info(msg) - server.serve_forever() + sys.exit(main()) From 434588b88ef9c5a4478313c4243c8d1b22c3dd67 Mon Sep 17 00:00:00 2001 From: nirs Date: Wed, 20 Nov 2019 23:23:01 +0200 Subject: [PATCH 7/9] storage adapter abstract class --- .../opt/emulated-vault/adapter/base.py | 292 ------------------ .../opt/emulated-vault/conf/cfg.ini | 2 +- .../{adapter => storage}/__init__.py | 0 .../emulated-vault/storage/adapter_base.py | 162 ++++++++++ .../{adapter/fs.py => storage/fs_adapter.py} | 118 ++----- .../opt/emulated-vault/vault-debug | 36 ++- .../opt/emulated-vault/vault-server | 38 ++- .../opt/emulated-vault/vault/__init__.py | 16 + .../opt/emulated-vault/vault/db.py | 152 +++++++++ 9 files changed, 403 insertions(+), 413 deletions(-) delete mode 100644 experimental/emulated_vault/opt/emulated-vault/adapter/base.py rename experimental/emulated_vault/opt/emulated-vault/{adapter => storage}/__init__.py (100%) create mode 100644 experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py rename experimental/emulated_vault/opt/emulated-vault/{adapter/fs.py => storage/fs_adapter.py} (53%) create mode 100644 experimental/emulated_vault/opt/emulated-vault/vault/__init__.py create mode 100644 experimental/emulated_vault/opt/emulated-vault/vault/db.py diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/base.py b/experimental/emulated_vault/opt/emulated-vault/adapter/base.py deleted file mode 100644 index af6cf80062..0000000000 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/base.py +++ /dev/null @@ -1,292 +0,0 @@ -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# - -import os - -class Base(object): - """ - Base adapter class. - This class implements the API required for storing and retriving the content kept in the vault. - - Implemented methods - :meth:`init_cfg` given a config-parser object, read the parameters required for - adapter's operation. Return "success" boolean value - :meth:`init_ping` prepare the adapter for beign able to provide info to reply for - "ping" requests. Return "success" boolean value - :meth:`ping` test the 'ping' status with the adapter. - Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. - Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding - variable key filters, and a key holding filters on the values as well. - Return "success" boolean indicating a sucessful write, and a key->value dictionary - for the relevant parameters - :meth:`setParameter` given a parameter key (in url-path format) and a value string, - keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter - from the DB. Return "success" boolean indicating a sucessful deletion - - Methods to be implemented at derived classes: - :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path - :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter - retrun the url-key - :meth:`_init_cfg` given a config-parser object, read the parameters required for - adapter's operation. Return "success" boolean value - :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for - "ping" requests. Return "success" boolean value - :meth:`_ping` test the 'ping' status with the adapter. - Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. - Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`_read_parameters_by_storage_path` given a storage and and a key holding - filters on the key and values. - Return "success" boolean indicating a sucessful write, and a key->value dictionary - for the relevant parameters - :meth:`_write_parameter_by_storage_path` given a storage path and a value string, - keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter - from the DB. Return "success" boolean indicating a sucessful deletion - """ - def __init__ (self, logger): - """ - The class constructor. - :param logger: logger to send log messages to - :type logger: a python logging logger class - """ - - self.logger = logger - - def init_cfg(self, fullConfig): - """ - Initialize the class basic parameters. Part of Adapter required API. - :param fullConfig: configuration to operate upon. - :type fullConfig: configparser.ConfigParser class - :return: 'True' for successful initialization - :rtype: Boolean - """ - return self._init_cfg(fullConfig) - - def init_ping(self): - """ - Initialize the class ability to answer for ping. Part of Adapter required API. - :return: 'True' for successful initialization - :rtype: Boolean - """ - return self._init_ping() - - def ping(self): - """ - get value for the ping request. Part of Adapter required API. - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - return self._ping() - - def getParameter(self, parameterUrlPath): - """ - Get value for the specified parameter. Part of Adapter required API. - :param parameterUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterUrlPath: str - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) - success, value = self._read_parameter_by_storage_path(parameterStoragePath) - if not success: - self.logger.error("Failed to bring parameter %s", parameterUrlPath) - return False, "" - if value is None: - self.logger.error("Could not find parameter %s", parameterUrlPath) - return True, None - self.logger.debug("Parameter get response for %s ready", parameterUrlPath) - return True, value - - def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): - """ - Get key/value dict of parameters by key pprefix - :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterKeyPrefixUrlPath: str - :param keyFilters: a dictionary of filter-name/filter-functions - - each function gets a single variable - a parameter key - - and return "True" if the variable should be included in the response. - :type keyFilters: Dict[str, function[str]] - :param filters: a dictionary of filter-name/filter-functions - - each function gets 2 variable - a parameter key and val - - and return "True" if the variable should be included in the response. - :type filters: Dict[str, function[str,str]] - :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) - :rtype: Tuple[boolean, Dict[str, str]] - """ - self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) - parameterStoragePathPrefix = self._get_parameter_storage_path(parameterKeyPrefixUrlPath) - success, items = self._read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) - if not success: - self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) - return False, "" - - filtered = {} - for key, val in items.items():#items() - supporting python 2&3 - skip = False - for filterName, filterfunc in filters.items():#items() - supporting python 2&3 - if not filterfunc(key, val): - self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) - skip = True - break - if skip: - continue - filtered[key] = val - - return True, filtered - - def setParameter(self, parameterUrlPath, value): - """ - Set value for the specified parameter. Part of Adapter required API. - :param parameterUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterUrlPath: str - :param value: the value to be kept - :type value: str - :return: 'True' for successful settings - :rtype: Boolean - """ - self.logger.debug("Set parameter %s", parameterUrlPath) - parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) - success = self._write_parameter_by_storage_path(parameterStoragePath, value) - if not success: - self.logger.error("Failed to set parameter %s", parameterUrlPath) - return False - return True - - def deleteParameter(self, parameterUrlPath): - """ - Delete the specified parameter. Part of Adapter required API. - :param parameterUrlPath: the key of the parameter as presented as url path - (tokens seperated by "/", with "/" as a prefix) - :type parameterUrlPath: str - :return: 'True' for successful deletion - :rtype: Boolean - """ - self.logger.debug("Delete parameter %s", parameterUrlPath) - parameterStoragePath = self._get_parameter_storage_path(parameterUrlPath) - success = self._remove_parameter_by_storage_path(parameterStoragePath) - if not success: - self.logger.error("Failed to delete parameter %s", parameterUrlPath) - return False - return True - - def _init_cfg(self, fullConfig): - """ - Method to be implemented at derived classes - Initialize the class basic parameters. Part of Adapter required API. - :param fullConfig: configuration to operate upon. - :type fullConfig: configparser.ConfigParser class - :return: 'True' for successful initialization - :rtype: Boolean - """ - raise NotImplementedError() - - def _get_parameter_storage_path(self, parameterUrlPath): - """ - Method to be implemented at derived classes - Conversion function - taking a key's path and translate to a file path on the file system - :param parameterUrlPath: the "url-path" like key of the variable - :type parameterUrlPath: str - :return: file path of where the value is be kept - :rtype: str - """ - raise NotImplementedError() - - def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): - """ - Method to be implemented at derived classes - Conversion function - taking file path on the file system and translate to key's path - :param parameterStoragePath: the file name holding a value - :type parameterUrlPath: str - :return: the matching variable url-path like key - :rtype: str - """ - raise NotImplementedError() - - def _init_ping(self): - """ - Method to be implemented at derived classes - Initialize the class ability to answer for ping. Part of Adapter required API. - :return: 'True' for successful initialization - :rtype: Boolean - """ - raise NotImplementedError() - - def _ping(self): - """ - Method to be implemented at derived classes - get value for the ping request. Part of Adapter required API. - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - raise NotImplementedError() - - def _read_parameter_by_storage_path(self, parameterStoragePath): - """ - Method to be implemented at derived classes - Reading the value from the provided file name. - :param parameterStoragePath: the file name - :type parameterUrlPath: str - :return: 'True' for successful retrivaland the retrieved value - :rtype: Tuple[boolean, str] - """ - raise NotImplementedError() - - def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): - """ - Method to be implemented at derived classes - Reading the values of the parameters the provided directory. - :param parameterStoragePathPrefix: the directory to look into - :type parameterStoragePathPrefix: str - :param keyFilters: filter-name/filter-func dict, holding functions that get a key as - input and retunn "true" if key should be included in the result - :type keyFilters: Dict[str,function[str]] - :return: 'True' for successful retrival and a dict for key-name/value - :rtype: Tuple[boolean, Dict[str, str]] - """ - raise NotImplementedError() - - - def _write_parameter_by_storage_path(self, parameterStoragePath, value): - """ - Method to be implemented at derived classes - Writing the value to the provided file name. - :param parameterStoragePath: the file name - :type parameterUrlPath: str - :param value: value to be writen - :type parameterUrlPath: str - :return: 'True' for successful writing - :rtype: Boolean - """ - raise NotImplementedError() - - def _remove_parameter_by_storage_path(self, parameterStoragePath): - """ - Method to be implemented at derived classes - Deleting the the provided file. - :param parameterStoragePath: the file name - :type parameterUrlPath: str - :return: 'True' for successful deletion - :rtype: Boolean - """ - raise NotImplementedError() \ No newline at end of file diff --git a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini index 2f79fe0fb0..49ad15ac8f 100644 --- a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini +++ b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini @@ -16,7 +16,7 @@ # [general] -adapter-type = fs +storage-adapter-type = fs # Optional: log-dir = /var/log/messages/emulated-vault [http-server] ssl-key-path = path/to/cert/key.pem diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py b/experimental/emulated_vault/opt/emulated-vault/storage/__init__.py similarity index 100% rename from experimental/emulated_vault/opt/emulated-vault/adapter/__init__.py rename to experimental/emulated_vault/opt/emulated-vault/storage/__init__.py diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py new file mode 100644 index 0000000000..f47102f1c7 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py @@ -0,0 +1,162 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +import sys +if sys.version_info >= (3, 0): + from abc import ABC, abstractmethod +else: + ABC = object + abstractmethod = lambda f: f + + + +class AdapterBase(ABC): + """ + Base adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + + Methods to be implemented at derived classes: + :meth:`get_parameter_storage_path` given a url-key of a parameter return the storage path + :meth:`get_parameter_storage_path_from_url_path` given a storage path of a parameter + retrun the url-key + :meth:`init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" bool value + :meth:`init` prepare the adapter - connecting to storgae / setting the conntent for + "ping" requests etc.. Return "success" bool value + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" bool & "value" kept as ping variable + :meth:`read_parameter_by_storage_path` given a storage path retrieve the parameter value. + Return a tuple: "success" bool & "value" kept in the parameter + :meth:`read_parameters_by_storage_path` given a storage and and a key holding + filters on the key and values. + Return "success" bool indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`write_parameter_by_storage_path` given a storage path and a value string, + keep the parameter value. Return "success" bool indicating a sucessful write + :meth:`remove_parameter_by_storage_path` given a storage path, delete the parameter + from the DB. Return "success" bool indicating a sucessful deletion + """ + + @abstractmethod + def init_cfg(self, fullConfig):# -> bool: + """ + Method to be implemented at derived classes + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: bool + """ + raise NotImplemented()#... + + @abstractmethod + def init(self):# -> bool: + """ + Method to be implemented at derived classes + Initialize the class - e.g. connection to storage & ability to answer for ping. + :return: 'True' for successful initialization + :rtype: bool + """ + raise NotImplemented()#... + + @abstractmethod + def get_parameter_storage_path(self, parameterUrlPath):# -> str: + """ + Method to be implemented at derived classes + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterUrlPath: the "url-path" like key of the variable + :type parameterUrlPath: str + :return: file path of where the value is be kept + :rtype: str + """ + raise NotImplemented()#... + + @abstractmethod + def get_parameter_storage_path_from_url_path(self, parameterStoragePath):# -> str: + """ + Method to be implemented at derived classes + Conversion function - taking file path on the file system and translate to key's path + :param parameterStoragePath: the file name holding a value + :type parameterUrlPath: str + :return: the matching variable url-path like key + :rtype: str + """ + raise NotImplemented()#... + + @abstractmethod + def ping(self):# -> bool: + """ + Method to be implemented at derived classes + Check ping connection + :return: 'True' for successful connection with the storage layer + :rtype: Tuple[bool, str] + """ + raise NotImplemented()#... + + @abstractmethod + def read_parameter_by_storage_path(self, parameterStoragePath):# -> (bool, str): + """ + Method to be implemented at derived classes + Reading the value from the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful retrivaland the retrieved value + :rtype: Tuple[bool, str] + """ + raise NotImplemented()#... + + @abstractmethod + def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters):# -> (bool, dict(str,str)): + """ + Method to be implemented at derived classes + Reading the values of the parameters the provided directory. + :param parameterStoragePathPrefix: the directory to look into + :type parameterStoragePathPrefix: str + :param keyFilters: filter-name/filter-func dict, holding functions that get a key as + input and retunn "true" if key should be included in the result + :type keyFilters: Dict[str,function[str]] + :return: 'True' for successful retrival and a dict for key-name/value + :rtype: Tuple[bool, Dict[str, str]] + """ + raise NotImplemented()#... + + + @abstractmethod + def write_parameter_by_storage_path(self, parameterStoragePath, value):# -> bool: + """ + Method to be implemented at derived classes + Writing the value to the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :param value: value to be writen + :type parameterUrlPath: str + :return: 'True' for successful writing + :rtype: bool + """ + raise NotImplemented()#... + + @abstractmethod + def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: + """ + Method to be implemented at derived classes + Deleting the the provided file. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: bool + """ + raise NotImplemented()#... \ No newline at end of file diff --git a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py b/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py similarity index 53% rename from experimental/emulated_vault/opt/emulated-vault/adapter/fs.py rename to experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py index 295ead4e29..e6b748bb65 100644 --- a/experimental/emulated_vault/opt/emulated-vault/adapter/fs.py +++ b/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py @@ -16,9 +16,9 @@ # import os -from . base import Base +from . adapter_base import AdapterBase -class Fs(Base): +class FsAdapter(AdapterBase): """ Fs (file system) adapter class. This class implements the API required for storing and retriving the content kept in the vault. @@ -26,43 +26,20 @@ class Fs(Base): Inputs configuration (held under "fs-adapter" section in the config file) include: :param db-base-os-path: The path in which the DB files are stored :param ping-os-path: The path of a variable mimicing the RIAK ping functionality - - Interface methods - :meth:`_get_parameter_storage_path` given a url-key of a parameter return the storage path - :meth:`_get_parameter_storage_path_from_url_path` given a storage path of a parameter - retrun the url-key - :meth:`_init_cfg` given a config-parser object, read the parameters required for - adapter's operation. Return "success" boolean value - :meth:`_init_ping` prepare the adapter for beign able to provide info to reply for - "ping" requests. Return "success" boolean value - :meth:`_ping` test the 'ping' status with the adapter. - Return a tuple: "success" boolean & "value" kept as ping variable - :meth:`_read_parameter_by_storage_path` given a storage path retrieve the parameter value. - Return a tuple: "success" boolean & "value" kept in the parameter - :meth:`_read_parameters_by_storage_path` given a storage and and a key holding - filters on the key and values. - Return "success" boolean indicating a sucessful write, and a key->value dictionary - for the relevant parameters - :meth:`_write_parameter_by_storage_path` given a storage path and a value string, - keep the parameter value. Return "success" boolean indicating a sucessful write - :meth:`_remove_parameter_by_storage_path` given a storage path, delete the parameter - from the DB. Return "success" boolean indicating a sucessful deletion """ + def __init__ (self, logger): """ The class constructor. :param logger: logger to send log messages to :type logger: a python logging logger class """ - Base.__init__(self, logger) + self.logger = logger - def _init_cfg(self, fullConfig): + def init_cfg(self, fullConfig):# -> bool: """ Initialize the class basic parameters. Part of Adapter required API. - :param fullConfig: configuration to operate upon. - :type fullConfig: configparser.ConfigParser class - :return: 'True' for successful initialization - :rtype: Boolean + Read the relevant section in the configuration """ myCfgData = dict(fullConfig.items("fs-adapter")) if fullConfig.has_section("fs-adapter") else {} self.basePath = myCfgData.get("db-base-os-path") @@ -75,59 +52,43 @@ def _init_cfg(self, fullConfig): return False return True - def _get_parameter_storage_path(self, parameterUrlPath): + def init(self):# -> bool: + """ + Initialize the class ability to answer for ping. Part of Adapter required API. + """ + value = ":)" + success = self.write_parameter_by_storage_path(self.pingStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", self.pingStoragePath) + return False + return True + + def get_parameter_storage_path(self, parameterUrlPath):# -> str: """ Conversion function - taking a key's path and translate to a file path on the file system - :param parameterUrlPath: the "url-path" like key of the variable - :type parameterUrlPath: str - :return: file path of where the value is be kept - :rtype: str """ return os.path.join(self.basePath, parameterUrlPath.lstrip("/").replace("/", os.path.sep)) - def _get_parameter_storage_path_from_url_path(self, parameterStoragePath): + def get_parameter_storage_path_from_url_path(self, parameterStoragePath):# -> str: """ Conversion function - taking file path on the file system and translate to key's path - :param parameterStoragePath: the file name holding a value - :type parameterUrlPath: str - :return: the matching variable url-path like key - :rtype: str """ return "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") - def _init_ping(self): + def ping(self):# -> bool: """ - Initialize the class ability to answer for ping. Part of Adapter required API. - :return: 'True' for successful initialization - :rtype: Boolean + Get value for the ping request by its path. Part of Adapter required API. """ - value = "OK" - success = self._write_parameter_by_storage_path(self.pingStoragePath, value) - if not success: - self.logger.error("Failed to set parameter %s", self.pingStoragePath) - return False - return True - - def _ping(self): - """ - get value for the ping request. Part of Adapter required API. - :return: A tuple - 'True' for successful retrival and the retrieved value - :rtype: Tuple[Boolean, str] - """ - success, value = self._read_parameter_by_storage_path(self.pingStoragePath) + success, value = self.read_parameter_by_storage_path(self.pingStoragePath) if not success or value is None: self.logger.error("no ping response") - return (False, "") + return False self.logger.debug("ping response: %s", value) - return (True, value) + return True - def _read_parameter_by_storage_path(self, parameterStoragePath): + def read_parameter_by_storage_path(self, parameterStoragePath):# -> (bool, str): """ Reading the value from the provided file name. - :param parameterStoragePath: the file name - :type parameterStoragePath: str - :return: 'True' for successful retrivaland the retrieved value - :rtype: Tuple[boolean, str] """ self.logger.debug("Get parameter by os path: %s", parameterStoragePath) try: @@ -140,16 +101,9 @@ def _read_parameter_by_storage_path(self, parameterStoragePath): self.logger.debug("Get parameter by os path %s succeed", parameterStoragePath) return True, value - def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters): + def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters):# -> (bool, dict(str, str)): """ Reading the values of the parameters the provided directory. - :param parameterStoragePathPrefix: the directory to look into - :type parameterStoragePathPrefix: str - :param keyFilters: filter-name/filter-func dict, holding functions that get a key as - input and retunn "true" if key should be included in the result - :type keyFilters: Dict[str,function[str]] - :return: 'True' for successful retrival and a dict for key-name/value - :rtype: Tuple[boolean, Dict[str, str]] """ self.logger.debug("Get parameters under os path %s", parameterStoragePathPrefix) parameters = {} @@ -162,14 +116,14 @@ def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilter filteredOut = False for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 if not filterfunc(fileName): - self.logger.debug("Parameter os path %s dropped, not matching filter %s", self._get_parameter_storage_path_from_url_path(fileName), filterName) + self.logger.debug("Parameter os path %s dropped, not matching filter %s", self.get_parameter_storage_path_from_url_path(fileName), filterName) filteredOut = True break if filteredOut: continue - parameterUrlPath = self._get_parameter_storage_path_from_url_path(fileName) - success, value = self._read_parameter_by_storage_path(fileName) + parameterUrlPath = self.get_parameter_storage_path_from_url_path(fileName) + success, value = self.read_parameter_by_storage_path(fileName) if not success: self.logger.error("%s parameter os path not found.", fileName) return False, None @@ -179,15 +133,9 @@ def _read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilter return True, parameters - def _write_parameter_by_storage_path(self, parameterStoragePath, value): + def write_parameter_by_storage_path(self, parameterStoragePath, value):# -> bool: """ Writing the value to the provided file name. - :param parameterStoragePath: the file name - :type parameterStoragePath: str - :param value: value to be writen - :type value: str - :return: 'True' for successful writing - :rtype: Boolean """ self.logger.debug("Set parameter by os path %s", parameterStoragePath) try: @@ -202,13 +150,9 @@ def _write_parameter_by_storage_path(self, parameterStoragePath, value): self.logger.debug("Set parameter os path %s done", parameterStoragePath) return True - def _remove_parameter_by_storage_path(self, parameterStoragePath): + def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: """ Deleting the the provided file. - :param parameterStoragePath: the file name - :type parameterStoragePath: str - :return: 'True' for successful deletion - :rtype: Boolean """ self.logger.debug("Delete parameter os path %s", parameterStoragePath) try: diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-debug b/experimental/emulated_vault/opt/emulated-vault/vault-debug index 691d34b2b3..c5be44ed99 100755 --- a/experimental/emulated_vault/opt/emulated-vault/vault-debug +++ b/experimental/emulated_vault/opt/emulated-vault/vault-debug @@ -26,7 +26,8 @@ import optparse import os import sys -import adapter.fs +import storage.fs_adapter +import vault.db if sys.version_info >= (3, 0): #python 3 @@ -114,24 +115,27 @@ def main(): (options, args) = parser.parse_args() - global adapter - adapterType = generalCfg.get("adapter-type") - if not adapterType: - logger.error("Missing adapter type cfg") + storageAdapterType = generalCfg.get("storage-adapter-type") + if not storageAdapterType: + logger.error("Missing storage adapter type cfg") return 1 - elif adapterType == "fs": - adapter = adapter.fs.Fs(logger=logger) + elif storageAdapterType == "fs": + storageAdapter = storage.fs_adapter.FsAdapter(logger=logger) else: - logger.error("Invalid adapter type '%s'", adapterType) + logger.error("Invalid storage adapter type '%s'", storageAdapterType) return 1 - if not adapter.init_cfg(config): - logger.error("Failed adapter initialization") + + if not storageAdapter.init_cfg(config): + logger.error("Failed storage adapter initialization") return 1 - if not adapter.init_ping(): - logger.error("Failed adaper ping initialization") + if not storageAdapter.init(): + logger.error("Failed storage adaper initialization") return 1 + global db + db = vault.db.Db(logger, storageAdapter) + if options.ping: if len(args) != 0: parser.error("Command should get no arguments.") @@ -144,7 +148,7 @@ def main(): if options.getParameter: if len(args) != 1: parser.error("Command should get a single argument - parameter key.") - success, value = adapter.getParameter(args[0]) + success, value = db.getParameter(args[0]) if not success: return 1 if value is None: @@ -155,7 +159,7 @@ def main(): if options.searchParameters: if len(args) != 1: parser.error("Command should get a single argument - parameter key.") - success, value = adapter.searchParameters(args[0], keyFilters={}, filters={}) + success, value = db.searchParameters(args[0], keyFilters={}, filters={}) if not success: return 1 print(json.dumps(value)) @@ -164,7 +168,7 @@ def main(): if options.setParameter: if len(args) != 2: parser.error("Command should get 2 arguments - parameter key and value.") - success = adapter.setParameter(args[0], args[1]) + success = db.setParameter(args[0], args[1]) if not success: return 1 return 0 @@ -172,7 +176,7 @@ def main(): if options.deleteParameter: if len(args) != 1: parser.error("Command should get a single argument - parameter key.") - success = adapter.deleteParameter(args[0]) + success = db.deleteParameter(args[0]) if not success: return 1 return 0 diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server index bedc66acf6..22cbb31bb6 100755 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -28,7 +28,8 @@ import re import ssl import sys -import adapter.fs +import storage.fs_adapter +import vault.db if sys.version_info >= (3, 0): #python 3 @@ -103,7 +104,7 @@ class RequestHandler(BaseHTTPRequestHandler): Actual GET logic for ping request :raises: Exception """ - success, value = adapter.ping() + success, value = db.ping() if not success: self.send_response(503) self.send_header('Content-type', 'text/html') @@ -149,7 +150,7 @@ class RequestHandler(BaseHTTPRequestHandler): if keyFind: keyFilters['key-match'] = lambda key: fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) - success, parameters = adapter.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) + success, parameters = db.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -172,7 +173,7 @@ class RequestHandler(BaseHTTPRequestHandler): :type parsed_path: str :raises: Exception """ - success, value = adapter.getParameter(parsed_path.path) + success, value = db.getParameter(parsed_path.path) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -205,7 +206,7 @@ class RequestHandler(BaseHTTPRequestHandler): if parsed_path.path.startswith('/riak/ssl/'): certificate_data = data.get('certificate', {}) data.update({"certificate.%s"%key: value for key, value in certificate_data.items()}) - success = adapter.setParameter(parsed_path.path, json.dumps(data)) + success = db.setParameter(parsed_path.path, json.dumps(data)) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -226,7 +227,7 @@ class RequestHandler(BaseHTTPRequestHandler): logger.info("DELETE %s", self.path) parsed_path = urlparse.urlparse(self.path) - success = adapter.deleteParameter(parsed_path.path) + success = db.deleteParameter(parsed_path.path) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') @@ -315,24 +316,27 @@ def main(): print("Missing configuration: {0}/{1}".format("http-server", "ssl-cert-path"), file=sys.stderr) return 1 - global adapter - adapterType = generalCfg.get("adapter-type") - if not adapterType: - logger.error("Missing adapter type cfg") + storageAdapterType = generalCfg.get("storage-adapter-type") + if not storageAdapterType: + logger.error("Missing storage adapter type cfg") return 1 - elif adapterType == "fs": - adapter = adapter.fs.Fs(logger=logger) + elif storageAdapterType == "fs": + storageAdapter = storage.fs_adapter.FsAdapter(logger=logger) else: - logger.error("Invalid adapter type '%s'", adapterType) + logger.error("Invalid storage adapter type '%s'", storageAdapterType) return 1 - if not adapter.init_cfg(config): - logger.error("Failed adapter initialization") + + if not storageAdapter.init_cfg(config): + logger.error("Failed storage adapter initialization") return 1 - if not adapter.init_ping(): - logger.error("Failed adaper ping initialization") + if not storageAdapter.init(): + logger.error("Failed storage adaper initialization") return 1 + global db + db = vault.db.Db(logger, storageAdapter) + server = HTTPServer((listenIP, listenPort), RequestHandler) if use_ssl: try: diff --git a/experimental/emulated_vault/opt/emulated-vault/vault/__init__.py b/experimental/emulated_vault/opt/emulated-vault/vault/__init__.py new file mode 100644 index 0000000000..dc9911a6d6 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault/__init__.py @@ -0,0 +1,16 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# diff --git a/experimental/emulated_vault/opt/emulated-vault/vault/db.py b/experimental/emulated_vault/opt/emulated-vault/vault/db.py new file mode 100644 index 0000000000..03aa8444fb --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault/db.py @@ -0,0 +1,152 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +class Db(object): + """ + DB class representing the DB layer. + + Implemented methods + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" bool & "value" kept as ping variable + :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. + Return a tuple: "success" bool & "value" kept in the parameter + :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding + variable key filters, and a key holding filters on the values as well. + Return "success" bool indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`setParameter` given a parameter key (in url-path format) and a value string, + keep the parameter value. Return "success" bool indicating a sucessful write + :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter + from the DB. Return "success" bool indicating a sucessful deletion + """ + + def __init__ (self, logger, storage_adaper): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + :param storage_adaper: an initalized storage adapter + :type storage_adaper: a storage.adapter_base.AdapterBase class + """ + + self.logger = logger + self.storage_adaper = storage_adaper + + def ping(self): + """ + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[bool, str] + """ + if self.storage_adaper.ping(): + return (True, "OK") + return (False, "") + + def getParameter(self, parameterUrlPath): + """ + Get value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[bool, str] + """ + parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + success, value = self.storage_adaper.read_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to bring parameter %s", parameterUrlPath) + return False, "" + if value is None: + self.logger.error("Could not find parameter %s", parameterUrlPath) + return True, None + self.logger.debug("Parameter get response for %s ready", parameterUrlPath) + return True, value + + def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): + """ + Get key/value dict of parameters by key pprefix + :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyPrefixUrlPath: str + :param keyFilters: a dictionary of filter-name/filter-functions - + each function gets a single variable - a parameter key - + and return "True" if the variable should be included in the response. + :type keyFilters: Dict[str, function[str]] + :param filters: a dictionary of filter-name/filter-functions - + each function gets 2 variable - a parameter key and val - + and return "True" if the variable should be included in the response. + :type filters: Dict[str, function[str,str]] + :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) + :rtype: Tuple[bool, Dict[str, str]] + """ + self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) + parameterStoragePathPrefix = self.storage_adaper.get_parameter_storage_path(parameterKeyPrefixUrlPath) + success, items = self.storage_adaper.read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) + if not success: + self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) + return False, "" + + filtered = {} + for key, val in items.items():#items() - supporting python 2&3 + skip = False + for filterName, filterfunc in filters.items():#items() - supporting python 2&3 + if not filterfunc(key, val): + self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) + skip = True + break + if skip: + continue + filtered[key] = val + + return True, filtered + + def setParameter(self, parameterUrlPath, value): + """ + Set value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :param value: the value to be kept + :type value: str + :return: 'True' for successful settings + :rtype: bool + """ + self.logger.debug("Set parameter %s", parameterUrlPath) + parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + success = self.storage_adaper.write_parameter_by_storage_path(parameterStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", parameterUrlPath) + return False + return True + + def deleteParameter(self, parameterUrlPath): + """ + Delete the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: bool + """ + self.logger.debug("Delete parameter %s", parameterUrlPath) + parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + success = self.storage_adaper.remove_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to delete parameter %s", parameterUrlPath) + return False + return True + From 57b144ac373c2b9f9cca1057de4454d42f8b975b Mon Sep 17 00:00:00 2001 From: nirs Date: Thu, 21 Nov 2019 00:18:22 +0200 Subject: [PATCH 8/9] emulated vault - further refinment of the APIs --- .../emulated-vault/storage/adapter_base.py | 17 ++++++------ .../opt/emulated-vault/storage/fs_adapter.py | 21 ++++++++++----- .../opt/emulated-vault/vault-server | 2 +- .../opt/emulated-vault/vault/db.py | 26 ++++++++++++++----- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py index f47102f1c7..65809c5921 100644 --- a/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py +++ b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py @@ -74,26 +74,26 @@ def init(self):# -> bool: raise NotImplemented()#... @abstractmethod - def get_parameter_storage_path(self, parameterUrlPath):# -> str: + def get_parameter_storage_path(self, parameterUrlPath):# -> (bool, str): """ Method to be implemented at derived classes Conversion function - taking a key's path and translate to a file path on the file system :param parameterUrlPath: the "url-path" like key of the variable :type parameterUrlPath: str - :return: file path of where the value is be kept - :rtype: str + :return: "success" bool and a file path of where the value is be kept + :rtype: Tuple[bool, str] """ raise NotImplemented()#... @abstractmethod - def get_parameter_storage_path_from_url_path(self, parameterStoragePath):# -> str: + def get_parameter_url_path_from_storage_path(self, parameterStoragePath):# -> (bool, str): """ Method to be implemented at derived classes Conversion function - taking file path on the file system and translate to key's path :param parameterStoragePath: the file name holding a value :type parameterUrlPath: str - :return: the matching variable url-path like key - :rtype: str + :return: "success" bool and the matching variable url-path like key + :rtype: Tuple[bool, str] """ raise NotImplemented()#... @@ -103,7 +103,7 @@ def ping(self):# -> bool: Method to be implemented at derived classes Check ping connection :return: 'True' for successful connection with the storage layer - :rtype: Tuple[bool, str] + :rtype: bool """ raise NotImplemented()#... @@ -159,4 +159,5 @@ def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: :return: 'True' for successful deletion :rtype: bool """ - raise NotImplemented()#... \ No newline at end of file + raise NotImplemented()#... + \ No newline at end of file diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py b/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py index e6b748bb65..95dbbd0c3a 100644 --- a/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py +++ b/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py @@ -63,17 +63,23 @@ def init(self):# -> bool: return False return True - def get_parameter_storage_path(self, parameterUrlPath):# -> str: + def get_parameter_storage_path(self, parameterUrlPath):# -> bool, str: """ Conversion function - taking a key's path and translate to a file path on the file system """ - return os.path.join(self.basePath, parameterUrlPath.lstrip("/").replace("/", os.path.sep)) + #verifying the existance of "/" at the beging + if not parameterUrlPath.startswith("/"): + self.logger.error("Failed to translate url path '%s': no leading '/'", parameterUrlPath) + return False, "" + # avoiding the use of os.path.join in purpuse. + # we do not want the "/" at the begining push us to the root path + return True, self.basePath + parameterUrlPath.replace("/", os.path.sep) - def get_parameter_storage_path_from_url_path(self, parameterStoragePath):# -> str: + def get_parameter_url_path_from_storage_path(self, parameterStoragePath):# -> str: """ Conversion function - taking file path on the file system and translate to key's path """ - return "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") + return True, "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") def ping(self):# -> bool: """ @@ -116,13 +122,16 @@ def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters filteredOut = False for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 if not filterfunc(fileName): - self.logger.debug("Parameter os path %s dropped, not matching filter %s", self.get_parameter_storage_path_from_url_path(fileName), filterName) + self.logger.debug("Parameter os path %s dropped, not matching filter '%s'", self.get_parameter_url_path_from_storage_path(fileName)[1], filterName) filteredOut = True break if filteredOut: continue - parameterUrlPath = self.get_parameter_storage_path_from_url_path(fileName) + success_path, parameterUrlPath = self.get_parameter_url_path_from_storage_path(fileName) + if not success_path: + self.logger.error("%s parameter os path is invalid.", fileName) + return False, None success, value = self.read_parameter_by_storage_path(fileName) if not success: self.logger.error("%s parameter os path not found.", fileName) diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server index 22cbb31bb6..a95b4a90f3 100755 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -150,7 +150,7 @@ class RequestHandler(BaseHTTPRequestHandler): if keyFind: keyFilters['key-match'] = lambda key: fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) - success, parameters = db.searchParameters("riak/ssl/", keyFilters=keyFilters, filters=filters) + success, parameters = db.searchParameters("/riak/ssl/", keyFilters=keyFilters, filters=filters) if not success: self.send_response(503) self.send_header('Content-Type', 'application/json') diff --git a/experimental/emulated_vault/opt/emulated-vault/vault/db.py b/experimental/emulated_vault/opt/emulated-vault/vault/db.py index 03aa8444fb..b39a0739b1 100644 --- a/experimental/emulated_vault/opt/emulated-vault/vault/db.py +++ b/experimental/emulated_vault/opt/emulated-vault/vault/db.py @@ -54,7 +54,7 @@ def ping(self): """ if self.storage_adaper.ping(): return (True, "OK") - return (False, "") + return False, None def getParameter(self, parameterUrlPath): """ @@ -65,11 +65,14 @@ def getParameter(self, parameterUrlPath): :return: A tuple - 'True' for successful retrival and the retrieved value :rtype: Tuple[bool, str] """ - parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + success_path, parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path: %s", parameterUrlPath) + return False, None success, value = self.storage_adaper.read_parameter_by_storage_path(parameterStoragePath) if not success: self.logger.error("Failed to bring parameter %s", parameterUrlPath) - return False, "" + return False, None if value is None: self.logger.error("Could not find parameter %s", parameterUrlPath) return True, None @@ -94,11 +97,14 @@ def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): :rtype: Tuple[bool, Dict[str, str]] """ self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) - parameterStoragePathPrefix = self.storage_adaper.get_parameter_storage_path(parameterKeyPrefixUrlPath) + success_path, parameterStoragePathPrefix = self.storage_adaper.get_parameter_storage_path(parameterKeyPrefixUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path prefix: %s", parameterKeyPrefixUrlPath) + return False, None success, items = self.storage_adaper.read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) if not success: self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) - return False, "" + return False, None filtered = {} for key, val in items.items():#items() - supporting python 2&3 @@ -126,7 +132,10 @@ def setParameter(self, parameterUrlPath, value): :rtype: bool """ self.logger.debug("Set parameter %s", parameterUrlPath) - parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + success_path, parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path: %s", parameterUrlPath) + return False, None success = self.storage_adaper.write_parameter_by_storage_path(parameterStoragePath, value) if not success: self.logger.error("Failed to set parameter %s", parameterUrlPath) @@ -143,7 +152,10 @@ def deleteParameter(self, parameterUrlPath): :rtype: bool """ self.logger.debug("Delete parameter %s", parameterUrlPath) - parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + success_path, parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path: %s", parameterUrlPath) + return False, None success = self.storage_adaper.remove_parameter_by_storage_path(parameterStoragePath) if not success: self.logger.error("Failed to delete parameter %s", parameterUrlPath) From f653e6a6eb8cb22265d0e8aa8c3f34d3ff535118 Mon Sep 17 00:00:00 2001 From: nirs Date: Mon, 25 Nov 2019 22:40:02 +0200 Subject: [PATCH 9/9] Emulated vault - code review fixes --- .../emulated-vault/storage/adapter_base.py | 20 +++++++++---------- .../opt/emulated-vault/vault-server | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py index 65809c5921..db0354322d 100644 --- a/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py +++ b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py @@ -61,7 +61,7 @@ def init_cfg(self, fullConfig):# -> bool: :return: 'True' for successful initialization :rtype: bool """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def init(self):# -> bool: @@ -71,7 +71,7 @@ def init(self):# -> bool: :return: 'True' for successful initialization :rtype: bool """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def get_parameter_storage_path(self, parameterUrlPath):# -> (bool, str): @@ -83,7 +83,7 @@ def get_parameter_storage_path(self, parameterUrlPath):# -> (bool, str): :return: "success" bool and a file path of where the value is be kept :rtype: Tuple[bool, str] """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def get_parameter_url_path_from_storage_path(self, parameterStoragePath):# -> (bool, str): @@ -95,7 +95,7 @@ def get_parameter_url_path_from_storage_path(self, parameterStoragePath):# -> (b :return: "success" bool and the matching variable url-path like key :rtype: Tuple[bool, str] """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def ping(self):# -> bool: @@ -105,7 +105,7 @@ def ping(self):# -> bool: :return: 'True' for successful connection with the storage layer :rtype: bool """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def read_parameter_by_storage_path(self, parameterStoragePath):# -> (bool, str): @@ -117,7 +117,7 @@ def read_parameter_by_storage_path(self, parameterStoragePath):# -> (bool, str): :return: 'True' for successful retrivaland the retrieved value :rtype: Tuple[bool, str] """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters):# -> (bool, dict(str,str)): @@ -132,7 +132,7 @@ def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters :return: 'True' for successful retrival and a dict for key-name/value :rtype: Tuple[bool, Dict[str, str]] """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod @@ -147,7 +147,7 @@ def write_parameter_by_storage_path(self, parameterStoragePath, value):# -> bool :return: 'True' for successful writing :rtype: bool """ - raise NotImplemented()#... + raise NotImplementedError()#... @abstractmethod def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: @@ -159,5 +159,5 @@ def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: :return: 'True' for successful deletion :rtype: bool """ - raise NotImplemented()#... - \ No newline at end of file + raise NotImplementedError()#... + diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server index a95b4a90f3..f09a3ec023 100755 --- a/experimental/emulated_vault/opt/emulated-vault/vault-server +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -158,7 +158,7 @@ class RequestHandler(BaseHTTPRequestHandler): self.wfile.write('{"Failure"}'.encode()) return - docs = [json.loads(val) for i in parameters.values()] + docs = [json.loads(val) for val in parameters.values()] toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} self.send_response(200)