diff --git a/CHANGELOG.md b/CHANGELOG.md index 27268c718d..eca735e288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added validation to prevent assigning servers to delivery services without required capabilities. - Added deep coverage zone routing percentage to the Traffic Portal dashboard. - Added a `traffic_ops/app/bin/osversions-convert.pl` script to convert the `osversions.cfg` file from Perl to JSON as part of the `/osversions` endpoint rewrite. +- Added [Experimental] - Emulated Vault suppling a HTTP server mimicking RIAK behavior for usage as traffic-control vault. ### Changed - Traffic Router: TR will now allow steering DSs and steering target DSs to have RGB enabled. (fixes #3910) diff --git a/experimental/emulated_vault/README.md b/experimental/emulated_vault/README.md new file mode 100644 index 0000000000..c25ff5ba62 --- /dev/null +++ b/experimental/emulated_vault/README.md @@ -0,0 +1,51 @@ + + +# Emulated Vault - Background + +The emulated_vault module supplies a HTTP server mimicking RIAK behavior for usage as traffic-control vault. +It may be used in order to replace RIAK traffic_vault, as it is much more simple to install. +The server may use different type of persistent storage (e.g. file-system), using the proper adapter. +The resiliency of the stored keys is derived from the resiliency of the underlying storage. + +# Installation + +Basic requirements: Centos ver >= 7; Python >= 2.7 + +In order to install the module on a server please: +1. Copy the module files to the server's root +2. Add the certificate and key to your favorite path +3. Adjust /opt/emulated_vault/conf/cfg.json - pointing at your certificate and key +4. "systemctl enable" the service + +Logs may be found under /opt/emulated_vault/var/log + +# Developer's Notes + +If you just want to play around with the module, you may of course run the server script on its own. +Before doing that, you would probably need to adjust the opt/emulated_vault/conf/cfg.json: +1. Changing the db-path to one you have access to +2. Disable ssl (just to make it easier) + +Additionally, the vault-debug script is also available to work against the DB with command line. +It is mostly useful when developing a new adapter. + +# Contact + +For additional information, questions or assistance, please approach [Nir B. Sopher](mailto:nir@apache.org) diff --git a/experimental/emulated_vault/etc/systemd/system/emulated-vault.service b/experimental/emulated_vault/etc/systemd/system/emulated-vault.service new file mode 100644 index 0000000000..38b9991e00 --- /dev/null +++ b/experimental/emulated_vault/etc/systemd/system/emulated-vault.service @@ -0,0 +1,29 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# +[Unit] +Description=Emulated Vault Service +After=network.target + +[Service] +Type=simple +User=root +ExecStart=/opt/emulated-vault/vault-server +Restart=on-failure + + +[Install] +WantedBy=multi-user.target diff --git a/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini new file mode 100644 index 0000000000..49ad15ac8f --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/conf/cfg.ini @@ -0,0 +1,30 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +[general] +storage-adapter-type = fs +# Optional: log-dir = /var/log/messages/emulated-vault +[http-server] +ssl-key-path = path/to/cert/key.pem +ssl-cert-path = path/to/cert/cert.pem +# Optional: use-ssl = False +# Optional: listen-ip = 1.2.3.4 +# Optional: listen-port = 12345 + +[fs-adapter] +db-base-os-path = /opt/emulated-vault/db +ping-os-path = /opt/emulated-vault/ping diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/__init__.py b/experimental/emulated_vault/opt/emulated-vault/storage/__init__.py new file mode 100644 index 0000000000..dc9911a6d6 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/storage/__init__.py @@ -0,0 +1,16 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py new file mode 100644 index 0000000000..db0354322d --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/storage/adapter_base.py @@ -0,0 +1,163 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +import sys +if sys.version_info >= (3, 0): + from abc import ABC, abstractmethod +else: + ABC = object + abstractmethod = lambda f: f + + + +class AdapterBase(ABC): + """ + Base adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + + Methods to be implemented at derived classes: + :meth:`get_parameter_storage_path` given a url-key of a parameter return the storage path + :meth:`get_parameter_storage_path_from_url_path` given a storage path of a parameter + retrun the url-key + :meth:`init_cfg` given a config-parser object, read the parameters required for + adapter's operation. Return "success" bool value + :meth:`init` prepare the adapter - connecting to storgae / setting the conntent for + "ping" requests etc.. Return "success" bool value + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" bool & "value" kept as ping variable + :meth:`read_parameter_by_storage_path` given a storage path retrieve the parameter value. + Return a tuple: "success" bool & "value" kept in the parameter + :meth:`read_parameters_by_storage_path` given a storage and and a key holding + filters on the key and values. + Return "success" bool indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`write_parameter_by_storage_path` given a storage path and a value string, + keep the parameter value. Return "success" bool indicating a sucessful write + :meth:`remove_parameter_by_storage_path` given a storage path, delete the parameter + from the DB. Return "success" bool indicating a sucessful deletion + """ + + @abstractmethod + def init_cfg(self, fullConfig):# -> bool: + """ + Method to be implemented at derived classes + Initialize the class basic parameters. Part of Adapter required API. + :param fullConfig: configuration to operate upon. + :type fullConfig: configparser.ConfigParser class + :return: 'True' for successful initialization + :rtype: bool + """ + raise NotImplementedError()#... + + @abstractmethod + def init(self):# -> bool: + """ + Method to be implemented at derived classes + Initialize the class - e.g. connection to storage & ability to answer for ping. + :return: 'True' for successful initialization + :rtype: bool + """ + raise NotImplementedError()#... + + @abstractmethod + def get_parameter_storage_path(self, parameterUrlPath):# -> (bool, str): + """ + Method to be implemented at derived classes + Conversion function - taking a key's path and translate to a file path on the file system + :param parameterUrlPath: the "url-path" like key of the variable + :type parameterUrlPath: str + :return: "success" bool and a file path of where the value is be kept + :rtype: Tuple[bool, str] + """ + raise NotImplementedError()#... + + @abstractmethod + def get_parameter_url_path_from_storage_path(self, parameterStoragePath):# -> (bool, str): + """ + Method to be implemented at derived classes + Conversion function - taking file path on the file system and translate to key's path + :param parameterStoragePath: the file name holding a value + :type parameterUrlPath: str + :return: "success" bool and the matching variable url-path like key + :rtype: Tuple[bool, str] + """ + raise NotImplementedError()#... + + @abstractmethod + def ping(self):# -> bool: + """ + Method to be implemented at derived classes + Check ping connection + :return: 'True' for successful connection with the storage layer + :rtype: bool + """ + raise NotImplementedError()#... + + @abstractmethod + def read_parameter_by_storage_path(self, parameterStoragePath):# -> (bool, str): + """ + Method to be implemented at derived classes + Reading the value from the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful retrivaland the retrieved value + :rtype: Tuple[bool, str] + """ + raise NotImplementedError()#... + + @abstractmethod + def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters):# -> (bool, dict(str,str)): + """ + Method to be implemented at derived classes + Reading the values of the parameters the provided directory. + :param parameterStoragePathPrefix: the directory to look into + :type parameterStoragePathPrefix: str + :param keyFilters: filter-name/filter-func dict, holding functions that get a key as + input and retunn "true" if key should be included in the result + :type keyFilters: Dict[str,function[str]] + :return: 'True' for successful retrival and a dict for key-name/value + :rtype: Tuple[bool, Dict[str, str]] + """ + raise NotImplementedError()#... + + + @abstractmethod + def write_parameter_by_storage_path(self, parameterStoragePath, value):# -> bool: + """ + Method to be implemented at derived classes + Writing the value to the provided file name. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :param value: value to be writen + :type parameterUrlPath: str + :return: 'True' for successful writing + :rtype: bool + """ + raise NotImplementedError()#... + + @abstractmethod + def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: + """ + Method to be implemented at derived classes + Deleting the the provided file. + :param parameterStoragePath: the file name + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: bool + """ + raise NotImplementedError()#... + diff --git a/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py b/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py new file mode 100644 index 0000000000..95dbbd0c3a --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/storage/fs_adapter.py @@ -0,0 +1,174 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +import os +from . adapter_base import AdapterBase + +class FsAdapter(AdapterBase): + """ + Fs (file system) adapter class. + This class implements the API required for storing and retriving the content kept in the vault. + This specific Adapter works using files upon a file-system. + Inputs configuration (held under "fs-adapter" section in the config file) include: + :param db-base-os-path: The path in which the DB files are stored + :param ping-os-path: The path of a variable mimicing the RIAK ping functionality + """ + + def __init__ (self, logger): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + """ + self.logger = logger + + def init_cfg(self, fullConfig):# -> bool: + """ + Initialize the class basic parameters. Part of Adapter required API. + Read the relevant section in the configuration + """ + myCfgData = dict(fullConfig.items("fs-adapter")) if fullConfig.has_section("fs-adapter") else {} + self.basePath = myCfgData.get("db-base-os-path") + if self.basePath is None: + self.logger.error("Missing %s/%s configuration", "fs-adapter", "db-base-os-path") + return False + self.pingStoragePath = myCfgData.get("ping-os-path") + if self.pingStoragePath is None: + self.logger.error("Missing %s/%s configuration", "fs-adapter", "ping-os-path") + return False + return True + + def init(self):# -> bool: + """ + Initialize the class ability to answer for ping. Part of Adapter required API. + """ + value = ":)" + success = self.write_parameter_by_storage_path(self.pingStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", self.pingStoragePath) + return False + return True + + def get_parameter_storage_path(self, parameterUrlPath):# -> bool, str: + """ + Conversion function - taking a key's path and translate to a file path on the file system + """ + #verifying the existance of "/" at the beging + if not parameterUrlPath.startswith("/"): + self.logger.error("Failed to translate url path '%s': no leading '/'", parameterUrlPath) + return False, "" + # avoiding the use of os.path.join in purpuse. + # we do not want the "/" at the begining push us to the root path + return True, self.basePath + parameterUrlPath.replace("/", os.path.sep) + + def get_parameter_url_path_from_storage_path(self, parameterStoragePath):# -> str: + """ + Conversion function - taking file path on the file system and translate to key's path + """ + return True, "/"+os.path.relpath(parameterStoragePath, self.basePath).replace(os.path.sep, "/") + + def ping(self):# -> bool: + """ + Get value for the ping request by its path. Part of Adapter required API. + """ + success, value = self.read_parameter_by_storage_path(self.pingStoragePath) + if not success or value is None: + self.logger.error("no ping response") + return False + self.logger.debug("ping response: %s", value) + return True + + def read_parameter_by_storage_path(self, parameterStoragePath):# -> (bool, str): + """ + Reading the value from the provided file name. + """ + self.logger.debug("Get parameter by os path: %s", parameterStoragePath) + try: + with open(parameterStoragePath) as fd: + value = fd.read() + except Exception as e: + self.logger.exception("%s parameter os path not found.", parameterStoragePath) + return False, None + + self.logger.debug("Get parameter by os path %s succeed", parameterStoragePath) + return True, value + + def read_parameters_by_storage_path(self, parameterStoragePathPrefix, keyFilters):# -> (bool, dict(str, str)): + """ + Reading the values of the parameters the provided directory. + """ + self.logger.debug("Get parameters under os path %s", parameterStoragePathPrefix) + parameters = {} + + fileNames = [] + for (dirpath, _, filenames) in os.walk(parameterStoragePathPrefix): + fileNames += [os.path.join(dirpath, file) for file in filenames] + + for fileName in fileNames: + filteredOut = False + for filterName, filterfunc in keyFilters.items():#items() - supporting python 2&3 + if not filterfunc(fileName): + self.logger.debug("Parameter os path %s dropped, not matching filter '%s'", self.get_parameter_url_path_from_storage_path(fileName)[1], filterName) + filteredOut = True + break + if filteredOut: + continue + + success_path, parameterUrlPath = self.get_parameter_url_path_from_storage_path(fileName) + if not success_path: + self.logger.error("%s parameter os path is invalid.", fileName) + return False, None + success, value = self.read_parameter_by_storage_path(fileName) + if not success: + self.logger.error("%s parameter os path not found.", fileName) + return False, None + + parameters[parameterUrlPath] = value + + return True, parameters + + + def write_parameter_by_storage_path(self, parameterStoragePath, value):# -> bool: + """ + Writing the value to the provided file name. + """ + self.logger.debug("Set parameter by os path %s", parameterStoragePath) + try: + dirname = os.path.dirname(parameterStoragePath) + if dirname and not os.path.exists(dirname): + os.makedirs(dirname) + with open(parameterStoragePath, "w") as fd: + fd.write(value) + except Exception as e: + self.logger.exception("could not post parameter os path %s", parameterStoragePath) + return False + self.logger.debug("Set parameter os path %s done", parameterStoragePath) + return True + + def remove_parameter_by_storage_path(self, parameterStoragePath):# -> bool: + """ + Deleting the the provided file. + """ + self.logger.debug("Delete parameter os path %s", parameterStoragePath) + try: + os.remove(_parameterStoragePath) + except Exception as e: + self.logger.exception("could not delete parameter os path %s", parameterStoragePath) + return False + self.logger.debug("Delete parameter os path %s done", parameterStoragePath) + return True + diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-debug b/experimental/emulated_vault/opt/emulated-vault/vault-debug new file mode 100755 index 0000000000..c5be44ed99 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault-debug @@ -0,0 +1,190 @@ +#! /usr/bin/python + +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +from __future__ import print_function + +import inspect +import json +import logging +import optparse +import os +import sys + +import storage.fs_adapter +import vault.db + +if sys.version_info >= (3, 0): + #python 3 + from configparser import ConfigParser +else: + #python 2 + from ConfigParser import ConfigParser + + +def main(): + progPath = inspect.stack()[-1][1] + progAbsPath = os.path.abspath( progPath ) + progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) + + confDir = os.path.join(progAbsPath, "conf") + confFile = os.path.join(confDir, "cfg.ini") + try: + config = ConfigParser() + config.read(confFile) + except IOError as e: + print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to read configuration: {0}".format(e), file=sys.stderr) + return 1 + + generalCfg = dict(config.items("general")) if config.has_section("general") else {} + logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) + debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") + mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") + try: + if not os.path.exists(logDir): + os.makedirs(logDir) + except IOError as e: + print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to create log dir: {0}".format(e), file=sys.stderr) + return 1 + + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + # create file handler which logs even debug messages + fhd = logging.FileHandler(debugLogFile) + fhd.setLevel(logging.DEBUG) + fhm = logging.FileHandler(mainLogFile) + fhm.setLevel(logging.INFO) + #TODO Set based on command line + verbose = logging.StreamHandler(sys.stdout) + verbose.setLevel(logging.INFO) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fhm.setFormatter(formatter) + fhd.setFormatter(formatter) + verbose.setFormatter(formatter) + # add the handlers to the logger + logger.addHandler(fhm) + logger.addHandler(fhd) + logger.addHandler(verbose) + + parser = optparse.OptionParser() + operationModeGroup = optparse.OptionGroup(parser, "Operation mode options", + "The vault script may work in one of the following modes.") + parser.add_option_group(operationModeGroup) + + operationModeGroup.add_option("--ping", + action="store_true", dest="ping", default=False, help="Ping") + + operationModeGroup.add_option("--get-parameter", + action="store_true", dest="getParameter", default=False, + help="Get parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--set-parameter", + action="store_true", dest="setParameter", default=False, + help="Set parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--delete-parameter", + action="store_true", dest="deleteParameter", default=False, + help="Delete parameter. Parameter relative path should be provided.") + + operationModeGroup.add_option("--search-parameters", + action="store_true", dest="searchParameters", default=False, + help="Search parameter. Parameter relative path should be provided.") + + (options, args) = parser.parse_args() + + storageAdapterType = generalCfg.get("storage-adapter-type") + if not storageAdapterType: + logger.error("Missing storage adapter type cfg") + return 1 + elif storageAdapterType == "fs": + storageAdapter = storage.fs_adapter.FsAdapter(logger=logger) + else: + logger.error("Invalid storage adapter type '%s'", storageAdapterType) + return 1 + + + if not storageAdapter.init_cfg(config): + logger.error("Failed storage adapter initialization") + return 1 + if not storageAdapter.init(): + logger.error("Failed storage adaper initialization") + return 1 + + global db + db = vault.db.Db(logger, storageAdapter) + + if options.ping: + if len(args) != 0: + parser.error("Command should get no arguments.") + success, value = adapter.ping() + if not success: + return 1 + print(value) + return 0 + + if options.getParameter: + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success, value = db.getParameter(args[0]) + if not success: + return 1 + if value is None: + return 1 + print(value) + return 0 + + if options.searchParameters: + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success, value = db.searchParameters(args[0], keyFilters={}, filters={}) + if not success: + return 1 + print(json.dumps(value)) + return 0 + + if options.setParameter: + if len(args) != 2: + parser.error("Command should get 2 arguments - parameter key and value.") + success = db.setParameter(args[0], args[1]) + if not success: + return 1 + return 0 + + if options.deleteParameter: + if len(args) != 1: + parser.error("Command should get a single argument - parameter key.") + success = db.deleteParameter(args[0]) + if not success: + return 1 + return 0 + + logger.error("Operation is not set") + parser.print_help() + parser.error("No operation mode specified") + return 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/experimental/emulated_vault/opt/emulated-vault/vault-server b/experimental/emulated_vault/opt/emulated-vault/vault-server new file mode 100755 index 0000000000..f09a3ec023 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault-server @@ -0,0 +1,355 @@ +#! /usr/bin/python + +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +from __future__ import print_function + +import fnmatch +import inspect +import json +import logging +import os +import re +import ssl +import sys + +import storage.fs_adapter +import vault.db + +if sys.version_info >= (3, 0): + #python 3 + from http.server import HTTPServer + from http.server import BaseHTTPRequestHandler + from configparser import ConfigParser + import urllib.parse as urlparse +else: + #python 2 + from BaseHTTPServer import HTTPServer + from BaseHTTPServer import BaseHTTPRequestHandler + from ConfigParser import ConfigParser + import urlparse + + +class RequestHandler(BaseHTTPRequestHandler): + """ + An HTTP server, emulating RIAK behavior on the calls done by traffic-ops. + The class implements BaseHTTPRequestHandler functions + """ + + def do_GET(self): + """ + Base class function implementation - to be called upon HTTP GET/HEAD requests + """ + try: + self._do_GET() + except Exception as e: + logger.exception("do_GET exception") + + do_HEAD = do_GET + + def do_POST(self): + """ + Base class function implementation - to be called upon HTTP POST/PUT requests + """ + try: + self._do_POST() + except Exception as e: + logger.exception("do_POST exception") + + do_PUT = do_POST + + def do_DELETE(self): + """ + Base class function implementation - to be called upon HTTP DEL request + """ + try: + self._do_DELETE() + except Exception as e: + logger.exception("do_DELETE exception") + + def _do_GET(self): + """ + Actual GET logic + :raises: Exception + """ + logger.debug("GET %s", self.path) + parsed_path = urlparse.urlparse(self.path) + + if parsed_path.path == "/ping": + logger.info("Ping") + self._do_GET_ping() + elif parsed_path.path == "/search/query/sslkeys": + logger.info("Search SSL: path=%s query=%s", parsed_path.path, parsed_path.query) + self._do_GET_sslkeys(parsed_path) + else: + self._do_GET_general(parsed_path) + + def _do_GET_ping(self): + """ + Actual GET logic for ping request + :raises: Exception + """ + success, value = db.ping() + if not success: + self.send_response(503) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write("Failure".encode()) + return + + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(value.encode()) + + def _do_GET_sslkeys(self, parsed_path): + """ + Actual GET logic for ssl keys request + :param parsed_path: key's url path + :type parsed_path: str + :raises: Exception + """ + filters = {} + keyFilters = {} + cdnFind = re.search(".*q=cdn:([^&]*).*", parsed_path.query) + if cdnFind: + def cdnFilter(key,val): + try: + data = json.loads(val) + return data['cdn']==cdnFind.group(1) + except Exception as e: + return False + filters['cdn'] = cdnFilter + + dsFind = re.search(".*q=deliveryservice:([^&]*).*", parsed_path.query) + if dsFind: + def dsFilter(key,val): + try: + data = json.loads(val) + return data['deliveryservice']==dsFind.group(1) + except Exception as e: + return False + filters['ds'] = dsFilter + + keyFind = re.search(".*q=_yz_rk:([^&]*).*", parsed_path.query) + if keyFind: + keyFilters['key-match'] = lambda key: fnmatch.fnmatch(os.path.basename(key), keyFind.group(1)) + + success, parameters = db.searchParameters("/riak/ssl/", keyFilters=keyFilters, filters=filters) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + docs = [json.loads(val) for val in parameters.values()] + toReturn = {"response":{"numFound":len(docs),"start":0, "docs":docs}} + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(json.dumps(toReturn).encode()) + + def _do_GET_general(self, parsed_path): + """ + Actual GET logic for general variable request + :param parsed_path: key's url path + :type parsed_path: str + :raises: Exception + """ + success, value = db.getParameter(parsed_path.path) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + if value is None: + self.send_response(404) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Not found"}'.encode()) + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(value.encode()) + + def _do_POST(self): + """ + Actual POST request logic + """ + logger.info("POST %s", self.path) + parsed_path = urlparse.urlparse(self.path) + content_len = int(self.headers.getheader('content-length')) + post_body = self.rfile.read(content_len) + data = json.loads(post_body) + #mimic vault beahvior + if parsed_path.path.startswith('/riak/ssl/'): + certificate_data = data.get('certificate', {}) + data.update({"certificate.%s"%key: value for key, value in certificate_data.items()}) + success = db.setParameter(parsed_path.path, json.dumps(data)) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + self.send_response(204) + self.send_header('Content-Type', 'application/json') + self.end_headers() + return + + + def _do_DELETE(self): + """ + Actual DEL request logic + """ + logger.info("DELETE %s", self.path) + parsed_path = urlparse.urlparse(self.path) + + success = db.deleteParameter(parsed_path.path) + if not success: + self.send_response(503) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Failure"}'.encode()) + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write('{"Deleted"}'.encode()) + return + + + +def main(): + progPath = inspect.stack()[-1][1] + progAbsPath = os.path.abspath( progPath ) + progAbsPath = os.path.dirname(os.path.normpath(progAbsPath)) + + confDir = os.path.join(progAbsPath, "conf") + confFile = os.path.join(confDir, "cfg.ini") + try: + config = ConfigParser() + config.read(confFile) + except IOError as e: + print("Failed to read configuration - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to read configuration: {0}".format(e), file=sys.stderr) + return 1 + + generalCfg = dict(config.items("general")) if config.has_section("general") else {} + logDir = generalCfg.get("log-dir", os.path.join(progAbsPath, "var/log")) + debugLogFile = os.path.join(logDir, "traffic-ops-vault-debug.log") + mainLogFile = os.path.join(logDir, "traffic-ops-vault.log") + try: + if not os.path.exists(logDir): + os.makedirs(logDir) + except IOError as e: + print("Failed to create log dir - I/O error({0}): {1}".format(e.errno, e.strerror), file=sys.stderr) + return 1 + except Exception as e: + print("Failed to create log dir: {0}".format(e), file=sys.stderr) + return 1 + + + global logger + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + # create file handler which logs even debug messages + fhd = logging.FileHandler(debugLogFile) + fhd.setLevel(logging.DEBUG) + fhm = logging.FileHandler(mainLogFile) + fhm.setLevel(logging.INFO) + # create formatter and add it to the handlers + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + fhm.setFormatter(formatter) + fhd.setFormatter(formatter) + # add the handlers to the logger + logger.addHandler(fhm) + logger.addHandler(fhd) + + httpServerCfg = dict(config.items("http-server")) if config.has_section("http-server") else {} + listenIP = httpServerCfg.get("listen-ip", "0.0.0.0") + try: + listenPort = int(httpServerCfg.get("listen-port", "8088")) + except Exception as e: + logger.exception("Failed %s integer conversion failed", "listen-port") + return 1 + use_ssl_val = httpServerCfg.get("use-ssl", "True") + if use_ssl_val in ["False", "false", "0"]: + use_ssl=False + elif use_ssl_val in ["True", "true", "1"]: + use_ssl=True + else: + logger.error("Invalid %s value", "use-ssl") + return 1 + if use_ssl: + sslKey = httpServerCfg.get("ssl-key-path") + if not sslKey: + print("Missing configuration: {0}/{1}".format("http-server", "ssl-key-path"), file=sys.stderr) + return 1 + sslCert = httpServerCfg.get("ssl-cert-path") + if not sslCert: + print("Missing configuration: {0}/{1}".format("http-server", "ssl-cert-path"), file=sys.stderr) + return 1 + + storageAdapterType = generalCfg.get("storage-adapter-type") + if not storageAdapterType: + logger.error("Missing storage adapter type cfg") + return 1 + elif storageAdapterType == "fs": + storageAdapter = storage.fs_adapter.FsAdapter(logger=logger) + else: + logger.error("Invalid storage adapter type '%s'", storageAdapterType) + return 1 + + + if not storageAdapter.init_cfg(config): + logger.error("Failed storage adapter initialization") + return 1 + if not storageAdapter.init(): + logger.error("Failed storage adaper initialization") + return 1 + + global db + db = vault.db.Db(logger, storageAdapter) + + server = HTTPServer((listenIP, listenPort), RequestHandler) + if use_ssl: + try: + server.socket = ssl.wrap_socket (server.socket, keyfile=sslKey, certfile=sslCert, server_side=True) + except Exception as e: + logger.exception("Failed on SSL init") + return 1 + + msg = 'Starting server at http%s://%s:%d'%("s" if use_ssl else "", listenIP, listenPort) + print(msg, file=sys.stderr) + logger.info(msg) + server.serve_forever() + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/experimental/emulated_vault/opt/emulated-vault/vault/__init__.py b/experimental/emulated_vault/opt/emulated-vault/vault/__init__.py new file mode 100644 index 0000000000..dc9911a6d6 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault/__init__.py @@ -0,0 +1,16 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# diff --git a/experimental/emulated_vault/opt/emulated-vault/vault/db.py b/experimental/emulated_vault/opt/emulated-vault/vault/db.py new file mode 100644 index 0000000000..b39a0739b1 --- /dev/null +++ b/experimental/emulated_vault/opt/emulated-vault/vault/db.py @@ -0,0 +1,164 @@ +# +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# + +class Db(object): + """ + DB class representing the DB layer. + + Implemented methods + :meth:`ping` test the 'ping' status with the adapter. + Return a tuple: "success" bool & "value" kept as ping variable + :meth:`getParameter` given a parameter key (in url-path format) retrieve the parameter value. + Return a tuple: "success" bool & "value" kept in the parameter + :meth:`searchParameter` given a parameters key prefix (in url-path format) and, a dict holding + variable key filters, and a key holding filters on the values as well. + Return "success" bool indicating a sucessful write, and a key->value dictionary + for the relevant parameters + :meth:`setParameter` given a parameter key (in url-path format) and a value string, + keep the parameter value. Return "success" bool indicating a sucessful write + :meth:`deleteParameter` given a parameter key (in url-path format), delete the parameter + from the DB. Return "success" bool indicating a sucessful deletion + """ + + def __init__ (self, logger, storage_adaper): + """ + The class constructor. + :param logger: logger to send log messages to + :type logger: a python logging logger class + :param storage_adaper: an initalized storage adapter + :type storage_adaper: a storage.adapter_base.AdapterBase class + """ + + self.logger = logger + self.storage_adaper = storage_adaper + + def ping(self): + """ + get value for the ping request. Part of Adapter required API. + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[bool, str] + """ + if self.storage_adaper.ping(): + return (True, "OK") + return False, None + + def getParameter(self, parameterUrlPath): + """ + Get value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: A tuple - 'True' for successful retrival and the retrieved value + :rtype: Tuple[bool, str] + """ + success_path, parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path: %s", parameterUrlPath) + return False, None + success, value = self.storage_adaper.read_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to bring parameter %s", parameterUrlPath) + return False, None + if value is None: + self.logger.error("Could not find parameter %s", parameterUrlPath) + return True, None + self.logger.debug("Parameter get response for %s ready", parameterUrlPath) + return True, value + + def searchParameters(self, parameterKeyPrefixUrlPath, keyFilters, filters): + """ + Get key/value dict of parameters by key pprefix + :param parameterKeyPrefixUrlPath: the key prefix for the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterKeyPrefixUrlPath: str + :param keyFilters: a dictionary of filter-name/filter-functions - + each function gets a single variable - a parameter key - + and return "True" if the variable should be included in the response. + :type keyFilters: Dict[str, function[str]] + :param filters: a dictionary of filter-name/filter-functions - + each function gets 2 variable - a parameter key and val - + and return "True" if the variable should be included in the response. + :type filters: Dict[str, function[str,str]] + :return: 'True' for successful retrival and the retrieved values dict (parameter-key/value) + :rtype: Tuple[bool, Dict[str, str]] + """ + self.logger.debug("Get parameters under path %s", parameterKeyPrefixUrlPath) + success_path, parameterStoragePathPrefix = self.storage_adaper.get_parameter_storage_path(parameterKeyPrefixUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path prefix: %s", parameterKeyPrefixUrlPath) + return False, None + success, items = self.storage_adaper.read_parameters_by_storage_path(parameterStoragePathPrefix, keyFilters=keyFilters) + if not success: + self.logger.error("Failed to bring parameters by prefix %s", parameterStoragePathPrefix) + return False, None + + filtered = {} + for key, val in items.items():#items() - supporting python 2&3 + skip = False + for filterName, filterfunc in filters.items():#items() - supporting python 2&3 + if not filterfunc(key, val): + self.logger.debug("Parameter %s dropped, not matching filter %s", key, filterName) + skip = True + break + if skip: + continue + filtered[key] = val + + return True, filtered + + def setParameter(self, parameterUrlPath, value): + """ + Set value for the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :param value: the value to be kept + :type value: str + :return: 'True' for successful settings + :rtype: bool + """ + self.logger.debug("Set parameter %s", parameterUrlPath) + success_path, parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path: %s", parameterUrlPath) + return False, None + success = self.storage_adaper.write_parameter_by_storage_path(parameterStoragePath, value) + if not success: + self.logger.error("Failed to set parameter %s", parameterUrlPath) + return False + return True + + def deleteParameter(self, parameterUrlPath): + """ + Delete the specified parameter. Part of Adapter required API. + :param parameterUrlPath: the key of the parameter as presented as url path + (tokens seperated by "/", with "/" as a prefix) + :type parameterUrlPath: str + :return: 'True' for successful deletion + :rtype: bool + """ + self.logger.debug("Delete parameter %s", parameterUrlPath) + success_path, parameterStoragePath = self.storage_adaper.get_parameter_storage_path(parameterUrlPath) + if not success_path: + self.logger.error("Invalid parameter url path: %s", parameterUrlPath) + return False, None + success = self.storage_adaper.remove_parameter_by_storage_path(parameterStoragePath) + if not success: + self.logger.error("Failed to delete parameter %s", parameterUrlPath) + return False + return True +