diff --git a/Makefile.am b/Makefile.am index 727deee2b55..f3f0783bb54 100644 --- a/Makefile.am +++ b/Makefile.am @@ -27,7 +27,7 @@ export CCACHE_BASEDIR # and mgmt, hence we have to build proxy/hdrs first. # depends on the generates ts/ts.h include file. -SUBDIRS = include src/tscpp/util lib src/tscore iocore proxy mgmt src plugins tools example rc configs tests +SUBDIRS = include src/tscpp/util lib src/tscore iocore proxy mgmt mgmt2 src plugins tools example rc configs tests if BUILD_DOCS SUBDIRS += doc include @@ -141,6 +141,7 @@ clang-format-lib: clang-format-mgmt: @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/mgmt + @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/mgmt2 clang-format-plugins: @$(top_srcdir)/tools/clang-format.sh $(top_srcdir)/plugins diff --git a/configure.ac b/configure.ac index 27aa527beeb..994163bbf9d 100644 --- a/configure.ac +++ b/configure.ac @@ -540,13 +540,24 @@ AM_CONDITIONAL([BUILD_IMAGE_MAGICK_PLUGINS], [test "x${enable_image_magick_plugi AC_MSG_CHECKING([whether to install example plugins]) AC_ARG_ENABLE([example-plugins], - [AS_HELP_STRING([--enable-example-plugins],[build and install example plugins])], + [AS_HELP_STRING([--enable-example-plugins],[Build and install example plugins])], [], [enable_example_plugins=no] ) AC_MSG_RESULT([$enable_example_plugins]) AM_CONDITIONAL([BUILD_EXAMPLE_PLUGINS], [ test "x${enable_example_plugins}" = "xyes" ]) +# +# JSONRPC 2.0 build options. +# +AC_MSG_CHECKING([whether to build & install non jsonrpc traffic_ctl]) +AC_ARG_ENABLE([legacy-traffic_ctl], + [AS_HELP_STRING([--enable-legacy-traffic_ctl],[build and install legacy traffic_ctl. No jsonrpc will be supported through traffic_ctl])], + [], + [enable_legacy_tc=no] +) +AC_MSG_RESULT([$enable_legacy_tc]) +AM_CONDITIONAL([BUILD_LEGACY_TC], [ test "x${enable_legacy_tc}" = "xyes" ]) # # Test tools. The test tools are always built, but not always installed. Installing @@ -2275,6 +2286,9 @@ AC_CONFIG_FILES([ mgmt/api/include/Makefile mgmt/utils/Makefile plugins/Makefile + mgmt2/Makefile + mgmt2/rpc/Makefile + mgmt2/config/Makefile proxy/Makefile proxy/hdrs/Makefile proxy/http/Makefile diff --git a/doc/admin-guide/index.en.rst b/doc/admin-guide/index.en.rst index 0557b691c4a..664ff41a84f 100644 --- a/doc/admin-guide/index.en.rst +++ b/doc/admin-guide/index.en.rst @@ -40,6 +40,7 @@ Table of Contents: layer-4-routing.en performance/index.en files/index.en + jsonrpc/index.en Audience ======== diff --git a/doc/admin-guide/jsonrpc/index.en.rst b/doc/admin-guide/jsonrpc/index.en.rst new file mode 100644 index 00000000000..84f0d1d5c34 --- /dev/null +++ b/doc/admin-guide/jsonrpc/index.en.rst @@ -0,0 +1,103 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. highlight:: cpp +.. default-domain:: cpp + +.. |RPC| replace:: JSONRPC 2.0 + +.. _admin-jsonrpc: + +JSONRPC +******* + +.. _admin-jsonrpc-description: + +Description +=========== + +|TS| Implements and exposes management calls using a JSONRPC API. This API is base on the following two things: + +* `JSON `_ format. Lightweight data-interchange format. It is easy for humans to read and write. + It is easy for machines to parse and generate. It's basically a collection of name/value pairs. + +* `JSONRPC 2.0 `_ protocol. Stateless, light-weight remote procedure call (RPC) protocol. + Primarily this specification defines several data structures and the rules around their processing. + + +In order for programs to communicate with |TS|, the server exposes a ``JSONRRPC 2.0`` API where programs can communicate with it. + + + + +.. _admnin-jsonrpc-configuration: + +Configuration +============= + +The |RPC| server can be configured using the following configuration file. + + +.. note:: + + |TS| will start the |RPC| server without any need for configuration. + + +If a non-default configuration is needed, the following describes the structure. + + +File `jsonrpc.yaml` is a YAML format. The default configuration is:: + + + #YAML + rpc: + enabled: true + unix: + lock_path_name: /tmp/conf_jsonrpc.lock + sock_path_name: /tmp/conf_jsonrpc.sock + backlog: 5 + max_retry_on_transient_errors: 64 + restricted_api: true + + +===================== ========================================================================================= +Field Name Description +===================== ========================================================================================= +``enabled`` Enable/disable toggle for the whole implementation, server will not start if this is false/no +``unix`` Specific definitions as per transport. +===================== ========================================================================================= + + +IPC Socket (``unix``): + +===================================== ========================================================================================= +Field Name Description +===================================== ========================================================================================= +``lock_path_name`` Lock path, including the file name. (changing this may have impacts in :program:`traffic_ctl`) +``sock_path_name`` Sock path, including the file name. This will be used as ``sockaddr_un.sun_path``. (changing this may have impacts in :program:`traffic_ctl`) +``backlog`` Check https://man7.org/linux/man-pages/man2/listen.2.html +``max_retry_on_transient_errors`` Number of times the implementation is allowed to retry when a transient error is encountered. +``restricted_api`` Used to set rpc unix socket permissions. If restricted `0700` will be set, otherwise `0777`. ``true`` by default. +===================================== ========================================================================================= + + +.. note:: + + Currently, there is only 1 communication mechanism supported. Unix Domain Sockets + diff --git a/doc/appendices/command-line/traffic_crashlog.en.rst b/doc/appendices/command-line/traffic_crashlog.en.rst index fe11a88b23a..9be4b672827 100644 --- a/doc/appendices/command-line/traffic_crashlog.en.rst +++ b/doc/appendices/command-line/traffic_crashlog.en.rst @@ -77,10 +77,6 @@ Options Caveats ======= -:program:`traffic_crashlog` makes use of various Traffic Server management -APIs. If :ref:`traffic_manager` is not available, the crash log will be -incomplete. - :program:`traffic_crashlog` may generate a crash log containing information you would rather not share outside your organization. Please examine the crash log carefully before posting it in a public forum. @@ -89,5 +85,4 @@ See also ======== :manpage:`records.config(5)`, -:manpage:`traffic_manager(8)`, :manpage:`traffic_server(8)` diff --git a/doc/appendices/command-line/traffic_ctl_jsonrpc.en.rst b/doc/appendices/command-line/traffic_ctl_jsonrpc.en.rst new file mode 100644 index 00000000000..89de8961fa8 --- /dev/null +++ b/doc/appendices/command-line/traffic_ctl_jsonrpc.en.rst @@ -0,0 +1,612 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. |RPC| replace:: JSONRPC 2.0 + +.. _JSONRPC: https://www.jsonrpc.org/specification +.. _JSON: https://www.json.org/json-en.html + +.. _traffic_ctl_jsonrpc: + +traffic_ctl +*********** + +Synopsis +======== + +:program:`traffic_ctl` [OPTIONS] SUBCOMMAND [OPTIONS] + + +.. note:: + + :program:`traffic_ctl` now uses a `JSONRPC`_ endpoint instead of ``traffic_manager``. ``traffic_manager`` is not + required. To build this version of :program:`traffic_ctl` ``--enable-jsonrpc-tc`` should be passed when configure the build. + +Description +=========== + +:program:`traffic_ctl` is used to display,manipulate and configure +a running Traffic Server. :program:`traffic_ctl` includes a number +of subcommands that control different aspects of Traffic Server: + + +:program:`traffic_ctl config` + Manipulate and display configuration records +:program:`traffic_ctl metric` + Manipulate performance and status metrics +:program:`traffic_ctl server` + Stop, restart and examine the server +:program:`traffic_ctl storage` + Manipulate cache storage +:program:`traffic_ctl plugin` + Interact with plugins. +:program:`traffic_ctl host` + Manipulate host status. parents for now but will be expanded to origins. +:program:`traffic_ctl rpc` + Interact directly with the |RPC| server in |TS| + + + + +Options +======= + +.. program:: traffic_ctl +.. option:: --debug + + Enable debugging output. + +.. option:: -V, --version + + Print version information and exit. + +.. option:: -f, --format + + Specify the output print style. + + =================== ======================================================================== + Options Description + =================== ======================================================================== + ``legacy`` Will honour the old :program:`traffic_ctl` output messages. This is the default format type. + ``pretty`` will print a different output, a prettier output. This depends on the implementation, + it's not required to always implement a pretty output + ``json`` It will show the response message formatted to `JSON`_. This is ideal if you want to redirect the stdout to a different source. + It will only stream the json response, no other messages. + ``data:`` This is an addon to the default format style, data can be: ``{req|resp|all}`` which will make :program:`traffic_ctl` + to print in json format the request or response or both. + =================== ======================================================================== + + In case of a record request(config) ``--records`` overrides this flag. + + Default: ``legacy`` + + Example: + + .. code-block:: + + traffic_ctl config get variable --format data:req + --> {request} + + .. code-block:: + + $ traffic_ctl config get variable --format data:resp + <-- {response} + + .. code-block:: + + $ traffic_ctl config get variable --format data:all + --> {request} + <-- {response} + + .. code-block:: + + $ traffic_ctl config get variable --format json + {response} + + There will be no print out beside the json response. This is ideal to redirect to a file. + + +.. option:: --records + + Option available only for records request. + +.. option:: --run-root + + Path to the runroot file. + +Subcommands +=========== + +.. _traffic-control-command-alarm: + +traffic_ctl alarm +----------------- + +.. warning:: + + Option not available in the |RPC| version. + +.. _traffic-control-command-config: + +traffic_ctl config +------------------ + +.. program:: traffic_ctl config +.. option:: defaults [--records] + + :ref:`admin_lookup_records` + + Display the default values for all configuration records. The ``--records`` flag has the same + behavior as :option:`traffic_ctl config get --records`. + +.. program:: traffic_ctl config +.. option:: describe RECORD [RECORD...] + + :ref:`admin_lookup_records` + + Display all the known information about a configuration record. This includes the current and + default values, the data type, the record class and syntax checking expression. + + Error output available if ``--format pretty`` is specified. + +.. program:: traffic_ctl config +.. option:: diff [--records] + + :ref:`admin_lookup_records` + + Display configuration records that have non-default values. The ``--records`` flag has the same + behavior as :option:`traffic_ctl config get --records`. + +.. program:: traffic_ctl config +.. option:: get [--records] RECORD [RECORD...] + + :ref:`admin_lookup_records` + + Display the current value of a configuration record. + + Error output available if ``--format pretty`` is specified. + +.. program:: traffic_ctl config get +.. option:: --records + + If this flag is provided, :option:`traffic_ctl config get` will emit results in + :file:`records.config` format. + +.. program:: traffic_ctl config +.. option:: match [--records] REGEX [REGEX...] + + :ref:`admin_lookup_records` + + Display the current values of all configuration variables whose names match the given regular + expression. The ``--records`` flag has the same behavior as :option:`traffic_ctl config get + --records`. + +.. program:: traffic_ctl config +.. option:: reload + + :ref:`admin_config_reload` + + Initiate a Traffic Server configuration reload. Use this command to update the running + configuration after any configuration file modification. If no configuration files have been + modified since the previous configuration load, this command is a no-op. + + The timestamp of the last reconfiguration event (in seconds since epoch) is published in the + `proxy.node.config.reconfigure_time` metric. + +.. program:: traffic_ctl config +.. option:: set RECORD VALUE + + :ref:`admin_config_set_records` + + Set the named configuration record to the specified value. Refer to the :file:`records.config` + documentation for a list of the configuration variables you can specify. Note that this is not a + synchronous operation. + +.. program:: traffic_ctl config +.. option:: status + + :ref:`admin_lookup_records` + + Display detailed status about the Traffic Server configuration system. This includes version + information, whether the internal configuration store is current and whether any daemon processes + should be restarted. + +.. program:: traffic_ctl config +.. option:: registry + + :ref:`filemanager.get_files_registry` + + Display information about the registered files in |TS|. This includes the full file path, config record name, parent config (if any) + if needs root access and if the file is required in |TS|. + +.. _traffic-control-command-metric: + +traffic_ctl metric +------------------ + +.. program:: traffic_ctl metric +.. option:: get METRIC [METRIC...] + + :ref:`admin_lookup_records` + + Display the current value of the specified statistics. + + Error output available if ``--format pretty`` is specified. + +.. program:: traffic_ctl metric +.. option:: match REGEX [REGEX...] + + :ref:`admin_lookup_records` + + Display the current values of all statistics whose names match + the given regular expression. + +.. program:: traffic_ctl metric +.. option:: zero METRIC [METRIC...] + + :ref:`admin_clear_metrics_records` + + Reset the named statistics to zero. + +.. program:: traffic_ctl metric +.. option:: describe RECORD [RECORD...] + + :ref:`admin_lookup_records` + + Display all the known information about a metric record. + + Error output available if ``--format pretty`` is specified. + +.. _traffic-control-command-server: + +traffic_ctl server +------------------ + +.. program:: traffic_ctl server + +.. program:: traffic_ctl server +.. option:: drain + + :ref:`admin_server_start_drain` + + :ref:`admin_server_stop_drain` + + Drop the number of active client connections. + +.. program:: traffic_ctl server +.. option:: status + + Option not yet available + +.. _traffic-control-command-storage: + +traffic_ctl storage +------------------- + +.. program:: traffic_ctl storage +.. option:: offline PATH [PATH ...] + + :ref:`admin_storage_set_device_offline` + + Mark a cache storage device as offline. The storage is identified by :arg:`PATH` which must match + exactly a path specified in :file:`storage.config`. This removes the storage from the cache and + redirects requests that would have used this storage to other storage. This has exactly the same + effect as a disk failure for that storage. This does not persist across restarts of the + :program:`traffic_server` process. + +.. program:: traffic_ctl storage +.. option:: status PATH [PATH ...] + + :ref:`admin_storage_get_device_status` + + Show the storage configuration status. + +.. _traffic-control-command-plugin: + +traffic_ctl plugin +------------------- + +.. program:: traffic_ctl plugin +.. option:: msg TAG DATA + + :ref:`admin_plugin_send_basic_msg` + + Send a message to plugins. All plugins that have hooked the + ``TSLifecycleHookID::TS_LIFECYCLE_MSG_HOOK`` will receive a callback for that hook. + The :arg:`TAG` and :arg:`DATA` will be available to the plugin hook processing. It is expected + that plugins will use :arg:`TAG` to select relevant messages and determine the format of the + :arg:`DATA`. + +.. _traffic-control-command-host: + +traffic_ctl host +---------------- +.. program:: traffic_ctl host + +A stat to track status is created for each host. The name is the host fqdn with a prefix of +"proxy.process.host_status". The value of the stat is a string which is the serialized +representation of the status. This contains the overall status and the status for each reason. The +stats may be viewed using the :program:`traffic_ctl metric` command or through the `stats_over_http` +endpoint. + +.. option:: --time count + + Set the duration of an operation to ``count`` seconds. A value of ``0`` means no duration, the + condition persists until explicitly changed. The default is ``0`` if an operation requires a time + and none is provided by this option. + +.. option:: --reason active | local | manual + + Sets the reason for the operation. + + ``active`` + Set the active health check reason. + + ``local`` + Set the local health check reason. + + ``manual`` + Set the administrative reason. This is the default reason if a reason is needed and not + provided by this option. + + Internally the reason can be ``self_detect`` if + :ts:cv:`proxy.config.http.parent_proxy.self_detect` is set to the value 2 (the default). This is + used to prevent parent selection from creating a loop by selecting itself as the upstream by + marking this reason as "down" in that case. + + .. note:: + + The up / down status values are independent, and a host is consider available if and only if + all of the statuses are "up". + +.. option:: status HOSTNAME [HOSTNAME ...] + + :ref:`admin_lookup_records` + + Get the current status of the specified hosts with respect to their use as targets for parent + selection. This returns the same information as the per host stat. + +.. option:: down HOSTNAME [HOSTNAME ...] + + :ref:`admin_host_set_status` + + Marks the listed hosts as down so that they will not be chosen as a next hop parent. If + :option:`--time` is included the host is marked down for the specified number of seconds after + which the host will automatically be marked up. A host is not marked up until all reason codes + are cleared by marking up the host for the specified reason code. + + Supports :option:`--time`, :option:`--reason`. + +.. option:: up HOSTNAME [HOSTNAME ...] + + :ref:`admin_host_set_status` + + Marks the listed hosts as up so that they will be available for use as a next hop parent. Use + :option:`--reason` to mark the host reason code. The 'self_detect' is an internal reason code + used by parent selection to mark down a parent when it is identified as itself and + + + Supports :option:`--reason`. + +.. _traffic_ctl_rpc: + +traffic_ctl rpc +--------------- +.. program:: traffic_ctl rpc + +A mechanism to interact directly with the |TS| |RPC| endpoint. This means that this is not tied to any particular API +but rather to the rpc endpoint, so you can directly send requests and receive responses from the server. + +.. option:: file + + Reads a file or a set of files from the disc, use the content of the files as message(s) to the |RPC| endpoint. All jsonrpc messages + will be validated before sending them. If the file contains invalid json|yaml format the message will not be send, in + case of a set of files, if a particular file is not a proper json/yaml format then that particular file will be skipped. + + Example: + + .. code-block:: bash + + traffic_ctl rpc file jsonrpc_cmd1.json jsonrpc_cmd2.yaml + +.. option:: get-api + + :ref:`show_registered_handlers` + + Request the entire admin api. This will retrieve all the registered methods and notifications on the server side. + + Example: + + .. code-block:: bash + + $ traffic_ctl rpc get-api + Methods: + - admin_host_set_status + - admin_server_stop_drain + - admin_server_start_drain + - admin_clear_metrics_records + - admin_clear_all_metrics_records + - admin_plugin_send_basic_msg + - admin_lookup_records + - admin_config_set_records + - admin_storage_get_device_status + - admin_storage_set_device_offline + - admin_config_reload + - show_registered_handlers + Notifications: + - some_registered_notification_handler + + +.. option:: input + + Input mode, traffic_ctl will provide a control input from a stream buffer. Once the content is written the terminal :program:`traffic_ctl` + will wait for the user to press Control-D to send the request to the rpc endpoint. + This feature allows you to directly interact with the jsonrpc endpoint and test your API easily and without the need to know the low level + implementation details of the transport layer. + :program:`traffic_ctl` will validate the input format, not the message content. The message content will be validated by the server. + See example `input_example_2`_. + + .. option:: --raw, -r + + No json/yaml parse validation will take place, the input content will be directly send to the server. + + Example: + + .. code-block:: + + $ traffic_ctl rpc input + >> Ctrl-D to fire the request + { + "id":"86e59b43-185b-4a0b-b1c1-babb1a3d5401", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"proxy.config.diags.debug.tags", + "rec_types":[ + "1", + "16" + ] + } + ] + } + + + <-- Server's response. + { + "jsonrpc":"2.0", + "result":{ + "recordList":[ + { + "record":{ + "record_name":"proxy.config.diags.debug.tags", + "record_type":"3", + "version":"0", + "raw_stat_block":"0", + "order":"423", + "config_meta":{ + "access_type":"0", + "update_status":"0", + "update_type":"1", + "checktype":"0", + "source":"3", + "check_expr":"null" + }, + "record_class":"1", + "overridable":"false", + "data_type":"STRING", + "current_value":"rpc", + "default_value":"http|dns" + } + } + ] + }, + "id":"86e59b43-185b-4a0b-b1c1-babb1a3d5401" + } + + +.. _input_example_2: + + Example 2: + + You can see a valid json ``{}`` but an invalid |RPC| message. In this case the server is responding. + + .. code-block:: + + $ traffic_ctl rpc input + >> Ctrl-D to fire the request + {} + + < -- Server's response + { + "jsonrpc":"2.0", + "error":{ + "code":-32600, + "message":"Invalid Request" + } + } + +Examples +======== + +Mark down a host with `traffic_ctl` and view the associated host stats:: + + .. code-block:: bash + + # traffic_ctl host down cdn-cache-02.foo.com --reason manual + + # traffic_ctl metric match host_status + proxy.process.host_status.cdn-cache-01.foo.com HOST_STATUS_DOWN,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:DOWN:1556896844:0,SELF_DETECT:UP:0 + proxy.process.host_status.cdn-cache-02.foo.com HOST_STATUS_UP,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:UP:0:0,SELF_DETECT:UP:0 + proxy.process.host_status.cdn-cache-origin-01.foo.com HOST_STATUS_UP,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:UP:0:0,SELF_DETECT:UP:0 + +In the example above, 'cdn-cache-01.foo.com' is unavailable, `HOST_STATUS_DOWN` and was marked down +for the `manual` reason, `MANUAL:DOWN:1556896844:0`, at the time indicated by the UNIX time stamp +`1556896844`. To make the host available, one would have to clear the `manual` reason using: + + .. code-block:: bash + + # traffic_ctl host up cdn-cache-01.foo.com --reason manual + +Configure Traffic Server to insert ``Via`` header in the response to the client + + .. code-block:: bash + + # traffic_ctl config set proxy.config.http.insert_response_via_str 1 + # traffic_ctl config reload + +Autest +====== + +If you want to interact with |TS| under a unit test, then a few things need to be considered. + +- Runroot needs to be configured in order to let `traffic_ctl` knows where to find the socket. + + There are currently two ways to do this: + + 1. Using `run-root` param. + + 1. Let `Test.MakeATSProcess` to create the runroot file under the |TS| config directory. This can be done by passing `dump_runroot=True` to the above function: + + .. code-block:: python + + ts = Test.MakeATSProcess(..., dump_runroot=True) + + + `dump_runroot` will write out some of the keys inside the runroot file, in this case the `runtimedir`. + + 1. Then you should specify the :option:`traffic_ctl --run-root` when invoking the command: + + .. code-block:: python + + tr.Processes.Default.Command = f'traffic_ctl config reload --run-root {ts.Disk.runroot_yaml.Name}' + + 2. Setting up the `TS_RUNROOT` environment variable. + This is very similar to `1` but, instead of passing the `--run-root` param to `traffic_ctl`, you just need to specify the + `TS_RUNROOT` environment variable. To do that, just do as 1.1 shows and then: + + .. code-block:: python + + ts.SetRunRootEnv() + + The above call will set the variable, please be aware that this variable will also be read by TS. + +See also +======== + +:manpage:`records.config(5)`, +:manpage:`storage.config(5)`, +:ref:`admnin-jsonrpc-configuration`, +:ref:`jsonrpc-protocol` diff --git a/doc/developer-guide/index.en.rst b/doc/developer-guide/index.en.rst index 471a6fe92d0..c273039c945 100644 --- a/doc/developer-guide/index.en.rst +++ b/doc/developer-guide/index.en.rst @@ -58,3 +58,4 @@ duplicate bugs is encouraged, but not required. design-documents/index.en layout/index.en testing/index.en + jsonrpc/index.en diff --git a/doc/developer-guide/jsonrpc/HandlerError.en.rst b/doc/developer-guide/jsonrpc/HandlerError.en.rst new file mode 100644 index 00000000000..03f824b9dfc --- /dev/null +++ b/doc/developer-guide/jsonrpc/HandlerError.en.rst @@ -0,0 +1,67 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. highlight:: cpp +.. default-domain:: cpp + +.. _jsonrpc-handler-errors: + +API Handler error codes +*********************** + +High level handler error codes, each particular handler can be fit into one of the following categories. +A good approach could be the following. This required coordination among all the errors, just for now, this soluction seems ok. + +.. code-block:: cpp + + enum YourOwnHandlerEnum { + FOO_ERROR = Codes::SOME_CATEGORY, + ... + }; + + +.. class:: Codes + + .. enumerator:: CONFIGURATION = 1 + + Errors during configuration api handling. + + .. enumerator:: METRIC = 1000 + + Errors during metrics api handling. + + .. enumerator:: RECORD = 2000 + + Errors during record api handling. + + .. enumerator:: SERVER = 3000 + + Errors during server api handling. + + .. enumerator:: STORAGE = 4000 + + Errors during storage api handling. + + .. enumerator:: PLUGIN = 4000 + + Errors during plugion api handling. + + .. enumerator:: GENERIC = 30000 + + Errors during generic api handling, general errors. diff --git a/doc/developer-guide/jsonrpc/index.en.rst b/doc/developer-guide/jsonrpc/index.en.rst new file mode 100644 index 00000000000..3706257fd52 --- /dev/null +++ b/doc/developer-guide/jsonrpc/index.en.rst @@ -0,0 +1,34 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _developer-guide-jsonrpc: + +JSONRPC +******* + +.. toctree:: + :maxdepth: 2 + + jsonrpc-architecture.en + jsonrpc-api.en + jsonrpc-node.en + jsonrpc-handler-development.en + jsonrpc-client-api.en + traffic_ctl-development.en + HandlerError.en diff --git a/doc/developer-guide/jsonrpc/jsonrpc-api.en.rst b/doc/developer-guide/jsonrpc/jsonrpc-api.en.rst new file mode 100644 index 00000000000..b18cc4b532b --- /dev/null +++ b/doc/developer-guide/jsonrpc/jsonrpc-api.en.rst @@ -0,0 +1,1799 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. highlight:: cpp +.. default-domain:: cpp + +.. |RPC| replace:: JSONRPC 2.0 + +.. _JSONRPC: https://www.jsonrpc.org/specification +.. _JSON: https://www.json.org/json-en.html + +.. |str| replace:: ``string`` +.. |arraynum| replace:: ``array[number]`` +.. |arraynumstr| replace:: ``array[number|string]`` +.. |arraystr| replace:: ``array[string]`` +.. |num| replace:: *number* +.. |strnum| replace:: *string|number* +.. |object| replace:: *object* +.. |array| replace:: *array* +.. |optional| replace:: ``optional`` +.. |method| replace:: ``method`` +.. |notification| replace:: ``notification`` +.. |arrayrecord| replace:: ``array[record]`` +.. |arrayerror| replace:: ``array[errors]`` + +.. _jsonrpc-api: + +API +*** + +.. _jsonrpc-api-description: + +Description +=========== + +|TS| Implements and exposes management calls using a JSONRPC API. This API is base on the following two things: + +* `JSON `_ format. Lightweight data-interchange format. It is easy for humans to read and write. + It is easy for machines to parse and generate. It's basically a collection of name/value pairs. + +* `JSONRPC 2.0 `_ protocol. Stateless, light-weight remote procedure call (RPC) protocol. + Primarily this specification defines several data structures and the rules around their processing. + + +In order for programs to communicate with |TS|, the server exposes a ``JSONRRPC 2.0`` API where programs can communicate with it. + + +.. _admin-jsonrpc-api: + +Administrative API +================== + +This section describes how to interact with the administrative RPC API to interact with |TS| + +.. + _This: We should explain how to deal with permission once it's implemented. + + + +.. _Records: + +Records +------- + +When interacting with the admin api, there are a few structures that need to be understood, this section will describe each of them. + + +.. _RecordRequest: + +RPC Record Request +~~~~~~~~~~~~~~~~~~ + +To obtain information regarding a particular record(s) from |TS|, we should use the following fields in an *unnamed* json structure. + + +====================== ============= ================================================================================================================ +Field Type Description +====================== ============= ================================================================================================================ +``record_name`` |str| The name we want to query from |TS|. This is |optional| if ``record_name_regex`` is used. +``record_name_regex`` |str| The regular expression we want to query from |TS|. This is |optional| if ``record_name`` is used. +``rec_types`` |arraynumstr| |optional| A list of types that should be used to match against the found record. These types refer to ``RecT``. + Other values (in decimal) than the ones defined by the ``RecT`` ``enum`` will be ignored. If no type is + specified, the server will not match the type against the found record. +====================== ============= ================================================================================================================ + +.. note:: + + If ``record_name`` and ``record_name_regex`` are both provided, the server will not use any of them. Only one should be provided. + + +Example: + + #. Single record: + + .. code-block:: json + + { + "id":"2947819a-8563-4f21-ba45-bde73210e387", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"proxy.config.exec_thread.autoconfig.scale", + "rec_types":[ + 1, + 16 + ] + } + ] + } + + #. Multiple records: + + .. code-block:: json + :emphasize-lines: 5-12 + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [{ + "record_name": "proxy.config.exec_thread.autoconfig.scale" + }, + { + "record_name": "proxy.config.log.rolling_interval_sec", + "rec_types": [1] + } + ] + } + + #. Batch Request + + .. code-block:: json + + [ + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [{ + "record_name_regex": "proxy.config.exec_thread.autoconfig.scale", + "rec_types": [1] + }] + }, { + "id": "dam7018e-0720-11eb-abe2-001fc69dd123", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [{ + "record_name_regex": "proxy.config.log.rolling_interval_sec", + "rec_types": [1] + }] + } + ] + + +.. _RecordResponse: + + +RPC Record Response +~~~~~~~~~~~~~~~~~~~ + +When querying for a record(s), in the majority of the cases the record api will respond with the following json structure. + +=================== ==================== ======================================================================== +Field Type Description +=================== ==================== ======================================================================== +``recordList`` |arrayrecord| A list of record |object|. See `RecordRequestObject`_ +``errorList`` |arrayerror| A list of error |object|. See `RecordErrorObject`_ +=================== ==================== ======================================================================== + + +.. _RecordErrorObject: + +RPC Record Error Object +~~~~~~~~~~~~~~~~~~~~~~~ + +All errors that are found during a record query, will be returned back to the caller in the ``error_list`` field as part of the `RecordResponse`_ object. +The record errors have the following fields. + + +=================== ============= =========================================================================== +Field Type Description +=================== ============= =========================================================================== +``code`` |str| |optional| An error code that should be used to get a description of the error.(Add error codes) +``record_name`` |str| |optional| The associated record name, this may be omitted sometimes. +``message`` |str| |optional| A descriptive message. The server can omit this value. +=================== ============= =========================================================================== + + +Example: + + .. code-block:: json + :linenos: + + { + "code": "2007", + "record_name": "proxy.config.exec_thread.autoconfig.scale" + } + + +Examples: + +#. Request a non existing record among with an invalid type for a record: + + .. code-block:: json + :linenos: + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [ + { + "record_name": "non.existing.record" + }, + { + "record_name": "proxy.process.http.total_client_connections_ipv4", + "rec_types": [1] + } + ] + } + + Line ``7`` requests a non existing record and in line ``11`` we request a type that does not match the record's type. + + .. code-block:: json + :linenos: + + { + "jsonrpc":"2.0", + "result":{ + "errorList":[ + { + "code":"2000", + "record_name":"non.existing.record" + }, + { + "code":"2007", + "record_name":"proxy.process.http.total_client_connections_ipv4" + } + ] + }, + "id":"ded7018e-0720-11eb-abe2-001fc69cc946" + } + + In this case we get the response indicating that the requested fields couldn't be retrieved. See `RecordErrorObject`_ for more details. + + The error list can also be included among with a recordList + + .. code-block:: json + :linenos: + + { + "jsonrpc":"2.0", + "result":{ + "recordList":[] + ,"errorList":[ + { + "code":"2000", + "record_name":"non.existing.record" + }, + { + "code":"2007", + "record_name":"proxy.process.http.total_client_connections_ipv4" + } + ] + }, + "id":"ded7018e-0720-11eb-abe2-001fc69cc946" + } + + .. note:: + + If there is no errors to report, the "errorList" field will be represented as an empty list (``[]``). + +.. _RecordErrorObject-Enum: + + +JSONRPC Record Errors +~~~~~~~~~~~~~~~~~~~~~ + +The following errors could be generated when requesting record from the server. + +.. class:: RecordError + + .. enumerator:: RECORD_NOT_FOUND = 2000 + + Record not found. + + .. enumerator:: RECORD_NOT_CONFIG = 2001 + + Record is not a configuration type. + + .. enumerator:: RECORD_NOT_METRIC = 2002 + + Record is not a metric type. + + .. enumerator:: INVALID_RECORD_NAME = 2003 + + Invalid Record Name. + + .. enumerator:: VALIDITY_CHECK_ERROR = 2004 + + Validity check failed. + + .. enumerator:: GENERAL_ERROR = 2005 + + Error reading the record. + + .. enumerator:: RECORD_WRITE_ERROR = 2006 + + Generic error while writting the record. ie: RecResetStatRecord() returns REC_ERR_OKAY + + .. enumerator:: REQUESTED_TYPE_MISMATCH = 2007 + + The requested record's type does not match againts the passed type list. + + .. enumerator:: INVALID_INCOMING_DATA = 2008 + + This could be caused by an invalid value in the incoming request which may cause the parser to fail. + + +.. _RecordRequestObject: + +RPC Record Object +~~~~~~~~~~~~~~~~~ + +This is mapped from a ``RecRecord``, when requesting for a record the following information will be populated into a json |object|. +The ``record`` structure has the following members. + +=================== ======== ================================================================== +Record Field Type Description +=================== ======== ================================================================== +``current_value`` |str| Current value that is held by the record. +``default_value`` |str| Record's default value. +``name`` |str| Record's name +``order`` |str| Record's order +``overridable`` |str| Records's overridable configuration. +``raw_stat_block`` |str| Raw Stat Block Id. +``record_class`` |str| Record type. Mapped from ``RecT`` +``record_type`` |str| Record's data type. Mapped from RecDataT +``version`` |str| Record's version. +``stats_meta`` |object| Stats metadata `stats_meta`_ +``config_meta`` |object| Config metadata `config_meta`_ +=================== ======== ================================================================== + +* it will be either ``config_meta`` or ``stats_meta`` object, but never both* + + +.. _config_meta: + +Config Metadata + +=================== ======== ================================================================== +Record Field Type Description +=================== ======== ================================================================== +`access_type` |str| Access type. This is mapped from ``TSRecordAccessType``. +`check_expr` |str| Syntax checks regular expressions. +`checktype` |str| Check type, This is mapped from ``RecCheckT``. +`source` |str| Source of the configuration value. Mapped from RecSourceT +`update_status` |str| Update status flag. +`update_type` |str| How the records get updated. Mapped from RecUpdateT +=================== ======== ================================================================== + + +.. _stats_meta: + +Stats Metadata (TBC) + +=================== ======== ================================================================== +Record Field Type Description +=================== ======== ================================================================== +`persist_type` |str| Persistent type. This is mapped from ``RecPersistT`` +=================== ======== ================================================================== + + +Example with config meta: + + .. code-block:: json + :linenos: + + { + "record":{ + "record_name":"proxy.config.diags.debug.tags", + "record_type":"3", + "version":"0", + "raw_stat_block":"0", + "order":"421", + "config_meta":{ + "access_type":"0", + "update_status":"0", + "update_type":"1", + "checktype":"0", + "source":"3", + "check_expr":"null" + }, + "record_class":"1", + "overridable":"false", + "data_type":"STRING", + "current_value":"rpc", + "default_value":"http|dns" + } + } + +Example with stats meta: + + .. code-block:: json + :linenos: + + { + "record": { + "current_value": "0", + "data_type": "COUNTER", + "default_value": "0", + "order": "8", + "overridable": "false", + "raw_stat_block": "10", + "record_class": "2", + "record_name": "proxy.process.http.total_client_connections_ipv6", + "record_type": "4", + "stat_meta": { + "persist_type": "1" + }, + "version": "0" + } + } + +.. _jsonrpc-admin-api: + +JSONRPC API +=========== + +* `admin_lookup_records`_ + +* `admin_clear_all_metrics_records`_ + +* `admin_config_set_records`_ + +* `admin_config_reload`_ + +* `admin_clear_metrics_records`_ + +* `admin_clear_all_metrics_records`_ + +* `admin_host_set_status`_ + +* `admin_server_stop_drain`_ + +* `admin_server_start_drain`_ + +* `admin_plugin_send_basic_msg`_ + +* `admin_storage_get_device_status`_ + +* `admin_storage_set_device_offline`_ + +* `show_registered_handlers`_ + +* `get_service_descriptor`_ + +* `filemanager.get_files_registry`_ + +.. _jsonapi-management-records: + + +Records +======= + +.. _admin_lookup_records: + + +admin_lookup_records +-------------------- + +|method| + +Description +~~~~~~~~~~~ + +Obtain record(s) from TS. + + +Parameters +~~~~~~~~~~ + +* ``params``: A list of `RecordRequest`_ objects. + + +Result +~~~~~~ + +A list of `RecordResponse`_ . In case of any error obtaining the requested record, the `RecordErrorObject`_ |object| will be included. + + +Examples +~~~~~~~~ + +#. Request a configuration record, no errors: + + .. code-block:: json + + { + "id":"b2bb16a5-135a-4c84-b0a7-8d31ebd82542", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"proxy.config.log.rolling_interval_sec", + "rec_types":[ + "1", + "16" + ] + } + ] + } + +Response: + + .. code-block:: json + + { + "jsonrpc":"2.0", + "result":{ + "recordList":[ + { + "record":{ + "record_name":"proxy.config.log.rolling_interval_sec", + "record_type":"1", + "version":"0", + "raw_stat_block":"0", + "order":"410", + "config_meta":{ + "access_type":"0", + "update_status":"0", + "update_type":"1", + "checktype":"1", + "source":"3", + "check_expr":"^[0-9]+$" + }, + "record_class":"1", + "overridable":"false", + "data_type":"INT", + "current_value":"86400", + "default_value":"86400" + } + } + ] + ,"errorList":[] + }, + "id":"b2bb16a5-135a-4c84-b0a7-8d31ebd82542" + } + + +#. Request a configuration record, some errors coming back: + + .. code-block:: json + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [ + { + "rec_types": [1], + "record_name": "proxy.config.log.rolling_interval_sec" + }, + { + "record_name": "proxy.config.log.rolling_interv" + } + ] + } + + +Response: + + .. code-block:: json + + { + "jsonrpc":"2.0", + "result":{ + "recordList":[ + { + "record":{ + "record_name":"proxy.config.log.rolling_interval_sec", + "record_type":"1", + "version":"0", + "raw_stat_block":"0", + "order":"410", + "config_meta":{ + "access_type":"0", + "update_status":"0", + "update_type":"1", + "checktype":"1", + "source":"3", + "check_expr":"^[0-9]+$" + }, + "record_class":"1", + "overridable":"false", + "data_type":"INT", + "current_value":"86400", + "default_value":"86400" + } + } + ], + "errorList":[ + { + "code":"2000", + "record_name":"proxy.config.log.rolling_interv" + } + ] + }, + "id":"ded7018e-0720-11eb-abe2-001fc69cc946" + } + + +Request using a `regex` instead of the full name. + +.. note:: + + Regex lookups use ``record_name_regex` and not ``record_name``. Check `RecordRequestObject`_ . + +Examples +~~~~~~~~ + +#. Request a mix(config and stats) of records record using a regex, no errors: + + .. code-block:: json + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [ + { + "rec_types": [1], + "record_name_regex": "proxy.config.exec_thread.autoconfig.sca*" + }, + { + "rec_types": [2], + "record_name_regex": "proxy.process.http.total_client_connections_ipv" + } + ] + } + + + Response: + + .. code-block:: json + + { + "jsonrpc":"2.0", + "result":{ + "recordList":[ + { + "record":{ + "record_name":"proxy.config.exec_thread.autoconfig.scale", + "record_type":"2", + "version":"0", + "raw_stat_block":"0", + "order":"355", + "config_meta":{ + "access_type":"2", + "update_status":"0", + "update_type":"2", + "checktype":"0", + "source":"3", + "check_expr":"null" + }, + "record_class":"1", + "overridable":"false", + "data_type":"FLOAT", + "current_value":"1", + "default_value":"1" + } + }, + { + "record":{ + "record_name":"proxy.process.http.total_client_connections_ipv4", + "record_type":"4", + "version":"0", + "raw_stat_block":"9", + "order":"7", + "stat_meta":{ + "persist_type":"1" + }, + "record_class":"2", + "overridable":"false", + "data_type":"COUNTER", + "current_value":"0", + "default_value":"0" + } + }, + { + "record":{ + "record_name":"proxy.process.http.total_client_connections_ipv6", + "record_type":"4", + "version":"0", + "raw_stat_block":"10", + "order":"8", + "stat_meta":{ + "persist_type":"1" + }, + "record_class":"2", + "overridable":"false", + "data_type":"COUNTER", + "current_value":"0", + "default_value":"0" + } + } + ] + ,"errorList":[] + }, + "id":"ded7018e-0720-11eb-abe2-001fc69cc946" + } + + + +#. Request a configuration record using a regex with some errors coming back: + + .. code-block:: json + :linenos: + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [ + { + "rec_types": [1], + "record_name_regex": "proxy.config.exec_thread.autoconfig.sca*" + }, + { + "rec_types": [987], + "record_name_regex": "proxy.process.http.total_client_connections_ipv" + } + ] + } + + + Note the invalid ``rec_type`` at line ``11`` + + Response: + + .. code-block:: json + :linenos: + + { + "jsonrpc":"2.0", + "result":{ + "recordList":[ + { + "record":{ + "record_name":"proxy.config.exec_thread.autoconfig.scale", + "record_type":"2", + "version":"0", + "raw_stat_block":"0", + "order":"355", + "config_meta":{ + "access_type":"2", + "update_status":"0", + "update_type":"2", + "checktype":"0", + "source":"3", + "check_expr":"null" + }, + "record_class":"1", + "overridable":"false", + "data_type":"FLOAT", + "current_value":"1", + "default_value":"1" + } + } + ], + "errorList":[ + { + "code":"2008", + "message":"Invalid request data provided" + } + ] + }, + "id":"ded7018e-0720-11eb-abe2-001fc69cc946" + } + + + + We get a valid record that was found based on the passed criteria, ``proxy.config.exec_thread.autoconfig.sca*`` and the ``rec_type`` *1*. + Also we get a particular error that was caused by the invalid rec types ``987`` + + +#. Request all config records + + .. code-block:: json + :linenos: + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [{ + + "record_name_regex": ".*", + "rec_types": [1, 16] + + }] + + } + + + + *Note the `.*` regex we use to match them all. `rec_types` refer to ``RecT` , which in this case we are interested in `CONFIG` + records and `LOCAL` records.* + + + Response: + + All the configuration records. See `RecordResponse`_. The JSONRPC record handler is not limiting the response size. + + +.. note:: + + It will retrieve ALL the configuration records, keep in mind that it might be a large response. + + + +.. _admin_config_set_records: + +admin_config_set_records +------------------------ + +|method| + +Description +~~~~~~~~~~~ + +Set a value for a particular record. + + + +Parameters +~~~~~~~~~~ + +=================== ============= ================================================================================================================ +Field Type Description +=================== ============= ================================================================================================================ +``record_name`` |str| The name of the record that wants to be updated. +``new_value`` |str| The associated record value. Use always a |str| as the internal library will translate to the appropriate type. +=================== ============= ================================================================================================================ + + +Example: + + .. code-block:: json + + [ + { + "record_name": "proxy.config.exec_thread.autoconfig.scale", + "record_value": "1.5" + } + ] + + +Result +~~~~~~ + +A list of updated record names. :ref:`RecordErrorObject-Enum` will be included. + +Examples +~~~~~~~~ + + +Request: + +.. code-block:: json + :linenos: + + { + "id": "a32de1da-08be-11eb-9e1e-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_config_set_records", + "params": [ + { + "record_name": "proxy.config.exec_thread.autoconfig.scale", + "record_value": "1.3" + } + ] + } + + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc":"2.0", + "result":[ + { + "record_name":"proxy.config.exec_thread.autoconfig.scale" + } + ], + "id":"a32de1da-08be-11eb-9e1e-001fc69cc946" + } + +.. _admin_config_reload: + +admin_config_reload +------------------- + +|method| + +Description +~~~~~~~~~~~ + +Instruct |TS| to start the reloading process. You can find more information about config reload here(add link TBC) + + +Parameters +~~~~~~~~~~ + +* ``params``: Omitted + +.. note:: + + There is no need to add any parameters here. + +Result +~~~~~~ + +A |str| with the success message indicating that the command was acknowledged by the server. + +Examples +~~~~~~~~ + + +Request: + +.. code-block:: json + :linenos: + + { + "id": "89fc5aea-0740-11eb-82c0-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_config_reload" + } + + +Response: + +The response will contain the default `success_response` or an :cpp:class:`RPCErrorCode`. + + +Validation: + +You can request for the record `proxy.node.config.reconfigure_time` which will be updated with the time of the requested update. + + +.. _jsonrpc-api-management-metrics: + +Metrics +======= + +.. _admin_clear_metrics_records: + +admin_clear_metrics_records +--------------------------- + +|method| + +Description +~~~~~~~~~~~ + +Clear one or more metric values. This API will take the incoming metric names and reset their associated value. The format for the incoming +request should follow the `RecordRequest`_ . + + + +Parameters +~~~~~~~~~~ + +* ``params``: A list of `RecordRequest`_ objects. + +.. note:: + + Only the ``rec_name`` will be used, if this is not provided, the API will report it back as part of the `RecordErrorObject`_ . + + +Result +~~~~~~ + +This api will only inform for errors during the metric update, all errors will be inside the `RecordErrorObject`_ object. +Successfully metric updates will not report back to the client. So it can be assumed that the records were properly updated. + +.. note:: + + As per our internal API if the metric could not be updated because there is no change in the value, ie: it's already ``0`` this will be reported back to the client as part of the `RecordErrorObject`_ + +Examples +~~~~~~~~ + + +Request: + +.. code-block:: json + :linenos: + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_clear_metrics_records", + "params": [ + { + "record_name": "proxy.process.http.total_client_connections_ipv6" + }, + { + "record_name": "proxy.config.log.rolling_intervi_should_fail" + } + ] + } + + +Response: + +.. code-block:: json + + { + "jsonrpc": "2.0", + "result": { + "errorList": [{ + "code": "2006", + "record_name": "proxy.config.log.rolling_intervi_should_fail" + }] + }, + "id": "ded7018e-0720-11eb-abe2-001fc69cc946" + } + + +.. _admin_clear_all_metrics_records: + +admin_clear_all_metrics_records +------------------------------- + +|method| + +Description +~~~~~~~~~~~ + +Clear all the metrics. + + +Parameters +~~~~~~~~~~ + +* ``params``: This can be Omitted + + +Result +~~~~~~ + +This api will only inform for errors during the metric update. Errors will be tracked down in the :cpp:class:`RPCErrorCode` field. + +.. note:: + + As per our internal API if the metric could not be updated because there is no change in the value, ie: it's already ``0`` this + will be reported back to the client as part of the `RecordErrorObject`_ + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id": "dod7018e-0720-11eb-abe2-001fc69cc997", + "jsonrpc": "2.0", + "method": "admin_clear_all_metrics_records" + } + + + +Response: + +The response will contain the default `success_response` or an :cpp:class:`RPCErrorCode`. + + +.. _admin_host_set_status: + +admin_host_set_status +--------------------- + +Description +~~~~~~~~~~~ + +A stat to track status is created for each host. The name is the host fqdn with a prefix of `proxy.process.host_status`. The value of +the stat is a string which is the serialized representation of the status. This contains the overall status and the status for each reason. +The stats may be viewed using the `admin_lookup_records`_ rpc api or through the ``stats_over_http`` endpoint. + +Parameters +~~~~~~~~~~ + + +=================== ============= ================================================================================================= +Field Type Description +=================== ============= ================================================================================================= +``operation`` |str| The name of the record that is meant to be updated. +``host`` |arraystr| A list of hosts that we want to interact with. +``reason`` |str| Reason for the operation. +``time`` |str| Set the duration of an operation to ``count`` seconds. A value of ``0`` means no duration, the + condition persists until explicitly changed. The default is ``0`` if an operation requires a time + and none is provided by this option. optional when ``op=up`` +=================== ============= ================================================================================================= + +operation: + +=================== ============= ================================================================================================= +Field Type Description +=================== ============= ================================================================================================= +``up`` |str| Marks the listed hosts as ``up`` so that they will be available for use as a next hop parent. Use + ``reason`` to mark the host reason code. The 'self_detect' is an internal reason code + used by parent selection to mark down a parent when it is identified as itself and +``down`` |str| Marks the listed hosts as down so that they will not be chosen as a next hop parent. If + ``time`` is included the host is marked down for the specified number of seconds after + which the host will automatically be marked up. A host is not marked up until all reason codes + are cleared by marking up the host for the specified reason code. +=================== ============= ================================================================================================= + +reason: + +=================== ============= ================================================================================================= +Field Type Description +=================== ============= ================================================================================================= +``active`` |str| Set the active health check reason. +``local`` |str| Set the local health check reason. +``manual`` |str| Set the administrative reason. This is the default reason if a reason is needed and not provided + by this option. If an invalid reason is provided ``manual`` will be defaulted. +=================== ============= ================================================================================================= + +Internally the reason can be ``self_detect`` if +:ts:cv:`proxy.config.http.parent_proxy.self_detect` is set to the value 2 (the default). This is +used to prevent parent selection from creating a loop by selecting itself as the upstream by +marking this reason as "down" in that case. + +.. note:: + + The up / down status values are independent, and a host is consider available if and only if + all of the statuses are "up". + + +Result +~~~~~~ + +The response will contain the default `success_response` or an :cpp:class:`RPCErrorCode`. + + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id": "c6d56fba-0cbd-11eb-926d-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_host_set_status", + "params": { + "operation": "up", + "host": ["host1"], + "reason": "manual", + "time": "100" + } + } + + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc": "2.0", + "result": "success", + "id": "c6d56fba-0cbd-11eb-926d-001fc69cc946" + } + + + +Getting the host status +~~~~~~~~~~~~~~~~~~~~~~~ + +Get the current status of the specified hosts with respect to their use as targets for parent selection. This returns the same +information as the per host stat. + +Although there is no specialized API that you can call to get a status from a particular host you can work away by pulling the right records. +For instance, the ``host1`` that we just set up can be easily fetch for a status: + +Request: + +.. code-block:: json + :linenos: + + { + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_lookup_records", + "params": [{ + "record_name": "proxy.process.host_status.host1" + } + ] + } + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc": "2.0", + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "result": { + "recordList": [{ + "record": { + "record_name": "proxy.process.host_status.host1", + "record_type": "3", + "version": "0", + "raw_stat_block": "0", + "order": "1134", + "stat_meta": { + "persist_type": "1" + }, + "record_class": "2", + "overridable": "false", + "data_type": "STRING", + "current_value": "HOST_STATUS_UP,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:UP:0:0,SELF_DETECT:UP:0", + "default_value": "HOST_STATUS_UP,ACTIVE:UP:0:0,LOCAL:UP:0:0,MANUAL:UP:0:0,SELF_DETECT:UP:0" + } + }] + ,"errorList":[] + } + } + + +.. _admin_server_stop_drain: + +admin_server_stop_drain +----------------------- + +|method| + +Description +~~~~~~~~~~~ + +Stop the drain requests process. Recover server from the drain mode + +Parameters +~~~~~~~~~~ + +* ``params``: Omitted + +Result +~~~~~~ + +The response will contain the default `success_response` or an :cpp:class:`RPCErrorCode`. + + +Examples +~~~~~~~~ + +.. code-block:: json + :linenos: + + { + "id": "35f0b246-0cc4-11eb-9a79-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_server_stop_drain" + } + + + +.. _admin_server_start_drain: + +admin_server_start_drain +------------------------ + +|method| + +Description +~~~~~~~~~~~ + +Drain TS requests. + +Parameters +~~~~~~~~~~ + +======================= ============= ================================================================================================================ +Field Type Description +======================= ============= ================================================================================================================ +``no_new_connections`` |str| Wait for new connections down to threshold before starting draining, ``yes|true|1``. Not yet supported +======================= ============= ================================================================================================================ + + +Result +~~~~~~ + +The response will contain the default `success_response` or an :cpp:class:`RPCErrorCode`. + +.. note:: + + If the Server is already running a proper error will be sent back to the client. + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id": "30700808-0cc4-11eb-b811-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_server_start_drain", + "params": { + "no_new_connections": "yes" + } + } + + +Response could be either: + +#. The response will contain the default `success_response` + +#. Response from a server that is already in drain mode. + +.. code-block:: json + :linenos: + + { + "jsonrpc": "2.0", + "id": "30700808-0cc4-11eb-b811-001fc69cc946", + "error": { + + "code": 9, + "message": "Error during execution", + "data": [{ + + "code": 3000, + "message": "Server already draining." + }] + + } + + } + + +.. _admin_plugin_send_basic_msg: + +admin_plugin_send_basic_msg +--------------------------- + +|method| + +Description +~~~~~~~~~~~ + +Interact with plugins. Send a message to plugins. All plugins that have hooked the ``TSLifecycleHookID::TS_LIFECYCLE_MSG_HOOK`` will receive a callback for that hook. +The :arg:`tag` and :arg:`data` will be available to the plugin hook processing. It is expected that plugins will use :arg:`tag` to select relevant messages and determine the format of the :arg:`data`. + +Parameters +~~~~~~~~~~ + +======================= ============= ================================================================================================================ +Field Type Description +======================= ============= ================================================================================================================ +``tag`` |str| A tag name that will be read by the interested plugin +``data`` |str| Data to be send, this is |optional| +======================= ============= ================================================================================================================ + + +Result +~~~~~~ + +The response will contain the default `success_response` or an :cpp:class:`RPCErrorCode`. + +Examples +~~~~~~~~ + + .. code-block:: json + :linenos: + + { + "id": "19095bf2-0d3b-11eb-b41a-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_plugin_send_basic_msg", + "params": { + "data": "ping", + "tag": "pong" + } + } + + + + +.. _admin_storage_get_device_status: + +admin_storage_get_device_status +------------------------------- + +|method| + +Description +~~~~~~~~~~~ + +Show the storage configuration. + +Parameters +~~~~~~~~~~ + +A list of |str| names for the specific storage we want to interact with. The storage identification used in the param list should match +exactly a path specified in :file:`storage.config`. + +Result +~~~~~~ + +cachedisk + +======================= ============= ============================================================================================= +Field Type Description +======================= ============= ============================================================================================= +``path`` |str| Storage identification. The storage is identified by :arg:`path` which must match exactly a + path specified in :file:`storage.config`. +``status`` |str| Disk status. ``online`` or ``offline`` +``error_count`` |str| Number of errors on the particular disk. +======================= ============= ============================================================================================= + + +Examples +~~~~~~~~ + +Request: + + +.. code-block:: json + :linenos: + + { + "id": "8574edba-0d40-11eb-b2fb-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_storage_get_device_status", + "params": ["/some/path/to/ats/trafficserver/cache.db", "/some/path/to/ats/var/to_remove/cache.db"] + } + + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc": "2.0", + "result": [{ + "cachedisk": { + "path": "/some/path/to/ats/trafficserver/cache.db", + "status": "online", + "error_count": "0" + } + }, + { + "cachedisk": { + "path": "/some/path/to/ats/var/to_remove/cache.db", + "status": "online", + "error_count": "0" + } + } + ], + "id": "8574edba-0d40-11eb-b2fb-001fc69cc946" + } + + + +.. _admin_storage_set_device_offline: + +admin_storage_set_device_offline +-------------------------------- + +|method| + +Description +~~~~~~~~~~~ + +Mark a cache storage device as ``offline``. The storage is identified by :arg:`path` which must match exactly a path specified in +:file:`storage.config`. This removes the storage from the cache and redirects requests that would have used this storage to +other storage. This has exactly the same effect as a disk failure for that storage. This does not persist across restarts of the +:program:`traffic_server` process. + +Parameters +~~~~~~~~~~ + +A list of |str| names for the specific storage we want to interact with. The storage identification used in the param list should match +exactly a path specified in :file:`storage.config`. + +Result +~~~~~~ + +A list of |object| which the following fields: + + +=========================== ============= ============================================================================================= +Field Type Description +=========================== ============= ============================================================================================= +``path`` |str| Storage identification. The storage is identified by :arg:`path` which must match exactly a + path specified in :file:`storage.config`. +``has_online_storage_left`` |str| A flag indicating if there is any online storage left after this operation. +=========================== ============= ============================================================================================= + + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id": "53dd8002-0d43-11eb-be00-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_storage_set_device_offline", + "params": ["/some/path/to/ats/var/to_remove/cache.db"] + } + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc": "2.0", + "result": [{ + "path": "/some/path/to/ats/var/to_remove/cache.db", + "has_online_storage_left": "true" + }], + "id": "53dd8002-0d43-11eb-be00-001fc69cc946" + } + + +.. _show_registered_handlers: + +show_registered_handlers +------------------------ + +|method| + +Description +~~~~~~~~~~~ + +List all the registered RPC public handlers. + +Parameters +~~~~~~~~~~ + +* ``params``: Omitted + +Result +~~~~~~ + +An |object| with the following fields: + + +================== ============= =========================================== +Field Type Description +================== ============= =========================================== +``methods`` |str| A list of exposed method handler names. +``notifications`` |str| A list of exposed notification handler names. +================== ============= =========================================== + + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id": "f4477ac4-0d44-11eb-958d-001fc69cc946", + "jsonrpc": "2.0", + "method": "show_registered_handlers" + } + + +Response: + +.. code-block:: json + :linenos: + + { + "id": "f4477ac4-0d44-11eb-958d-001fc69cc946", + "jsonrpc": "2.0", + "result": { + "methods": [ + "admin_host_set_status", + "admin_server_stop_drain", + "admin_server_start_drain", + "admin_clear_metrics_records", + "admin_clear_all_metrics_records", + "admin_plugin_send_basic_msg", + "admin_lookup_records", + "admin_config_set_records", + "admin_storage_get_device_status", + "admin_storage_set_device_offline", + "admin_config_reload", + "show_registered_handlers" + ], + "notifications": [] + } + } + +.. _get_service_descriptor: + +get_service_descriptor +------------------------ + +|method| + +Description +~~~~~~~~~~~ + +List and describe all the registered RPC handler. + +Parameters +~~~~~~~~~~ + +* ``params``: Omitted + +Result +~~~~~~ + +An |object| with the following fields: + + +``methods`` object + +=============== ============= =========================================== +Field Type Description +=============== ============= =========================================== +``name`` |str| Handler's name. Call name +``type`` |str| Either 'method' or 'notification' +``provider`` |str| Provider's information. +``schema`` |str| A json-schema definition +=============== ============= =========================================== + + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id": "f4477ac4-0d44-11eb-958d-001fc69cc946", + "jsonrpc": "2.0", + "method": "get_service_descriptor" + } + + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc":"2.0", + "result":{ + "methods":[ + { + "name":"admin_host_set_status", + "type":"method", + "provider":"Traffic Server JSONRPC 2.0 API", + "schema":{ + } + }, + { + "name":"some_plugin_call", + "type":"notification", + "provider":"ABC Plugin's details.", + "schema":{ + } + }] + } + } + + + +.. _filemanager.get_files_registry: + +filemanager.get_files_registry +------------------------------ + +|method| + +Description +~~~~~~~~~~~ + +Fetch the registered config files within ATS. All configured files in the system will be retrieved by this API. This basically drops +the `FileManager` binding files. File Manager keeps track of all the configured files in |TS|. + +Parameters +~~~~~~~~~~ + +* ``params``: Omitted + +Result +~~~~~~ + +An list of |object| with the following fields: + + +``config_registry`` object: + +======================= ============= =========================================== +Field Type Description +======================= ============= =========================================== +``file_path`` |str| File path, includes the full path and the file name configured in the record config name(if it's the case) +``config_record_name`` |str| Internal record config variable name. +``parent_config`` |str| Parent's configuration file name. e.g. If a top level remap.config includes additional mapping files, + then the top level file will be set in this field. +``root_access_needed`` |str| Elevated access needed. +``is_required`` |str| If it's required by |TS|. This specifies if |TS| treat this file as required to start the system(e.g. storage.config) +======================= ============= =========================================== + + +Examples +~~~~~~~~ + +Request: + +.. code-block:: json + :linenos: + + { + "id":"14c10697-5b09-40f6-b7e5-4be85f64aa5e", + "jsonrpc":"2.0", + "method":"filemanager.get_files_registry" + } + + +Response: + +.. code-block:: json + :linenos: + + { + "jsonrpc":"2.0", + "result":{ + "config_registry":[ + { + "file_path":"/home/xyz/ats/etc/trafficserver/sni.yaml", + "config_record_name":"proxy.config.ssl.servername.filename", + "parent_config":"N/A", + "root_access_needed":"false", + "is_required":"false" + }, + { + "file_path":"/home/xyz/ats/etc/trafficserver/storage.config", + "config_record_name":"", + "parent_config":"N/A", + "root_access_needed":"false", + "is_required":"true" + }, + { + "file_path":"/home/xyz/ats/etc/trafficserver/jsonrpc3.yaml", + "config_record_name":"proxy.config.jsonrpc.filename", + "parent_config":"N/A", + "root_access_needed":"false", + "is_required":"false" + } + ] + } + } diff --git a/doc/developer-guide/jsonrpc/jsonrpc-architecture.en.rst b/doc/developer-guide/jsonrpc/jsonrpc-architecture.en.rst new file mode 100644 index 00000000000..511e4e99b75 --- /dev/null +++ b/doc/developer-guide/jsonrpc/jsonrpc-architecture.en.rst @@ -0,0 +1,540 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. highlight:: cpp +.. default-domain:: cpp + +.. _JSONRPC: https://www.jsonrpc.org/specification +.. _JSON: https://www.json.org/json-en.html + + +.. |str| replace:: ``string`` +.. |arraynum| replace:: ``array[number]`` +.. |arraystr| replace:: ``array[string]`` +.. |num| replace:: *number* +.. |strnum| replace:: *string|number* +.. |object| replace:: *object* +.. |array| replace:: *array* +.. |optional| replace:: ``optional`` +.. |arrayrecord| replace:: ``array[record]`` +.. |arrayerror| replace:: ``array[errors]`` +.. |RPC| replace:: JSONRPC 2.0 + +Architecture +************ + + +Protocol +======== + +The RPC mechanism implements the `JSONRPC`_ protocol. You can refer to this section `jsonrpc-protocol`_ for more information. + +Server +====== + +.. _jsonrpc-architecture-ipc: + +IPC +--- + +The current server implementation runs on an IPC Socket(Unix Domain Socket). This server implements an iterative server style. +The implementation runs on a dedicated ``TSThread`` and as their style express, this performs blocking calls to all the registered handlers. +Configuration for this particular server style can be found in the admin section :ref:`admnin-jsonrpc-configuration`. + + +Using the JSONRPC mechanism +=========================== + +As a user, currently, :program:`traffic_ctl` exercises this new protocol, please refer to the :ref:`traffic_ctl_jsonrpc` section. + +As a developer, please refer to the :ref:`jsonrpc_development` for a more detailed guide. + + + +JSON Parsing +============ + +Our JSONRPC protocol implementation uses lib yamlcpp for parsing incoming and outgoing requests, +this allows the server to accept either JSON or YAML format messages which then will be parsed by the protocol implementation. This seems handy +for user that want to feed |TS| with existing yaml configuration without the need to translate yaml into json. + +.. note:: + + :program:`traffic_ctl` have an option to read files from disc and push them into |TS| through the RPC server. Files should be a + valid `JSONRPC`_ message. Please check `traffic_ctl rpc` for more details. + + +In order to programs communicate with |TS| , This one implements a simple RPC mechanism to expose all the registered API handlers. + +You can check all current API by: + + .. code-block:: bash + + traffic_ctl rpc get-api + +or by using the ``show_registered_handlers`` API method. + + +.. _jsonrpc-protocol: + +JSONRPC 2.0 Protocol +==================== + +JSON-RPC is a stateless, light-weight remote procedure call (RPC) protocol. Primarily this specification defines several data structures +and the rules around their processing. It is transport agnostic in that the concepts can be used within the same process, over sockets, +over http, or in many various message passing environments. It uses JSON (RFC 4627) as data format. + +Overview +======== + +.. note:: + + Although most of the protocol specs are granted, we have implemented some exceptions. All the modifications will be properly documented. + + +There are a set of mandatory fields that must be included in a `JSONRPC`_ message as well as some optional fields, all this is documented here, +you also can find this information in the `JSONRPC`_ link. + +.. _jsonrpc-request: + +Requests +-------- + +Please find the `jsonrpc 2.0 request` schema for reference ( `mgmt2/rpc/schema/jsonrpc_request_schema.json` ). + +* Mandatory fields. + + + ============ ====== ======================================================================================= + Field Type Description + ============ ====== ======================================================================================= + ``jsonrpc`` |str| Protocol version. |TS| follows the version 2.0 so this field should be only ``2.0`` + ``method`` |str| Method name that is intended to be invoked. + ============ ====== ======================================================================================= + + +* Optional parameters: + + + + * ``params``: + + A Structured value that holds the parameter values to be used during the invocation of the method. This member + may be omitted. If passed then a parameters for the rpc call must be provided as a Structured value. + Either by-position through an Array or by-name through an Object. + + #. ``by-position`` |array| + + params must be an ``array``, containing the values in the server expected order. + + + .. code-block:: json + + { + "params": [ + "apache", "traffic", "server" + ] + } + + + .. code-block:: json + + { + "params": [ + 1, 2, 3, 4 + ] + } + + + .. code-block:: json + + { + "params": [{ + "name": "Apache" + },{ + "name": "Traffic" + },{ + "name": "Server" + }] + } + + #. ``by-name``: |object| + + Params must be an ``object``, with member names that match the server expected parameter names. + The absence of expected names may result in an error being generated by the server. The names must + match exactly, including case, to the method's expected parameters. + + .. code-block:: json + + { + "params": { + "name": "Apache" + } + } + + * ``id``: |str|. + + An identifier established by the Client. If present, the request will be treated as a jsonrpc method and a + response should be expected from the server. If it is not present, the server will treat the request as a + notification and the client should not expect any response back from the server. + *Although a |number| can be specified here we will convert this internally to a |str|. The response will be a |str|.* + + +.. note:: + + The |RPC| protocol supports batch requests so, multiple independent requests can be send to the server in a single json message. Batch + request are basically an array of the above request, simply enclose them as an array ``[ .. ]``. The response for batch requests will be also + in a form of an array. + +.. _jsonrpc-response: + +Responses +--------- + +Although each individual API call will describe the response details and some specific errors, in this section we will describe a high +level protocol response, some defined by the `JSONRPC`_ specs and some by |TS| + +Please find the `jsonrpc 2.0 response` schema for reference( `mgmt2/rpc/schema/jsonrpc_response_schema.json` ). + + + +The responses have the following structure: + + + ============ ======== ============================================== + Field Type Description + ============ ======== ============================================== + ``jsonrpc`` |strnum| A Number that indicates the error type that occurred. + ``result`` Result of the invoked operation. See `jsonrpc-result`_ + ``id`` |strnum| It will be the same as the value of the id member in the `jsonrpc-request`_ . + We will not be using `null` if the `id` could not be fetch from the request, + in that case the field will not be set. + ``error`` |object| Error object, it will be present in case of an error. See `jsonrpc-error`_ + ============ ======== ============================================== + +Example 1: + +Request + + .. code-block:: json + + { + "jsonrpc": "2.0", + "result": ["hello", 5], + "id": "9" + } + + +Response + + .. code-block:: json + + { + "jsonrpc":"2.0", + "error":{ + "code":5, + "message":"Missing method field" + }, + "id":"9" + } + + +As the protocol specifies |TS| have their own set of error, in the example above it's clear that the incoming request is missing +the method name, which |TS| sends a clear response error back. + +.. _jsonrpc-result: + +Result +------ + + +* This member is required and will be present on success. +* This member will not exist if there was an error invoking the method. +* The value of this member is determined by the method invoked on the Server. + +In |TS| a RPC method that does not report any error and have nothing to send back to the client will use the following format to +express that the call was successfully handled and the command was executed. + + +.. _success_response: + + +Example: + + .. code-block:: json + :emphasize-lines: 4 + + { + "id": "89fc5aea-0740-11eb-82c0-001fc69cc946", + "jsonrpc": "2.0", + "result": "success" + } + +``"result": "success"`` will be set. + +.. _jsonrpc-error: + +Errors +------ + +The specs define the error fields that the client must expect to be sent back from the Server in case of any error. + + +=============== ======== ============================================== +Field Type Description +=============== ======== ============================================== +``code`` |num| A Number that indicates the error type that occurred. +``message`` |str| A String providing a short description of the error. +``data`` |object| This is an optional field that contains additional error data. Depending on the API this could contain data. +=============== ======== ============================================== + +# data. + +This can be used for nested error so |TS| can inform a detailed error. + + =============== ======== ============================================== + Field Type Description + =============== ======== ============================================== + ``code`` |str| The error code. Integer type. + ``message`` |str| The explanatory string for this error. + =============== ======== ============================================== + + + + +Examples: + +# Fetch a config record response from |TS| + +Request: + + .. code-block:: json + + { + "id":"0f0780a5-0758-4f51-a177-752facc7c0eb", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"proxy.config.diags.debug.tags", + "rec_types":[ + "1", + "16" + ] + } + ] + } + +Response: + + .. code-block:: json + + { + "jsonrpc":"2.0", + "result":{ + "recordList":[ + { + "record":{ + "record_name":"proxy.config.diags.debug.tags", + "record_type":"3", + "version":"0", + "raw_stat_block":"0", + "order":"423", + "config_meta":{ + "access_type":"0", + "update_status":"0", + "update_type":"1", + "checktype":"0", + "source":"3", + "check_expr":"null" + }, + "record_class":"1", + "overridable":"false", + "data_type":"STRING", + "current_value":"rpc", + "default_value":"http|dns" + } + } + ] + }, + "id":"0f0780a5-0758-4f51-a177-752facc7c0eb" + } + + +# Getting errors from |TS| + +Request an invalid record (invalid name) + + .. code-block:: json + + { + "id":"f212932f-b260-4f01-9648-8332200524cc", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"invalid.record", + "rec_types":[ + "1", + "16" + ] + } + ] + } + + +Response: + + .. code-block:: json + + { + "jsonrpc":"2.0", + "result":{ + "errorList":[ + { + "code":"2000", + "record_name":"invalid.record" + } + ] + }, + "id":"f212932f-b260-4f01-9648-8332200524cc" + } + + +Parse Error from an incomplete request + + .. code-block:: + + {[[ invalid json + + + .. code-block:: json + + { + "jsonrpc":"2.0", + "error":{ + "code":-32700, + "message":"Parse error" + } + } + + + +Invalid method invocation. + +Request: + + .. code-block:: json + + { + "id":"f212932f-b260-4f01-9648-8332200524cc", + "jsonrpc":"2.0", + "method":"some_non_existing_method", + "params":{ + + } + } + +Response: + + .. code-block::json + + { + "error": { + "code": -32601, + "message": "Method not found" + }, + "id": "ded7018e-0720-11eb-abe2-001fc69cc946", + "jsonrpc": "2.0" + } + + +.. _rpc-error-code: + +Internally we have defined an ``enum`` class that keeps track of the errors that the server will inform in most of the cases. +Some of this errors are already defined by the `JSONRPC`_ specs and some (``>=1``) are defined by |TS|. + +.. class:: RPCErrorCode + + Defines the API error codes that will be used in case of any RPC error. + + .. enumerator:: INVALID_REQUEST = -32600 + .. enumerator:: METHOD_NOT_FOUND = -32601 + .. enumerator:: INVALID_PARAMS = -32602 + .. enumerator:: INTERNAL_ERROR = -32603 + .. enumerator:: PARSE_ERROR = -32700 + + `JSONRPC`_ defined errors. + + .. enumerator:: InvalidVersion = 1 + + The passed version is invalid. must be 2.0 + + .. enumerator:: InvalidVersionType = 2 + + The passed version field type is invalid. must be a ``string`` + + .. enumerator:: MissingVersion = 3 + + Version field is missing from the request. This field is mandatory. + + .. enumerator:: InvalidMethodType = 4 + + The passed method field type is invalid. must be a ``string`` + + .. enumerator:: MissingMethod = 5 + + Method field is missing from the request. This field is mandatory. + + .. enumerator:: InvalidParamType = 6 + + The passed parameter field type is not valid. + + .. enumerator:: InvalidIdType = 7 + + The passed id field type is invalid. + + .. enumerator:: NullId = 8 + + The passed if is ``null`` + + .. enumerator:: ExecutionError = 9 + + An error occurred during the execution of the RPC call. This error is used as a generic High level error. The details details about + the error, in most cases are specified in the ``data`` field. + + +.. information: + + According to the |RPC| specs, if you get an error, the ``result`` field will not be set. |TS| will grant this. + + + +Development Guide +================= + +* For details on how to implement JSONRPC handler and expose them through the rpc server, please refer to :ref:`jsonrpc_development`. +* If you need to call a new JSONRPC API through :program:`traffic_ctl`, please refer to :ref:`developer-guide-traffic_ctl-development` +* To interact directly with the |RPC| node, please check :ref:`jsonrpc-node` + +See also +======== + +:ref:`admnin-jsonrpc-configuration`, +:ref:`traffic_ctl_jsonrpc` diff --git a/doc/developer-guide/jsonrpc/jsonrpc-client-api.en.rst b/doc/developer-guide/jsonrpc/jsonrpc-client-api.en.rst new file mode 100644 index 00000000000..80f08c789a5 --- /dev/null +++ b/doc/developer-guide/jsonrpc/jsonrpc-client-api.en.rst @@ -0,0 +1,56 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. |RPC| replace:: JSONRPC 2.0 + +.. _YAML: https://github.com/jbeder/yaml-cpp/wiki/Tutorial + +.. _developer-guide-jsonrpc-client-api: + +Traffic Server JSONRPC Node C++ Client Implementation +***************************************************** + +Basics +====== + +|TS| provides a set of basic C++ classes to perform request and handle responses from +server's rpc node. +Files under `include/shared/rpc` are meant to be used by client applications like :program:`traffic_ctl` +and :program:`traffic_top` to send a receive messages from the |TS| jsonrpc node. + +This helper classes provides: + + * ``RPCClient`` class which provides functionality to connect and invoke remote command inside the |TS| |RPC| node. + This class already knows where the unix socket is located. + + * ``IPCSocketClient`` class which provides the socket implementation for the IPC socket in the |RPC| node. If what + you want is just to invoke a remote function and get the response, then it's recommended to just use the ``RPCClient`` + class instead. + + * ``RPCRequests`` class which contains all the basic classes to map the basic |RPC| messages(requests and responses). + If what you want is a custom message you can just subclass ``shared::rpc::ClientRequest`` and override the ``get_method()`` + member function. You can check ``CtrlRPCRequests.h`` for examples. + The basic encoding and decoding for these structures are already implemented inside ``yaml_codecs.h``. In case you define + your own message then you should provide you own codec implementation, there are some examples available in ``ctrl_yaml_codecs.h`` + +Building +======== + +.. + _ TBC. diff --git a/doc/developer-guide/jsonrpc/jsonrpc-handler-development.en.rst b/doc/developer-guide/jsonrpc/jsonrpc-handler-development.en.rst new file mode 100644 index 00000000000..cc53413a9d9 --- /dev/null +++ b/doc/developer-guide/jsonrpc/jsonrpc-handler-development.en.rst @@ -0,0 +1,427 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license + agreements. See the NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software distributed under the License + is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + or implied. See the License for the specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. |RPC| replace:: JSONRPC 2.0 + +.. _JSONRPC: https://www.jsonrpc.org/specification +.. _JSON: https://www.json.org/json-en.html +.. _YAML: https://github.com/jbeder/yaml-cpp/wiki/Tutorial + +.. _jsonrpc_development: + +Handler implementation +********************** + +Use this section as a guide for developing new rpc methods inside |TS| and how to expose them through the |RPC| endpoint. +Before we start, it is worth mentioning some of the architecture of the current implementation. The whole RPC mechanism is divided in +few components. + +Json RPC manager +================ + +This class is the entrance point for both server calls and registered functions. + +.. figure:: ../../uml/images/JsonRPCManager.svg + +Dispatcher class +---------------- + +* Class that keeps track of all the registered methods and notifications that can be invoked by the RPC server. This class holds two + hash tables containing methods and notifications which use the method name as a key. +* This class internally consumes ``RPCRequestInfo`` objects and performs the invocation of the respective calls. +* This class handlers the responses from the registered callbacks and it fills the appropriated ``RPCResponseInfo`` which then is passed + back to the ``JsonRPCManager`` class. + + +JsonRPCManager class +-------------------- + +* Singleton class that handles the JSONRPC handler registration and JSONRPC handler invocation. +* This class is the main entrance point for the RPC server through the ``handle_call(std::string_view req)`` function. +* This class is the main entrance point for the handler to be able to register in the RPC logic. ``add_notification_handler`` and ``remove_notification_handler``. + + +Implementing new handlers +========================= + +There a a few basic concepts that needs to be known before implementing a new handler, this is an easy process and the complexity depends on +the nature of the handler that you want to implement. +Dealing with incoming and outgoing parameters is up to the developer, we will touch on some ways to deal with this through this guide. + +.. _jsonrpc_development-design: + +Design +------ + +As requirement from the ``JsonRPCManager`` in order to be able to register inside the RPC management a function should implement the +following signature: + +Methods: + +.. code-block:: cpp + + ts::Rv your_rpc_handler_function_name(std::string_view const &id, YAML::Node const ¶ms); + + + +Notifications: + +.. code-block:: cpp + + void your_rpc_handler_function_name(YAML::Node const ¶ms); + + +* Incoming method request's id will be passed to the handler, this is read only value as the server is expected to respond with the same value. +* ``YAML::Node`` params is expected to be a ``Sequence`` or a ``Map``, as per protocol this cannot be a single value, so do not expect things like: + ``param=123`` or ``param=SomeString``. +* The ``params`` can be empty and contains no data at all. + + +It is important to know that ``method`` handlers are expected to respond to the requests, while ``notifications``` will not respond with +any data nor error. You can find more information in :ref:`jsonrpc-protocol` or directly in the protocol specs `JSONRPC`_. + + +.. note:: + + If there is no explicit response from the method, the protocol implementation will respond with `success_response` unless an error + was specified. + + +Registration and Handling +------------------------- + +JSONRPC Manager API +~~~~~~~~~~~~~~~~~~~ + +Handler registration should be done by using the ``JsonRPCManager`` singleton object. Note that there are a set of convenient helper +functions that can be used to achieve registration through the singleton object. + +.. code-block:: cpp + + namespace rpc { + // this set of functions will call the singleton object and perform the same as by using the singleton directly. + add_method_handler(...) + add_notification_handler(...) + } + + +.. code-block:: cpp + + // Handler implementation + ts::Rv + my_handler_impl(std::string_view const &id, YAML::Node const ¶ms) + { + using namespace rpc::handlers::errors; + return make_errata(Codes::SERVER, "Something happened in the server"); + } + +The actual registration: + +.. code-block:: cpp + + #include "rpc/jsonrpc/JsonRPC.h" + ... + rpc::add_method_handler("my_handler_impl", &my_handler_impl); + + +This API also accepts a RPCRegistryInfo pointer which will provide a context data for the particular handler, for instance it will +display the provider's name when the service descriptor gets called. There is a global object created for this purpose which can be used +As a generic registry context object, ``core_ats_rpc_service_provider_handle`` is defined in the ``JsonRPC.h`` header. Please check +`get_service_descriptor` for more information. + + +Notification example: + +As mentioned before, notifications do not need to respond, as they are "fire and forget" calls, no id will be provided as part of the api. + +.. code-block:: cpp + + void + my_notification_handler(YAML::Node const ¶ms) + { + // do something + // all errors will be ignored by the server. + } + +Registration for notifications uses a different API: + +.. code-block:: cpp + + #include "rpc/jsonrpc/JsonRPC.h" + rpc::add_notification_handler("my_notification_handler", &my_notification_handler); + + + +The registration API allows the client to pass a ``RPCRegistryInfo`` which provides extra information for a particular handler. Non plugins handlers +should use the default provided Registry Info, located in the `JsonRPC.h` header file. + + +Error Handling +-------------- + + +JSONRPC Manager API +~~~~~~~~~~~~~~~~~~~ + +There are several ways to deal with internal handler errors. Errors are expected to be sent back to the client if the API was expressed that way +and if the request was a ``method``. +We have defined some generic errors that can be used to respond depending on the nature of the registered handler, +please check :ref:`jsonrpc-handler-errors` for more info. + +We recommend some ways to deal with this: + +#. Using the ``Errata`` from ``ts::Rv`` + +This can be set in case you would like to let the server to respond with an |RPC| error, ``ExecutionError`` will be used to catch all the +errors that are fired from within the function call, either by setting the proper errata or by throwing an exception. +Please check the `rpc-error-code` and in particular ``ExecutionError = 9``. Also check :ref:`jsonrpc-handler-errors` + +.. important:: + + Errors have preference over any other response, so if you set both, the errata and the ``YAML::Node`` response, then the former + will be set in the response. + +#. Defining a custom error object and make it part of the response object. + +* This is up to the developer and the errors can be part of the response ``YAML::Node``. +* The JSONRPC Dispatcher will read that there is no error returned from the call and use the result to build the response. If this is + what you are willing to respond to, then make sure that the error is not set in the ``ts::Rv``. + + +#. Exception. + + As long as the exception inherit from ``std::exception`` it will be handled by the jsonrpc manager, this error will be + handled as like using the ``Errata`` object, this kind of errors will be part of the ``ExecutionError``. + + The following example will generate this JSON response: + + .. code-block:: cpp + + ts::Rv + foo(std::string_view const &id, YAML::Node const ¶ms) + { + some_unhandled_operation_that_throws(); + } + + + .. code-block::json + + { + "jsonrpc":"2.0", + "error":{ + "code":9, + "message":"Error during execution" + }, + "id":"abcd-id" + } + + +Examples: + +* Respond with an error, no ``result`` field will be set in the response. + + .. code-block:: + + ts::Rv + respond_with_an_error(std::string_view const &id, YAML::Node const ¶ms) + { + using namespace rpc::handlers::errors; + return make_errata(Codes::SERVER, "Something happened in the server"); + } + + Server's response. + + .. code-block:: json + + { + "jsonrpc":"2.0", + "error":{ + "code":9, + "message":"Error during execution", + "data":[ + { + "code":3000, + "message":"Something happened in the server" + } + ] + }, + "id":"abcd-id" + } + + .. note:: + + ``make_errata`` hides some internal details when creating an errata. + +* Response with custom handler error. In this case, make sure that the API definition and documentation reflects this as so far we do not + have json schemas to enforce any of this on the client side. + + + .. code-block:: + + ts::Rv + respond_with_my_own_error(std::string_view const &id, YAML::Node const ¶ms) + { + YAML::Node resp; + resp["HandlerErrorDescription"] = "I can set up my own error in the result field."; + return resp; + } + + The "error" is part of the ``result``, in this case this could be used as any other field, the example would be the same. + + .. code-block:: json + + { + "jsonrpc":"2.0", + "result":{ + "HandlerErrorDescription":"I can set up my own error in the result field." + }, + "id":"abcd-id" + } + + +We have selected the ``ts::Rv`` as a message interface as this can hold the actual response/error. + + + +.. _jsonrpc-handler-unit-test: + +Unit test +========= + +All new methods exposed through the RPC server can be tested using the jsonrpc autest extension. + +jsonrpc_client.test.text +------------------------ + +This extension provides the ability to interact with the JSONRPC interface by using :program:`traffic_ctl` as a client. As a helper +for all new autest that needs to write and read jsonrpc message, there is also a new module `jsonrpc.py` which provides +a nice and easy interface to write methods and notifications. +This extension also provides the facility to write custom jsonrpc validations. Please check some of the following examples: + + +#. Write custom jsonrpc message + + .. code-block:: python + + ''' + The newly added jsonrpc method was named 'foo_bar' and is expected to accept a list of fqdn. + ''' + tr = Test.AddTestRun("Test JSONRPC foo_bar()") + ''' + The following call to the Request object will generate this: + { + "id": "850d32a8-d5a7-11eb-bebc-fa163e6d2ec5", + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "fqdn": ["yahoo.com", "trafficserver.org"] + } + } + ''' + req = Request.foo_bar(fqdn=["yahoo.com", "trafficserver.org"]) + tr.AddJsonRPCClientRequest(ts, req) + + +#. Custom response validation + + .. code-block:: python + + tr = Test.AddTestRun("Test update_host_status") + + Params = [ + {'name': 'yahoo', 'status': 'up'} + ] + + tr.AddJsonRPCClientRequest(ts, Request.update_host_status(hosts=Params)) + + def check_no_error_on_response(resp: Response): + # we only check if it's an error. + if resp.is_error(): + return (False, resp.error_as_str()) + return (True, "All good") + + tr.Processes.Default.Streams.stdout = Testers.CustomJSONRPCResponse(check_no_error_on_response) + + + +CustomJSONRPCResponse +~~~~~~~~~~~~~~~~~~~~~~~ + +A tester class that will let you write your own response validation by dealing with the jsonrpc.Response object, +please check the ``CustomJSONRPCResponse`` tester for more information. + + +AddJsonRPCClientRequest +~~~~~~~~~~~~~~~~~~~~~~~ + +This function will generate a json response as an output, internally it ses :program:`traffic_ctl file --format json` as client. +The output can be used and compared with a gold file. This also provides schema validation for the entire JSONRPC protocol as well as +the ``param`` field against a specific schema file. You can specify ``schema_file_name`` with a valid json schema file to validate +the entire JSONRPC 2.0 request(except the content of the ``params`` field). You can also set ``params_field_schema_file_name`` with a +valid json schema file to validate only the ``params`` field. + +Example: + + The following, beside sending the request, will perform a JSONRPC 2.0 schema validation as well as the ``param`` field. + + .. code-block:: python + + schema_file_name = 'schemas/jsonrpc20_request_schema.json' + params_schema_file_name = 'schemas/join_strings_request_params_schema.json' + tr.AddJsonRPCClientRequest( + ts, + file="join_strings_request.json", + schema_file_name=schema_file_name, + params_field_schema_file_name=params_schema_file_name) + + +JSONRPCResponseSchemaValidator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A tester class to perform a schema validation of the entire JSONRPC 2.0 request as well as the ``result`` param. +You can specify ``schema_file_name`` with a valid json schema file to validate the entire JSONRPC 2.0 response(except the content of the ``result`` field). +Also you can set ``result_field_schema_file_name`` with a valid json schema file to validate only the ``result`` field. + + +Example: + + The following will add a Tester for a JSONRPC 2.0 schema validation as well as the ``result`` field. + + .. code-block:: python + + tr = Test.AddTestRun('test valid response schema') + schema_file_name = 'schemas/jsonrpc20_response_schema.json' + result_schema_file_name = 'schemas/join_strings_request_result_schema.json' + + # Add a tester. + tr.Processes.Default.Streams.stdout = Testers.JSONRPCResponseSchemaValidator( + schema_file_name=response_schema_file_name, + result_field_schema_file_name=result_schema_file_name) + +Important Notes +=============== + +* You can refer to `YAML`_ for more info in how code/decode yaml content. +* Remember to update :ref:`jsonrpc-api` if you are adding a new handler. +* If a new handler needs to be exposed through :program:`traffic_ctl` please refer to :ref:`traffic_ctl_jsonrpc` for a general idea + and to :ref:`developer-guide-traffic_ctl-development` for how to implement a new command. +* To interact directly with the |RPC| node, please check :ref:`jsonrpc-node` + + +:ref:`admnin-jsonrpc-configuration` +:ref:`jsonrpc-protocol` +:ref:`developer-guide-traffic_ctl-development` + + diff --git a/doc/developer-guide/jsonrpc/jsonrpc-node.en.rst b/doc/developer-guide/jsonrpc/jsonrpc-node.en.rst new file mode 100644 index 00000000000..204e7b35717 --- /dev/null +++ b/doc/developer-guide/jsonrpc/jsonrpc-node.en.rst @@ -0,0 +1,59 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. _JSON: https://www.json.org/json-en.html +.. _JSONRPC: https://www.jsonrpc.org/specification +.. |RPC| replace:: JSONRPC 2.0 + + +.. _jsonrpc-node: + +JSONRPC Endpoint +**************** + +Traffic Server API clients can use different languages to connect and interact with the |RPC| node directly. +The goal of this section is to provide some tips on how to work with it. +To begin with, you should be familiar with the |RPC| protocol, you can check here :ref:`jsonrpc-protocol` and also `JSONRPC`_ . + + +IPC Node +======== + +You can directly connect to the Unix Domain Socket used for the |RPC| node, the location of the sockets +will depend purely on how did you configure the server, please check :ref:`admnin-jsonrpc-configuration` for +information regarding configuration. + + +Socket connectivity +------------------- + +|RPC| server will close the connection once the server processed the incoming requests, so clients should be aware +of this and if sending multiple requests they should reconnect to the node once the response arrives. The protocol +allows you to send a bunch of requests together, this is called batch messages so it's recommended to send them all instead +of having a connection open and sending requests one by one. This being a local socket opening and closing the connection should +not be a big concern. + + + +Using traffic_ctl +----------------- + +:program:`traffic_ctl` can also be used to directly send raw |RPC| messages to the server's node, :program:`traffic_ctl` provides +several options to achieve this, please check ``traffic_ctl_rpc``. + diff --git a/doc/developer-guide/jsonrpc/traffic_ctl-development.en.rst b/doc/developer-guide/jsonrpc/traffic_ctl-development.en.rst new file mode 100644 index 00000000000..3a219578c54 --- /dev/null +++ b/doc/developer-guide/jsonrpc/traffic_ctl-development.en.rst @@ -0,0 +1,214 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: ../../common.defs + +.. |RPC| replace:: JSONRPC 2.0 + +.. _YAML: https://github.com/jbeder/yaml-cpp/wiki/Tutorial + +.. _developer-guide-traffic_ctl-development: + +Traffic Control Development Guide +********************************* + +Traffic Control interacts with |TS| through the |RPC| endpoint. All interaction is done by following the |RPC| protocol. + +Overall structure +================= + +.. figure:: ../../uml/images/traffic_ctl-class-diagram.svg + + +* The whole point is to separate the command handling from the printing part. +* Printing should be done by an appropriate Printer implementation, this should support several kinds of printing formats. +* For now, everything is printing in the standard output, but a particular printer can be implemented in such way that + the output could be sent to a different destination. +* JSONRPC requests have a base class that hides some of the basic and common parts, like ``id``, and ``version``. When deriving + from this class, the only thing that needs to be override is the ``method`` + + +.. important:: + + CtrlCommand will invoke ``_invoked_func`` when executed, this should be set by the derived class + +* The whole design is that the command will execute the ``_invoked_func`` once invoked. This function ptr should be set by the + appropriated derived class based on the passed parameters. The derived class have the option to override the execute() which + is a ``virtual`` function and does something else. Check ``RecordCommand`` as an example. + + +Command implementation +====================== + +#. Add the right command to the ``ArgParser`` object inside the ``traffic_ctl.cc``. +#. If needed, define a new ``Command`` derived class inside the ``CtrlCommands`` file. if it's not an new command level, and it's a subcommand, + then you should check the existing command to decide where to place it. + + * Implement the member function that will be dealing with the particular command, ie: (config_status()) + + * If a new JsonRPC Message needs to be done, then implement it by deriving from ``shared::rpc::ClientRequest`` if a method is needed, or from + ``shared::rpc::ClientRequestNotification`` if it's a notification. More info can be found here :ref:`jsonrpc-request` and + :ref:`jsonrpc_development-design`. This can be done inside the ``RPCRequest.h`` file. + + + .. note:: + + Make sure you override the ``std::string get_method() const`` member function with the appropriate api method name. + + +#. If needed define a new ``Printer`` derived class inside the ``CtrlPrinter`` file. + + * If pretty printing format will be supported, then make sure you read the ``_format`` member you get from the ``BasePrinter`` class. + +#. If it's a new command level (like config, metric, etc), make sure you update the ``Command`` creation inside the ``traffic_ctl.cc`` file. + +Implementation Example +====================== + +Let's define a new command for a new specific API with name == ``admin_new_command_1`` with the following json structure: + +.. code-block: json + + { + "id":"0f0780a5-0758-4f51-a177-752facc7c0eb", + "jsonrpc":"2.0", + "method":"admin_new_command_1", + "params":{ + "named_var_1":"Some value here" + } + } + + +.. code-block:: bash + + $ traffic_ctl new-command new-subcommand1 + +#. Update ``traffic_ctl.cc``. I will ignore the details as they are trivial. + +#. Define a new Request. + + So based on the above json, we can model our request as: + + .. code-block:: cpp + + // RPCRequests.h + struct NewCommandJsonRPCRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + struct Params { + std::string named_var_1; + }; + NewCommandJsonRPCRequest(Params p) + { + super::params = p; // This will invoke the built-in conversion mechanism in the yamlcpp library. + } + // Important to override this function, as this is the only way that the "method" field will be set. + std::string + get_method() const { + return "admin_new_command_1"; + } + }; + +#. Implement the yamlcpp convert function, Yaml-cpp has a built-in conversion mechanism. You can refer to `YAML`_ for more info. + + .. code-block:: cpp + + // yaml_codecs.h + template <> struct convert { + static Node + encode(NewCommandJsonRPCRequest::Params const ¶ms) + { + Node node; + node["named_var_1"] = params.named_var_1; + return node; + } + }; + +#. Define a new command. For the sake of simplicity I'll only implement it in the ``.h`` files. + + .. code-block:: cpp + + // CtrlCommands.h & CtrlCommands.cc + struct NewCommand : public CtrlCommand { + NewCommand(ts::Arguments args): CtrlCommand(args) + { + // we are interested in the format. + auto fmt = parse_format(_arguments); + if (args.get("new-subcommand1") { + // we need to create the right printer. + _printer = std::make_sharec(fmt); + // we need to set the _invoked_func that will be called when execute() is called. + _invoked_func = [&]() { handle_new_subcommand1(); }; + } + // if more subcommands are needed, then add them here. + } + + private: + void handle_new_subcommand1() { + NewCommandJsonRPCRequest req{}; + // fill the req if needed. + auto response = invoke_rpc(req); + _printer->write_output(response); + } + }; + +#. Define a new printer to deal with this command. We will assume that the printing will be different for every subcommand. + so we will create our own one. + + + .. code-block:: cpp + + class MyNewSubcommandPrinter : public BasePrinter + { + void write_output(YAML::Node const &result) override { + // result will contain what's coming back from the server. + } + }; + + In case that the format type is important, then we should allow it by accepting the format being passed in the constructor. + And let it set the base one as well. + + .. code-block:: cpp + + MyNewSubcommandPrinter(BasePrinter::Format fmt) : BasePrinter(fmt) {} + + + + The way you print and the destination of the message is up to the developer's needs, either a terminal or some other place. If the response + from the server is a complex object, you can always model the response with your own type and use the built-in yamlcpp mechanism + to decode the ``YAML::Node``. ``write_output(YAML::Node const &result)`` will only have the result defined in the protocol, + check :ref:`jsonrpc-result` for more detail. So something like this can be easily achieved: + + .. code-block:: cpp + + void + GetHostStatusPrinter::write_output(YAML::Node const &result) + { + auto response = result.as(); // will invoke the yamlcpp decode. + // you can now deal with the Record object and not with the yaml node. + } + +Notes +===== + +There is code that was written in this way by design, ``RecordPrinter`` and ``RecordRequest`` are meant to be use by any command +that needs to query and print records without any major hassle. + + + +:ref:`admnin-jsonrpc-configuration`, +:ref:`traffic_ctl_jsonrpc`, +:ref:`jsonrpc_development` diff --git a/doc/uml/JsonRPCManager.uml b/doc/uml/JsonRPCManager.uml new file mode 100644 index 00000000000..ac1e3b158c8 --- /dev/null +++ b/doc/uml/JsonRPCManager.uml @@ -0,0 +1,32 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. +@startuml +class JsonRPCManager { +add_handler(name, Func) +add_notification_handler(name, Func) +rsponse handle_call(request) +JsonRPCManager instance() +} +note left: Singleton class. + +class Dispatcher { + _handlers: std::unordered_map +} + +class InternalHandler { + _func: std::variant +} + +class FunctionWrapper { + callback: std::function +} + +JsonRPCManager *-- Dispatcher +note right: Class that knows how to call each callback handler. Storage class. +Dispatcher *-- InternalHandler +InternalHandler *-- FunctionWrapper +@enduml \ No newline at end of file diff --git a/doc/uml/traffic_ctl-class-diagram.uml b/doc/uml/traffic_ctl-class-diagram.uml new file mode 100644 index 00000000000..a48483715ea --- /dev/null +++ b/doc/uml/traffic_ctl-class-diagram.uml @@ -0,0 +1,55 @@ +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +' Unless required by applicable law or agreed to in writing, software distributed under the License is distributed +' on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and limitations under the License. +@startuml +class traffic_ctl { + +command :std::shared_ptr +parser: ArgParser +} +note left: ArgParser Invoker +class CtrlCommand { +virtual void execute() +_printer: std::unique_ptr +} + +class DirectRPCCommand { +void execute() +} +class RecordCommand { +void execute() +} +class XCommand { +void execute() +} + + +traffic_ctl -|> CtrlCommand +CtrlCommand <|-- RecordCommand +CtrlCommand <|-- DirectRPCCommand +CtrlCommand <|-- XCommand + +abstract BasePrinter { +void write_output(JSONRPCResponse) +virtual void write_output(std::string_view) +} +CtrlCommand *-- BasePrinter + +class XPrinter{ +void write_output(JSONRPCResponse) +} +class DiffConfigPrinter{ +void write_output(JSONRPCResponse) +} + +class RecordPrinter{ +void write_output(JSONRPCResponse) +} + +BasePrinter <|-- DiffConfigPrinter +BasePrinter <|-- RecordPrinter +BasePrinter <|-- XPrinter +@enduml diff --git a/include/shared/rpc/IPCSocketClient.h b/include/shared/rpc/IPCSocketClient.h new file mode 100644 index 00000000000..841f2540695 --- /dev/null +++ b/include/shared/rpc/IPCSocketClient.h @@ -0,0 +1,92 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include + +namespace shared::rpc +{ +/// The goal of this class is abstract the Unix Socket implementation and provide a JSONRPC Node client for Tests and client's +/// applications like traffic_ctl and traffic_top. +/// To make the usage easy and more readable this class provides a chained API, so you can do this like this: +/// +/// IPCSocketClient client; +/// auto resp = client.connect().send(json).read(); +/// +/// There is also a @c RPCClient class which should be used unless you need some extra control of the socket client. +/// +/// Error handling: Enclose this inside a try/catch because if any error is detected functions will throw. +struct IPCSocketClient { + enum class ReadStatus { NO_ERROR = 0, BUFFER_FULL, STREAM_ERROR, UNKNOWN }; + using self_reference = IPCSocketClient &; + + IPCSocketClient(std::string path) : _path{std::move(path)} {} + IPCSocketClient() : _path{"/tmp/jsonrpc20.sock"} {} + + ~IPCSocketClient() { this->disconnect(); } + + /// Connect to the configured socket path. + self_reference connect(); + + /// Send all the passed string to the socket. + self_reference send(std::string_view data); + + /// Read all the content from the socket till the passed buffer is full. + ReadStatus read_all(ts::FixedBufferWriter &bw); + + /// Closes the socket. + void + disconnect() + { + this->close(); + } + + /// Close the socket. + void + close() + { + if (_sock > 0) { + ::close(_sock); + _sock = -1; + } + } + + /// Test if the socket was closed or it wasn't initialized. + bool + is_closed() const + { + return (_sock < 0); + } + +protected: + std::string _path; + struct sockaddr_un _server; + + int _sock{-1}; +}; +} // namespace shared::rpc diff --git a/include/shared/rpc/README.md b/include/shared/rpc/README.md new file mode 100644 index 00000000000..23a72ef9cae --- /dev/null +++ b/include/shared/rpc/README.md @@ -0,0 +1,9 @@ +# JSONRPC 2.0 Client API utility definitions. + +All this definitions are meant to be used by clients of the JSONRPC node which +are looking to interact with it in a different C++ application, like traffic_ctl +and traffic_top. +All this definitions under the shared::rpc namespace are a client lightweight +version of the ones used internally by the JSONRPC node server/handlers, they +should not be mixed with the ones defined in `mgmt2/rpc/jsonrpc` which are for +internal use only. diff --git a/include/shared/rpc/RPCClient.h b/include/shared/rpc/RPCClient.h new file mode 100644 index 00000000000..5cdfea84315 --- /dev/null +++ b/include/shared/rpc/RPCClient.h @@ -0,0 +1,107 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +/// JSONRPC 2.0 RPC network client. + +#include +#include + +#include +#include +#include + +#include "IPCSocketClient.h" +#include "yaml_codecs.h" + +namespace shared::rpc +{ +/// +/// @brief Wrapper class to interact with the RPC node. Do not use this internally, this is for client's applications only. +/// +class RPCClient +{ + // Large buffer, as we may query a full list of records. + // TODO: should we add a parameter to increase the buffer? or maybe a record limit on the server's side? + static constexpr int BUFFER_SIZE{356000}; + +public: + RPCClient() : _client(Layout::get()->runtimedir + "/jsonrpc20.sock") {} + + /// @brief invoke the remote function using the passed jsonrpc message string. + /// This function will connect with the remote rpc node and send the passed json string. If you don't want to deal with the + /// endode/decode you can just call @c invoke(JSONRPCRequest const &req). + /// @throw runtime_error + std::string + invoke(std::string_view req) + { + std::string text; // for error messages. + ts::LocalBufferWriter bw; + try { + _client.connect(); + if (!_client.is_closed()) { + _client.send(req); + switch (_client.read_all(bw)) { + case IPCSocketClient::ReadStatus::NO_ERROR: { + _client.disconnect(); + return {bw.data(), bw.size()}; + } + case IPCSocketClient::ReadStatus::BUFFER_FULL: { + throw std::runtime_error( + ts::bwprint(text, "Buffer full, not enough space to read the response. Buffer size: {}", BUFFER_SIZE)); + } break; + default: + throw std::runtime_error("Something happened, we can't read the response"); + break; + } + } else { + throw std::runtime_error(ts::bwprint(text, "Node seems not available: {}", std ::strerror(errno))); + } + } catch (std::exception const &ex) { + _client.disconnect(); + throw std::runtime_error(ts::bwprint(text, "RPC Node Error: {}", ex.what())); + } + + return {}; + } + + /// @brief Invoke the rpc node passing the JSONRPC objects. + /// This function will connect with the remote rpc node and send the passed objects which will be encoded and decoded using the + /// yamlcpp_json_emitter impl. + /// @note If you inherit from @c JSONRPCRequest make sure the base members are properly filled before calling this function, the + /// encode/decode will only deal with the @c JSONRPCRequest members, unless you pass your own codec class. By default @c + /// yamlcpp_json_emitter is used. If you pass your own Codecs, make sure you follow the yamlcpp_json_emitter API. + /// @throw runtime_error + /// @throw YAML::Exception + template + JSONRPCResponse + invoke(JSONRPCRequest const &req) + { + static_assert(internal::has_decode::value || internal::has_encode::value, + "You need to implement encode/decode in your own codec impl."); + // We should add a static_assert and make sure encode/decode are part of Codec type. + auto const &reqStr = Codec::encode(req); + return Codec::decode(invoke(reqStr)); + } + +private: + IPCSocketClient _client; +}; +} // namespace shared::rpc \ No newline at end of file diff --git a/include/shared/rpc/RPCRequests.h b/include/shared/rpc/RPCRequests.h new file mode 100644 index 00000000000..b12c666dcf4 --- /dev/null +++ b/include/shared/rpc/RPCRequests.h @@ -0,0 +1,228 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include +#include +#include +#include + +/// JSONRPC 2.0 Client API utility definitions. Only client applications should use these definitions. Internal handlers should not +/// use these definitions. Check @c mgmg2/rpc/jsonrpc/Defs.h instead + +/// @brief @c JSONRPC 2.0 message mapping classes. +/// This is a very thin API to deal with encoding/decoding jsonrpc 2.0 messages. +/// More info can be found https://www.jsonrpc.org/specification +namespace shared::rpc +{ +struct JSONRPCRequest { + std::string jsonrpc{"2.0"}; //!< Always 2.0 as this is the only version that teh server supports. + std::string method; //!< remote method name. + std::string id; //!< optional, only needed for method calls. + YAML::Node params; //!< This is defined by each remote API. + + virtual std::string + get_method() const + { + return "method"; + } +}; + +struct JSONRPCResponse { + std::string id; //!< Always 2.0 as this is the only version that teh server supports. + std::string jsonrpc; //!< Always 2.0 + YAML::Node result; //!< Server's response, this could be decoded by using the YAML::convert mechanism. This depends solely on the + //!< server's data. Check docs and schemas. + YAML::Node error; //!< Server's error. + + /// Handy function to check if the server sent any error + bool + is_error() const + { + return !error.IsNull(); + } + + YAML::Node fullMsg; +}; + +struct JSONRPCError { + int32_t code; //!< High level error code. + std::string message; //!< High level message + // the following data is defined by TS, it will be a key/value pair. + std::vector> data; + friend std::ostream &operator<<(std::ostream &os, const JSONRPCError &err); +}; + +/** + All of the following definitions have the main purpose to have a object style idiom when dealing with request and responses from/to + the JSONRPC server. This structures will then be used by the YAML codec implementation by using the YAML::convert style. +*/ + +/// +/// @brief Base Client JSONRPC client request. +/// +/// This represents a base class that implements the basic jsonrpc 2.0 required fields. We use UUID as an id generator +/// but this was an arbitrary choice, there is no conditions that forces to use this, any random id could work too. +/// When inherit from this class the @c id and the @c jsonrpc fields which are constant in all the request will be automatically +/// generated. +// TODO: fix this as id is optional. +struct ClientRequest : JSONRPCRequest { + using super = JSONRPCRequest; + ClientRequest() { super::id = _idGen.getString(); } + +private: + struct IdGenerator { + IdGenerator() { _uuid.initialize(TS_UUID_V4); } + const char * + getString() + { + return _uuid.valid() ? _uuid.getString() : "fix.this.is.not.an.id"; + } + ATSUuid _uuid; + }; + IdGenerator _idGen; +}; + +/// @brief Class definition just to make clear that it will be a notification and no ID will be set. +struct ClientRequestNotification : JSONRPCRequest { + ClientRequestNotification() {} +}; +/** + * Specific JSONRPC request implementation should be placed here. All this definitions helps for readability and in particular + * to easily emit json(or yaml) from this definitions. + */ + +//------------------------------------------------------------------------------------------------------------------------------------ + +// handy definitions. +static const std::vector CONFIG_REC_TYPES = {1, 16}; +static const std::vector METRIC_REC_TYPES = {2, 4, 32}; +static constexpr bool NOT_REGEX{false}; +static constexpr bool REGEX{true}; + +/// +/// @brief Record lookup API helper class. +/// +/// This utility class is used to encapsulate the basic data that contains a record lookup request. +/// Requests that are meant to interact with the admin_lookup_records API should inherit from this class if a special treatment is +/// needed. Otherwise use it directly. +/// +struct RecordLookupRequest : ClientRequest { + using super = ClientRequest; + struct Params { + std::string recName; + bool isRegex{false}; + std::vector recTypes; + }; + std::string + get_method() const + { + return "admin_lookup_records"; + } + template + void + emplace_rec(Args &&... p) + { + super::params.push_back(Params{std::forward(p)...}); + } +}; + +struct RecordLookUpResponse { + /// Response Records API mapping utility classes. + /// This utility class is used to hold the decoded response. + struct RecordParamInfo { + std::string name; + int32_t type; + int32_t version; + bool registered; + int32_t rsb; + int32_t order; + int32_t rclass; + bool overridable; + std::string dataType; + std::string currentValue; + std::string defaultValue; + + struct ConfigMeta { + int32_t accessType; + int32_t updateStatus; + int32_t updateType; + int32_t checkType; + int32_t source; + std::string checkExpr; + }; + struct StatMeta { + int32_t persistType; + }; + std::variant meta; + }; + /// Record request error mapping class. + struct RecordError { + std::string code; + std::string recordName; + std::string message; //!< optional. + friend std::ostream &operator<<(std::ostream &os, const RecordError &re); + }; + + std::vector recordList; + std::vector errorList; +}; + +//------------------------------------------------------------------------------------------------------------------------------------ + +inline std::ostream & +operator<<(std::ostream &os, const RecordLookUpResponse::RecordError &re) +{ + std::string text; + os << ts::bwprint(text, "{:16s}: {}\n", "Record Name ", re.recordName); + os << ts::bwprint(text, "{:16s}: {}\n", "Code", re.code); + if (!re.message.empty()) { + os << ts::bwprint(text, "{:16s}: {}\n", "Message", re.message); + } + return os; +} + +inline std::ostream & +operator<<(std::ostream &os, const JSONRPCError &err) +{ + os << "Error found.\n"; + os << "code: " << err.code << '\n'; + os << "message: " << err.message << '\n'; + if (err.data.size() > 0) { + os << "---\nAdditional error information found:\n"; + auto my_print = [&](auto const &e) { + os << "+ code: " << e.first << '\n'; + os << "+ message: " << e.second << '\n'; + }; + + auto iter = std::begin(err.data); + + my_print(*iter); + ++iter; + for (; iter != std::end(err.data); ++iter) { + os << "---\n"; + my_print(*iter); + } + } + + return os; +} +} // namespace shared::rpc \ No newline at end of file diff --git a/include/shared/rpc/yaml_codecs.h b/include/shared/rpc/yaml_codecs.h new file mode 100644 index 00000000000..42610dde570 --- /dev/null +++ b/include/shared/rpc/yaml_codecs.h @@ -0,0 +1,243 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include + +#include "RPCRequests.h" + +/// JSONRPC 2.0 Client API request/response codecs only. If you need to define your own specific codecs they should then be defined +/// in a different file, unless they are strongly related to the ones defined here. + +namespace helper +{ +// For some fields, If we can't get the value, then just send the default/empty value. Let the +// traffic_ctl display something. +template +inline auto +try_extract(YAML::Node const &node, const char *name, bool throwOnFail = false) +{ + try { + if (auto n = node[name]) { + return n.as(); + } + } catch (YAML::Exception const &ex) { + if (throwOnFail) { + throw ex; + } + } + return T{}; +} +} // namespace helper +/** + * YAML namespace. All json rpc request codecs can be placed here. It will read all the definitions from "requests.h" + * It's noted that there may be some duplicated with the rpc server implementation structures but as this is very simple idiom where + * we define the data as plain as possible and it's just used for printing purposes, No major harm on having them duplicated + */ + +namespace YAML +{ +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, shared::rpc::JSONRPCError &error) + { + error.code = helper::try_extract(node, "code"); + error.message = helper::try_extract(node, "message"); + if (auto data = node["data"]) { + for (auto &&err : data) { + error.data.emplace_back(helper::try_extract(err, "code"), helper::try_extract(err, "message")); + } + } + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, shared::rpc::RecordLookUpResponse::RecordParamInfo::ConfigMeta &meta) + { + meta.accessType = helper::try_extract(node, "access_type"); + meta.updateStatus = helper::try_extract(node, "update_status"); + meta.updateType = helper::try_extract(node, "update_type"); + meta.checkType = helper::try_extract(node, "checktype"); + meta.source = helper::try_extract(node, "source"); + meta.checkExpr = helper::try_extract(node, "check_expr"); + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, shared::rpc::RecordLookUpResponse::RecordParamInfo::StatMeta &meta) + { + meta.persistType = helper::try_extract(node, "persist_type"); + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, shared::rpc::RecordLookUpResponse::RecordParamInfo &info) + { + info.name = helper::try_extract(node, "record_name"); + info.type = helper::try_extract(node, "record_type"); + info.version = helper::try_extract(node, "version"); + info.registered = helper::try_extract(node, "registered"); + info.rsb = helper::try_extract(node, "raw_stat_block"); + info.order = helper::try_extract(node, "order"); + info.rclass = helper::try_extract(node, "record_class"); + info.overridable = helper::try_extract(node, "overridable"); + info.dataType = helper::try_extract(node, "data_type"); + info.currentValue = helper::try_extract(node, "current_value"); + info.defaultValue = helper::try_extract(node, "default_value"); + try { + if (auto n = node["config_meta"]) { + info.meta = n.as(); + } else if (auto n = node["stat_meta"]) { + info.meta = n.as(); + } + } catch (Exception const &ex) { + return false; + } + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, shared::rpc::RecordLookUpResponse &info) + { + try { + auto records = node["recordList"]; + for (auto &&item : records) { + if (auto record = item["record"]) { + info.recordList.push_back(record.as()); + } + } + + auto errors = node["errorList"]; + for (auto &&item : errors) { + info.errorList.push_back(item.as()); + } + } catch (Exception const &ex) { + return false; + } + return true; + } +}; + +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(shared::rpc::RecordLookupRequest::Params const &info) + { + Node record; + if (info.isRegex) { + record["record_name_regex"] = info.recName; + } else { + record["record_name"] = info.recName; + } + + record["rec_types"] = info.recTypes; + return record; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, shared::rpc::RecordLookUpResponse::RecordError &err) + { + err.code = helper::try_extract(node, "code"); + err.recordName = helper::try_extract(node, "record_name"); + err.message = helper::try_extract(node, "message"); + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +} // namespace YAML +namespace internal +{ +template class, typename = std::void_t<>> struct detect_member_function : std::false_type { +}; +template class Op> +struct detect_member_function>> : std::true_type { +}; +// Help to detect if codec implements the right functions. +template using encode_op = decltype(std::declval().encode({})); +template using decode_op = decltype(std::declval().decode({})); + +// Compile time check for encode member function +template using has_encode = detect_member_function; +// Compile time check for decode member function +template using has_decode = detect_member_function; +} // namespace internal +/** + * Handy classes to deal with the json emitters. If yaml needs to be emitted, then this should be changed and do not use the + * double quoted flow. + */ +class yamlcpp_json_emitter +{ +public: + static std::string + encode(shared::rpc::JSONRPCRequest const &req) + { + YAML::Emitter json; + json << YAML::DoubleQuoted << YAML::Flow; + json << YAML::BeginMap; + + if (!req.id.empty()) { + json << YAML::Key << "id" << YAML::Value << req.id; + } + json << YAML::Key << "jsonrpc" << YAML::Value << req.jsonrpc; + json << YAML::Key << "method" << YAML::Value << req.get_method(); + if (!req.params.IsNull()) { + json << YAML::Key << "params" << YAML::Value << req.params; + } + json << YAML::EndMap; + return json.c_str(); + } + + static shared::rpc::JSONRPCResponse + decode(const std::string &response) + { + shared::rpc::JSONRPCResponse resp; + resp.fullMsg = YAML::Load(response.data()); + if (resp.fullMsg.Type() != YAML::NodeType::Map) { + throw std::runtime_error{"error parsing response, response is not a structure"}; + } + + if (resp.fullMsg["result"]) { + resp.result = resp.fullMsg["result"]; + } else if (resp.fullMsg["error"]) { + resp.error = resp.fullMsg["error"]; + } + + if (auto id = resp.fullMsg["id"]) { + resp.id = id.as(); + } + if (auto jsonrpc = resp.fullMsg["jsonrpc"]) { + resp.jsonrpc = jsonrpc.as(); + } + + return resp; + } +}; \ No newline at end of file diff --git a/include/tscore/ArgParser.h b/include/tscore/ArgParser.h index a7b4b7a4f0a..2e6b93e3ff6 100644 --- a/include/tscore/ArgParser.h +++ b/include/tscore/ArgParser.h @@ -250,6 +250,9 @@ class ArgParser // get the error message std::string get_error() const; + // Add App's description. + void add_description(std::string const &descr); + protected: // Converted from 'const char **argv' for the use of parsing and help AP_StrVec _argv; diff --git a/include/tscore/Errata.h b/include/tscore/Errata.h index b06d66480f8..227a9456a02 100644 --- a/include/tscore/Errata.h +++ b/include/tscore/Errata.h @@ -63,12 +63,12 @@ */ #pragma once - #include #include #include #include #include +#include #include "NumericType.h" #include "IntrusivePtr.h" @@ -138,6 +138,9 @@ class Errata Errata(Message const &msg ///< Message to push ); + /// Constructor with @a id and @a std::error_code + Errata(std::error_code const &ec ///< Standard error code. + ); /// Move constructor. Errata(self &&that); /// Move constructor from @c Message. @@ -570,6 +573,11 @@ struct RvBase { RvBase(Errata const &s ///< Status to copy ); + /** Construct with specific status. + */ + RvBase(Errata &&s ///< Status to move + ); + //! Test the return value for success. bool isOK() const; @@ -624,6 +632,7 @@ template struct Rv : public RvBase { Errata const &s ///< A pre-existing status object ); + Rv(Errata &&errata); /** User conversion to the result type. This makes it easy to use the function normally or to pass the @@ -815,6 +824,12 @@ inline Errata::Errata(Message &&msg) { this->push(std::move(msg)); } +inline Errata::Errata(std::error_code const &ec) +{ + auto cond = ec.category().default_error_condition(ec.value()); + this->push(cond.value(), // we use the classification from the error_condition. + ec.value(), ec.message()); +} inline Errata::operator bool() const { @@ -862,6 +877,16 @@ Errata::push(Id id, Code code, Args const &... args) -> self & return *this; } +inline Errata & +Errata::push(Errata const &err) +{ + for (auto const &e : err) { + this->push(e.m_id, e.m_code, e.m_text); + // e.m_errata?? + } + return *this; +} + inline Errata::Message const & Errata::top() const { @@ -939,6 +964,7 @@ Errata::const_iterator::operator--() inline RvBase::RvBase() {} inline RvBase::RvBase(Errata const &errata) : _errata(errata) {} +inline RvBase::RvBase(Errata &&errata) : _errata(std::move(errata)) {} inline bool RvBase::isOK() const { @@ -958,6 +984,7 @@ RvBase::doNotLog() template Rv::Rv() : _result() {} template Rv::Rv(Result const &r) : _result(r) {} template Rv::Rv(Result const &r, Errata const &errata) : super(errata), _result(r) {} +template Rv::Rv(Errata &&errata) : super(std::move(errata)) {} template Rv::operator Result const &() const { return _result; diff --git a/include/tscore/Filenames.h b/include/tscore/Filenames.h index 60d8441bd1d..f2f0718f9bb 100644 --- a/include/tscore/Filenames.h +++ b/include/tscore/Filenames.h @@ -41,6 +41,7 @@ namespace filename constexpr const char *SSL_MULTICERT = "ssl_multicert.config"; constexpr const char *SPLITDNS = "splitdns.config"; constexpr const char *SNI = "sni.yaml"; + constexpr const char *JSONRPC = "jsonrpc.yaml"; /////////////////////////////////////////////////////////////////// // Various other file names diff --git a/include/tscpp/util/ts_meta.h b/include/tscpp/util/ts_meta.h index 116650fd9eb..a38742ec020 100644 --- a/include/tscpp/util/ts_meta.h +++ b/include/tscpp/util/ts_meta.h @@ -94,5 +94,12 @@ namespace meta CaseVoidFunc() { } + + // helper type for the visitor + template struct overloaded : Ts... { + using Ts::operator()...; + }; + // explicit deduction guide (not needed as of C++20) + template overloaded(Ts...) -> overloaded; } // namespace meta } // namespace ts diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc index 808db21ab0e..fe6fad8ce6c 100644 --- a/mgmt/RecordsConfig.cc +++ b/mgmt/RecordsConfig.cc @@ -1527,7 +1527,9 @@ static const RecordElement RecordsConfig[] = //# 0 - no checking. 1 - log in mismatch. 2 - enforcing //# //########### - {RECT_CONFIG, "proxy.config.http.host_sni_policy", RECD_INT, "2", RECU_NULL, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL}, + {RECT_CONFIG, "proxy.config.http.host_sni_policy", RECD_INT, "2", RECU_NULL, RR_NULL, RECC_NULL, "[0-2]", RECA_NULL} + , + {RECT_CONFIG, "proxy.config.jsonrpc.filename", RECD_STRING, ts::filename::JSONRPC, RECU_RESTART_TS, RR_NULL, RECC_NULL, nullptr, RECA_NULL} }; // clang-format on diff --git a/mgmt2/Makefile.am b/mgmt2/Makefile.am new file mode 100644 index 00000000000..1b1e3f16b5d --- /dev/null +++ b/mgmt2/Makefile.am @@ -0,0 +1,20 @@ +# +# Makefile.am for the Enterprise Management module. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SUBDIRS = config rpc diff --git a/mgmt2/config/AddConfigFilesHere.cc b/mgmt2/config/AddConfigFilesHere.cc new file mode 100644 index 00000000000..685817ccb4d --- /dev/null +++ b/mgmt2/config/AddConfigFilesHere.cc @@ -0,0 +1,85 @@ +/** @file + + A brief file description + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include "tscore/ink_platform.h" +#include "tscore/Filenames.h" +#include "records/P_RecCore.h" +#include "tscore/Diags.h" +#include "FileManager.h" +#include "tscore/Errata.h" + +static constexpr bool REQUIRED{true}; +static constexpr bool NOT_REQUIRED{false}; +/**************************************************************************** + * + * AddConfigFilesHere.cc - Structs for config files and + * + * + ****************************************************************************/ +void +registerFile(const char *configName, const char *defaultName, bool isRequired) +{ + bool found = false; + const char *fname = REC_readString(configName, &found); + if (!found) { + fname = defaultName; + } + FileManager::instance().addFile(fname, configName, false, isRequired); +} + +// +// initializeRegistry() +// +// Code to initialize of registry of objects that represent +// Web Editable configuration files +// +// thread-safe: NO! - Should only be executed once from the main +// web interface thread, before any child +// threads have been spawned +void +initializeRegistry() +{ + static int run_already = 0; + + if (run_already == 0) { + run_already = 1; + } else { + ink_assert(!"Configuration Object Registry Initialized More than Once"); + } + + registerFile("proxy.config.log.config.filename", ts::filename::LOGGING, NOT_REQUIRED); + registerFile("", ts::filename::STORAGE, REQUIRED); + registerFile("proxy.config.socks.socks_config_file", ts::filename::SOCKS, NOT_REQUIRED); + registerFile(ts::filename::RECORDS, ts::filename::RECORDS, NOT_REQUIRED); + registerFile("proxy.config.cache.control.filename", ts::filename::CACHE, NOT_REQUIRED); + registerFile("proxy.config.cache.ip_allow.filename", ts::filename::IP_ALLOW, NOT_REQUIRED); + registerFile("proxy.config.http.parent_proxy.file", ts::filename::PARENT, NOT_REQUIRED); + registerFile("proxy.config.url_remap.filename", ts::filename::REMAP, NOT_REQUIRED); + registerFile("", ts::filename::VOLUME, NOT_REQUIRED); + registerFile("proxy.config.cache.hosting_filename", ts::filename::HOSTING, NOT_REQUIRED); + registerFile("", ts::filename::PLUGIN, NOT_REQUIRED); + registerFile("proxy.config.dns.splitdns.filename", ts::filename::SPLITDNS, NOT_REQUIRED); + registerFile("proxy.config.ssl.server.multicert.filename", ts::filename::SSL_MULTICERT, NOT_REQUIRED); + registerFile("proxy.config.ssl.servername.filename", ts::filename::SNI, NOT_REQUIRED); + registerFile("proxy.config.jsonrpc.filename", ts::filename::JSONRPC, NOT_REQUIRED); +} diff --git a/mgmt2/config/FileManager.cc b/mgmt2/config/FileManager.cc new file mode 100644 index 00000000000..a2880cf833a --- /dev/null +++ b/mgmt2/config/FileManager.cc @@ -0,0 +1,443 @@ +/** @file + + Code for class to manage configuration updates + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +#include "FileManager.h" + +#include +#include + +#include "InkAPIInternal.h" // TODO: this brings a lot of dependencies, double check this. + +#include "tscore/ink_platform.h" +#include "tscore/ink_file.h" +#include "ConfigManager.h" +#include "records/P_RecCore.h" +#include "tscore/Diags.h" +#include "tscore/Filenames.h" +#include "tscore/I_Layout.h" +#include + +#if HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC +#define TS_ARCHIVE_STAT_MTIME(t) ((t).st_mtime * 1000000000 + (t).st_mtimespec.tv_nsec) +#elif HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC +#define TS_ARCHIVE_STAT_MTIME(t) ((t).st_mtime * 1000000000 + (t).st_mtim.tv_nsec) +#else +#define TS_ARCHIVE_STAT_MTIME(t) ((t).st_mtime * 1000000000) +#endif + +static constexpr auto logTag{"filemanager"}; +namespace +{ +ts::Errata +handle_file_reload(std::string const &fileName, std::string const &configName) +{ + Debug(logTag, "handling reload %s - %s", fileName.c_str(), configName.c_str()); + ts::Errata ret; + // TODO: make sure records holds the name after change, if not we should change it. + if (fileName == ts::filename::RECORDS) { + if (RecReadConfigFile() == REC_ERR_OKAY) { + RecConfigWarnIfUnregistered(); + } else { + std::string str; + ret.push(1, ts::bwprint(str, "Error reading {}.", fileName)); + } + } else { + RecT rec_type; + char *data = const_cast(configName.c_str()); + if (RecGetRecordType(data, &rec_type) == REC_ERR_OKAY && rec_type == RECT_CONFIG) { + RecSetSyncRequired(data); + } else { + std::string str; + ret.push(1, ts::bwprint(str, "Unknown file change {}.", configName)); + } + } + + return ret; +} + +// JSONRPC endpoint defs. +const std::string CONFIG_REGISTRY_KEY_STR{"config_registry"}; +const std::string FILE_PATH_KEY_STR{"file_path"}; +const std::string RECORD_NAME_KEY_STR{"config_record_name"}; +const std::string PARENT_CONFIG_KEY_STR{"parent_config"}; +const std::string ROOT_ACCESS_NEEDED_KEY_STR{"root_access_needed"}; +const std::string IS_REQUIRED_KEY_STR{"is_required"}; +const std::string NA_STR{"N/A"}; + +} // namespace + +FileManager::FileManager() +{ + ink_mutex_init(&accessLock); + this->registerCallback(&handle_file_reload); + + // Register the files registry jsonrpc endpoint + rpc::add_method_handler( + "filemanager.get_files_registry", + [this](std::string_view const &id, const YAML::Node &req) -> ts::Rv { + return get_files_registry_rpc_endpoint(id, req); + }, + &rpc::core_ats_rpc_service_provider_handle); +} + +// FileManager::~FileManager +// +// There is only FileManager object in the process and it +// should never need to be destructed except at +// program exit +// +FileManager::~FileManager() +{ + // Let other operations finish and do not start any new ones + ink_mutex_acquire(&accessLock); + + for (auto &&it : bindings) { + delete it.second; + } + + ink_mutex_release(&accessLock); + ink_mutex_destroy(&accessLock); +} + +// void FileManager::addFile(char* fileName, const configFileInfo* file_info, +// ConfigManager* parentConfig) +// +// for the baseFile, creates a ConfigManager object for it +// +// if file_info is not null, a WebFileEdit object is also created for +// the file +// +// Pointers to the new objects are stored in the bindings hashtable +// +void +FileManager::addFile(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig) +{ + ink_mutex_acquire(&accessLock); + addFileHelper(fileName, configName, root_access_needed, isRequired, parentConfig); + ink_mutex_release(&accessLock); +} + +// caller must hold the lock +void +FileManager::addFileHelper(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig) +{ + ink_assert(fileName != nullptr); + ConfigManager *configManager = new ConfigManager(fileName, configName, root_access_needed, isRequired, parentConfig); + bindings.emplace(configManager->getFileName(), configManager); +} + +// bool FileManager::getConfigManagerObj(char* fileName, ConfigManager** rbPtr) +// +// Sets rbPtr to the ConfigManager object associated +// with the passed in fileName. +// +// If there is no binding, false is returned +// +bool +FileManager::getConfigObj(const char *fileName, ConfigManager **rbPtr) +{ + ink_mutex_acquire(&accessLock); + auto it = bindings.find(fileName); + bool found = it != bindings.end(); + ink_mutex_release(&accessLock); + + *rbPtr = found ? it->second : nullptr; + return found; +} + +ts::Errata +FileManager::fileChanged(std::string const &fileName, std::string const &configName) +{ + Debug("filemanager", "file changed %s", fileName.c_str()); + ts::Errata ret; + + std::lock_guard guard(_callbacksMutex); + for (auto const &call : _configCallbacks) { + if (auto const &r = call(fileName, configName); !r) { + Debug("filemanager", "something back from callback %s", fileName.c_str()); + std::for_each(r.begin(), r.end(), [&ret](auto &&e) { ret.push(e); }); + } + } + + return ret; +} + +// TODO: To do the following here, we have to pull up a lot of dependencies we don't really +// need, #include "InkAPIInternal.h" brings plenty of them. Double check this approach. RPC will +// also be able to pass messages to plugins, once that's designed it can also cover this. +void +FileManager::registerConfigPluginCallbacks(ConfigUpdateCbTable *cblist) +{ + _pluginCallbackList = cblist; +} + +void +FileManager::invokeConfigPluginCallbacks() +{ + Debug("filemanager", "invoke plugin callbacks"); + static const std::string_view s{"*"}; + if (_pluginCallbackList) { + _pluginCallbackList->invoke(s.data()); + } +} + +// void FileManger::rereadConfig() +// +// Iterates through the list of managed files and +// calls ConfigManager::checkForUserUpdate on them +// +// although it is tempting, DO NOT CALL FROM SIGNAL HANDLERS +// This function is not Async-Signal Safe. It +// is thread safe +ts::Errata +FileManager::rereadConfig() +{ + ts::Errata ret; + + ConfigManager *rb; + std::vector changedFiles; + std::vector parentFileNeedChange; + size_t n; + ink_mutex_acquire(&accessLock); + for (auto &&it : bindings) { + rb = it.second; + // ToDo: rb->isVersions() was always true before, because numberBackups was always >= 1. So ROLLBACK_CHECK_ONLY could not + // happen at all... + if (rb->checkForUserUpdate(FileManager::ROLLBACK_CHECK_AND_UPDATE)) { + Debug(logTag, "File %s changed.", it.first.c_str()); + auto const &r = fileChanged(rb->getFileName(), rb->getConfigName()); + + if (!r) { + std::for_each(r.begin(), r.end(), [&ret](auto &&e) { ret.push(e); }); + } + + changedFiles.push_back(rb); + if (rb->isChildManaged()) { + if (std::find(parentFileNeedChange.begin(), parentFileNeedChange.end(), rb->getParentConfig()) == + parentFileNeedChange.end()) { + parentFileNeedChange.push_back(rb->getParentConfig()); + } + } + } + } + + std::vector childFileNeedDelete; + n = changedFiles.size(); + for (size_t i = 0; i < n; i++) { + if (changedFiles[i]->isChildManaged()) { + continue; + } + // for each parent file, if it is changed, then delete all its children + for (auto &&it : bindings) { + rb = it.second; + if (rb->getParentConfig() == changedFiles[i]) { + if (std::find(childFileNeedDelete.begin(), childFileNeedDelete.end(), rb) == childFileNeedDelete.end()) { + childFileNeedDelete.push_back(rb); + } + } + } + } + n = childFileNeedDelete.size(); + for (size_t i = 0; i < n; i++) { + bindings.erase(childFileNeedDelete[i]->getFileName()); + delete childFileNeedDelete[i]; + } + ink_mutex_release(&accessLock); + + n = parentFileNeedChange.size(); + for (size_t i = 0; i < n; i++) { + if (std::find(changedFiles.begin(), changedFiles.end(), parentFileNeedChange[i]) == changedFiles.end()) { + if (auto const &r = fileChanged(parentFileNeedChange[i]->getFileName(), parentFileNeedChange[i]->getConfigName()); !r) { + std::for_each(r.begin(), r.end(), [&ret](auto &&e) { ret.push(e); }); + } + } + } + // INKqa11910 + // need to first check that enable_customizations is enabled + bool found; + int enabled = static_cast(REC_readInteger("proxy.config.body_factory.enable_customizations", &found)); + + if (found && enabled) { + if (auto const &r = fileChanged("proxy.config.body_factory.template_sets_dir", "proxy.config.body_factory.template_sets_dir"); + !r) { + std::for_each(r.begin(), r.end(), [&ret](auto &&e) { ret.push(e); }); + } + } + + if (auto const &r = fileChanged("proxy.config.ssl.server.ticket_key.filename", "proxy.config.ssl.server.ticket_key.filename"); + !r) { + std::for_each(r.begin(), r.end(), [&ret](auto &&e) { ret.push(e); }); + } + + return ret; +} + +bool +FileManager::isConfigStale() +{ + ConfigManager *rb; + bool stale = false; + + ink_mutex_acquire(&accessLock); + for (auto &&it : bindings) { + rb = it.second; + if (rb->checkForUserUpdate(FileManager::ROLLBACK_CHECK_ONLY)) { + stale = true; + break; + } + } + + ink_mutex_release(&accessLock); + return stale; +} + +// void configFileChild(const char *parent, const char *child) +// +// Add child to the bindings with parentConfig +void +FileManager::configFileChild(const char *parent, const char *child) +{ + ConfigManager *parentConfig = nullptr; + ink_mutex_acquire(&accessLock); + if (auto it = bindings.find(parent); it != bindings.end()) { + Debug(logTag, "Adding child file %s to %s parent", child, parent); + parentConfig = it->second; + addFileHelper(child, "", parentConfig->rootAccessNeeded(), parentConfig->getIsRequired(), parentConfig); + } + ink_mutex_release(&accessLock); +} + +auto +FileManager::get_files_registry_rpc_endpoint(std::string_view const &id, YAML::Node const ¶ms) -> ts::Rv +{ + // If any error, the rpc manager will catch it and respond with it. + YAML::Node configs{YAML::NodeType::Sequence}; + { + ink_scoped_mutex_lock lock(accessLock); + for (auto &&it : bindings) { + if (ConfigManager *cm = it.second; cm) { + YAML::Node element{YAML::NodeType::Map}; + std::string sysconfdir(RecConfigReadConfigDir()); + element[FILE_PATH_KEY_STR] = Layout::get()->relative_to(sysconfdir, cm->getFileName()); + element[RECORD_NAME_KEY_STR] = cm->getConfigName(); + element[PARENT_CONFIG_KEY_STR] = (cm->isChildManaged() ? cm->getParentConfig()->getFileName() : NA_STR); + element[ROOT_ACCESS_NEEDED_KEY_STR] = cm->rootAccessNeeded(); + element[IS_REQUIRED_KEY_STR] = cm->getIsRequired(); + configs.push_back(element); + } + } + } + + YAML::Node registry; + registry[CONFIG_REGISTRY_KEY_STR] = configs; + return registry; +} + +/// ConfigFile + +FileManager::ConfigManager::ConfigManager(const char *fileName_, const char *configName_, bool root_access_needed_, + bool isRequired_, ConfigManager *parentConfig_) + : root_access_needed(root_access_needed_), isRequired(isRequired_), parentConfig(parentConfig_) +{ + // ExpandingArray existVer(25, true); // Existing versions + struct stat fileInfo; + ink_assert(fileName_ != nullptr); + + // parent must not also have a parent + if (parentConfig) { + ink_assert(parentConfig->parentConfig == nullptr); + } + + // Copy the file name. + fileName = ats_strdup(fileName_); + configName = ats_strdup(configName_); + + ink_mutex_init(&fileAccessLock); + // Check to make sure that our configuration file exists + // + if (statFile(&fileInfo) < 0) { + Debug(logTag, "%s Unable to load: %s", fileName, strerror(errno)); + + if (isRequired) { + Debug(logTag, " Unable to open required configuration file %s\n\t failed :%s", fileName, strerror(errno)); + } + } else { + fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); + } +} + +FileManager::ConfigManager::~ConfigManager() +{ + ats_free(fileName); +} + +// +// +// int ConfigManager::statFile() +// +// A wrapper for stat() +// +int +FileManager::ConfigManager::statFile(struct stat *buf) +{ + int statResult; + std::string sysconfdir(RecConfigReadConfigDir()); + std::string filePath = Layout::get()->relative_to(sysconfdir, fileName); + + statResult = root_access_needed ? elevating_stat(filePath.c_str(), buf) : stat(filePath.c_str(), buf); + + return statResult; +} + +// bool ConfigManager::checkForUserUpdate(RollBackCheckType how) +// +// Called to check if the file has been changed by the user. +// Timestamps are compared to see if a change occurred +bool +FileManager::ConfigManager::checkForUserUpdate(FileManager::RollBackCheckType how) +{ + struct stat fileInfo; + bool result; + + ink_mutex_acquire(&fileAccessLock); + + if (this->statFile(&fileInfo) < 0) { + ink_mutex_release(&fileAccessLock); + return false; + } + + if (fileLastModified < TS_ARCHIVE_STAT_MTIME(fileInfo)) { + if (how == FileManager::ROLLBACK_CHECK_AND_UPDATE) { + fileLastModified = TS_ARCHIVE_STAT_MTIME(fileInfo); + // TODO: syslog???? + } + Debug(logTag, "User has changed config file %s\n", fileName); + result = true; + } else { + result = false; + } + + ink_mutex_release(&fileAccessLock); + return result; +} diff --git a/mgmt2/config/FileManager.h b/mgmt2/config/FileManager.h new file mode 100644 index 00000000000..d7812f3a408 --- /dev/null +++ b/mgmt2/config/FileManager.h @@ -0,0 +1,174 @@ +/** @file + + Interface for class to manage configuration updates + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#pragma once + +#include "tscore/ink_mutex.h" +#include "tscore/List.h" + +#include "tscore/Errata.h" + +#include + +#include +#include +#include +#include +#include + +class ConfigUpdateCbTable; + +class FileManager +{ +public: + enum RollBackCheckType { + ROLLBACK_CHECK_AND_UPDATE, + ROLLBACK_CHECK_ONLY, + }; + class ConfigManager + { + public: + // fileName_ should be rooted or a base file name. + ConfigManager(const char *fileName_, const char *configName_, bool root_access_needed, bool isRequired_, + ConfigManager *parentConfig_); + ~ConfigManager(); + + // Manual take out of lock required + void + acquireLock() + { + ink_mutex_acquire(&fileAccessLock); + }; + + void + releaseLock() + { + ink_mutex_release(&fileAccessLock); + }; + + // Check if a file has changed, automatically holds the lock. Used by FileManager. + bool checkForUserUpdate(FileManager::RollBackCheckType); + + // These are getters, for FileManager to get info about a particular configuration. + const char * + getFileName() const + { + return fileName; + } + + const char * + getConfigName() const + { + return configName; + } + + bool + isChildManaged() const + { + return parentConfig != nullptr; + } + + ConfigManager * + getParentConfig() const + { + return parentConfig; + } + + bool + rootAccessNeeded() const + { + return root_access_needed; + } + + bool + getIsRequired() const + { + return isRequired; + } + + // FileManager *configFiles = nullptr; // Manager to notify on an update. + + // noncopyable + ConfigManager(const ConfigManager &) = delete; + ConfigManager &operator=(const ConfigManager &) = delete; + + private: + int statFile(struct stat *buf); + + ink_mutex fileAccessLock; + char *fileName; + char *configName; + bool root_access_needed; + bool isRequired; + ConfigManager *parentConfig; + time_t fileLastModified = 0; + }; + + using CallbackType = std::function; + + FileManager(); + ~FileManager(); + void addFile(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig = nullptr); + + bool getConfigObj(const char *fileName, ConfigManager **rbPtr); + + void + registerCallback(CallbackType f) + { + std::lock_guard guard(_callbacksMutex); + _configCallbacks.push_front(std::move(f)); + } + + ts::Errata fileChanged(std::string const &fileName, std::string const &configName); + ts::Errata rereadConfig(); + bool isConfigStale(); + void configFileChild(const char *parent, const char *child); + + void registerConfigPluginCallbacks(ConfigUpdateCbTable *cblist); + void invokeConfigPluginCallbacks(); + + static FileManager & + instance() + { + static FileManager configFiles; + return configFiles; + } + +private: + ink_mutex accessLock; // Protects bindings hashtable + ConfigUpdateCbTable *_pluginCallbackList; + + std::mutex _callbacksMutex; + std::mutex _accessMutex; + + std::forward_list _configCallbacks; + + std::unordered_map bindings; + void addFileHelper(const char *fileName, const char *configName, bool root_access_needed, bool isRequired, + ConfigManager *parentConfig); + /// JSONRPC endpoint + ts::Rv get_files_registry_rpc_endpoint(std::string_view const &id, YAML::Node const ¶ms); +}; + +void initializeRegistry(); // implemented in AddConfigFilesHere.cc diff --git a/mgmt2/config/Makefile.am b/mgmt2/config/Makefile.am new file mode 100644 index 00000000000..7e4ab952d26 --- /dev/null +++ b/mgmt2/config/Makefile.am @@ -0,0 +1,63 @@ +# +# Makefile.am for the Enterprise Management module. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + + +AM_CPPFLAGS += \ + $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/iocore/utils \ + -I$(abs_top_srcdir)/include \ + -I$(abs_top_srcdir)/lib/ \ + -I$(abs_top_srcdir)/mgmt/rpc \ + -I$(abs_top_srcdir)/mgmt/ \ + -I$(abs_top_srcdir)/mgmt2/ \ + -I$(abs_top_srcdir)/mgmt/utils \ + -I$(abs_top_srcdir)/proxy/ \ + -I$(abs_top_srcdir)/proxy/http \ + -I$(abs_top_srcdir)/proxy/hdrs \ + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ + +# ^^ all the proxy/* is to include the PluginCallbacks. + +noinst_LTLIBRARIES = libconfigmanager.la +#check_PROGRAMS = test_configfiles + + +TESTS = $(check_PROGRAMS) + +# Protocol library only, no transport. +libconfigmanager_COMMON = \ + FileManager.h \ + FileManager.cc \ + AddConfigFilesHere.cc + + +libconfigmanager_la_SOURCES = \ + $(libconfigmanager_COMMON) + +libconfigmanager_la_LIBADD = \ + $(top_builddir)/src/tscore/libtscore.la + + +include $(top_srcdir)/build/tidy.mk + +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) + diff --git a/mgmt2/rpc/Makefile.am b/mgmt2/rpc/Makefile.am new file mode 100644 index 00000000000..26ca1923644 --- /dev/null +++ b/mgmt2/rpc/Makefile.am @@ -0,0 +1,190 @@ +# +# Makefile.am for the RPC/jsonrpc module +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +AM_CPPFLAGS += \ + $(iocore_include_dirs) \ + -I$(abs_top_srcdir)/iocore/utils \ + -I$(abs_top_srcdir)/include \ + -I$(abs_top_srcdir)/lib/ \ + -I$(abs_top_srcdir)/mgmt2/ \ + $(TS_INCLUDES) \ + @YAMLCPP_INCLUDES@ + + +noinst_LTLIBRARIES = libjsonrpc_protocol.la libjsonrpc_server.la librpcpublichandlers.la +check_PROGRAMS = test_jsonrpc test_jsonrpcserver + + +TESTS = $(check_PROGRAMS) + +# TODO: Remove libmgmt_p.la This can only be removed once we remove the ProcessManager dependency and the librecords +# Message stuff between TM and TS +# TODO: handlers - mgmt/utils needed as ProcessManager.h is included in many pleaces, we should be able to remove it once +# we move away from TM + +########################################################################################### +# Protocol library only, no transport. + +libjsonrpc_protocol_COMMON = \ + jsonrpc/error/RPCError.cc \ + jsonrpc/error/RPCError.h \ + jsonrpc/JsonRPCManager.cc \ + jsonrpc/JsonRPCManager.h + +libjsonrpc_protocol_la_SOURCES = \ + $(libjsonrpc_protocol_COMMON) + + +test_jsonrpc_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -I$(abs_top_srcdir)/tests/include \ + @YAMLCPP_INCLUDES@ + +test_jsonrpc_LDFLAGS = \ + @AM_LDFLAGS@ + +test_jsonrpc_SOURCES = \ + jsonrpc/unit_tests/unit_test_main.cc \ + jsonrpc/unit_tests/test_basic_protocol.cc + +# TODO: Remove libmgmt_p.la + +test_jsonrpc_LDADD = \ + libjsonrpc_protocol.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @YAMLCPP_LIBS@ @HWLOC_LIBS@ + + + +########################################################################################### +# RPC server only. +libjsonrpc_server_COMMON = \ + server/RPCServer.cc \ + server/RPCServer.h \ + server/CommBase.cc \ + server/CommBase.h \ + server/IPCSocketServer.cc \ + server/IPCSocketServer.h \ + config/JsonRPCConfig.cc \ + config/JsonRPCConfig.h + +libjsonrpc_server_la_SOURCES = \ + $(libjsonrpc_server_COMMON) + +test_jsonrpcserver_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + -I$(abs_top_srcdir)/tests/include \ + -I$(abs_top_srcdir)/tests \ + @YAMLCPP_INCLUDES@ + +test_jsonrpcserver_LDFLAGS = \ + @AM_LDFLAGS@ + +test_jsonrpcserver_SOURCES = \ + server/unit_tests/unit_test_main.cc \ + $(shared_rpc_ipc_client_SOURCES) \ + server/unit_tests/test_rpcserver.cc + +test_jsonrpcserver_LDADD = \ + libjsonrpc_protocol.la \ + libjsonrpc_server.la \ + $(top_builddir)/src/tscpp/util/libtscpputil.la \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/lib/records/librecords_p.a \ + $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/src/tscore/libtscore.la \ + $(top_builddir)/mgmt/libmgmt_p.la \ + $(top_builddir)/proxy/shared/libUglyLogStubs.a \ + @YAMLCPP_LIBS@ @HWLOC_LIBS@ + + +########################################################################################### +# Handlers only + +AM_CPPFLAGS += \ + -I$(abs_top_srcdir)/mgmt/ \ + -I$(abs_top_srcdir)/mgmt/utils \ + -I$(abs_top_srcdir)/proxy/http \ + -I$(abs_top_srcdir)/proxy/hdrs \ + -I$(abs_top_srcdir)/proxy/ + +librpcpublichandlers_COMMON = \ + handlers/common/RecordsUtils.cc \ + handlers/common/RecordsUtils.h \ + handlers/config/Configuration.cc \ + handlers/config/Configuration.h \ + handlers/records/Records.cc \ + handlers/records/Records.h \ + handlers/storage/Storage.h \ + handlers/storage/Storage.cc \ + handlers/server/Server.h \ + handlers/server/Server.cc \ + handlers/plugins/Plugins.h \ + handlers/plugins/Plugins.cc \ + handlers/Admin.h + +librpcpublichandlers_la_SOURCES = \ + $(librpcpublichandlers_COMMON) \ + $(shared_overridable_txn_vars_SOURCES) + + +# distclean +# This is a workaround to deal with a newer version of automake, apparently there +# is an issue when including subdir-objects and sources outside of subtree. +# If we include a file from another subdir as is(was) the case of overridable_txn, +# then the distclean will try to remove the file from the original folder as well +# from here. To overcome this issue, we create a file here that will be used +# for building. +# We have also added a proper cleaning for it. +shared_overridable_txn_vars_SOURCES = overridable_txn_vars.cc +nodist_librpcpublichandlers_la_SOURCES = $(shared_overridable_txn_vars_SOURCES) + +shared_rpc_ipc_client_SOURCES = IPCSocketClient.cc + +# This may not be needed. Ok for now. +CLEANDIST = $(shared_overridable_txn_vars_SOURCES) $(shared_rpc_ipc_client_SOURCES) + +clean-local: + rm -f $(shared_overridable_txn_vars_SOURCES) $(shared_rpc_ipc_client_SOURCES) + +distclean-local: + rm -f $(shared_overridable_txn_vars_SOURCES) $(shared_rpc_ipc_client_SOURCES) + +# Build with this file instead of the original one. +$(shared_overridable_txn_vars_SOURCES): + echo "#include \"$(top_builddir)/src/shared/$@\"" >$@ + +$(shared_rpc_ipc_client_SOURCES): + echo "#include \"$(top_builddir)/src/shared/rpc/$@\"" >$@ + +include $(top_srcdir)/build/tidy.mk + +clang-tidy-local: $(DIST_SOURCES) + $(CXX_Clang_Tidy) + diff --git a/mgmt2/rpc/config/JsonRPCConfig.cc b/mgmt2/rpc/config/JsonRPCConfig.cc new file mode 100644 index 00000000000..aabb8e0bc15 --- /dev/null +++ b/mgmt2/rpc/config/JsonRPCConfig.cc @@ -0,0 +1,103 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include + +#include "JsonRPCConfig.h" + +#include "tscore/Diags.h" +#include "tscore/ts_file.h" +#include "records/I_RecCore.h" + +#include "rpc/jsonrpc/JsonRPCManager.h" + +namespace +{ +static constexpr auto RPC_ENABLED_KEY_NAME{"enabled"}; +static constexpr auto COMM_CONFIG_KEY_UNIX{"unix"}; +} // namespace +namespace rpc::config +{ +void +RPCConfig::load(YAML::Node const ¶ms) +{ + try { + if (auto n = params[RPC_ENABLED_KEY_NAME]) { + _rpcEnabled = n.as(); + } else { + Warning("%s not present.", RPC_ENABLED_KEY_NAME); + } + + if (auto n = params[COMM_CONFIG_KEY_UNIX]) { + _commConfig = n; + _selectedCommType = CommType::UNIX; + } else { + Note("%s not present.", COMM_CONFIG_KEY_UNIX); + } + + } catch (YAML::Exception const &ex) { + Warning("We found an issue when reading the parameter: %s . Using defaults", ex.what()); + } +} + +YAML::Node +RPCConfig::get_comm_config_params() const +{ + return _commConfig; +} + +RPCConfig::CommType +RPCConfig::get_comm_type() const +{ + return _selectedCommType; +} + +bool +RPCConfig::is_enabled() const +{ + return _rpcEnabled; +} + +void +RPCConfig::load_from_file(std::string const &filePath) +{ + std::error_code ec; + std::string content{ts::file::load(ts::file::path{filePath}, ec)}; + + if (ec) { + Warning("Cannot open the config file: %s - %s", filePath.c_str(), strerror(ec.value())); + // The rpc will be enabled by default with the default values. + return; + } + + YAML::Node rootNode; + try { + rootNode = YAML::Load(content); + + // read configured parameters. + if (auto rpc = rootNode["rpc"]) { + this->load(rpc); + } + } catch (std::exception const &ex) { + Warning("Something happened parsing the content of %s : %s", filePath.c_str(), ex.what()); + return; + }; +} + +} // namespace rpc::config diff --git a/mgmt2/rpc/config/JsonRPCConfig.h b/mgmt2/rpc/config/JsonRPCConfig.h new file mode 100644 index 00000000000..20e533fc2bb --- /dev/null +++ b/mgmt2/rpc/config/JsonRPCConfig.h @@ -0,0 +1,92 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include + +#include "yaml-cpp/yaml.h" + +namespace rpc::config +{ +/// +/// @brief This class holds and parse all the configuration needed to run the JSONRPC server, communication implementation +/// can use this class to feed their own configuration, though it's not mandatory as their API @see +/// BaseCommInterface::configure uses a YAML::Node this class can be used on top of it and parse the "comm_config" from a +/// wider file. +/// +/// The configuration is divided into two +/// sections: +/// a) General RPC configuration: +/// "communication_type" Defines the communication that should be used by the server. @see Commype +/// "rpc_enabled" Used to set the toggle to disable or enable the whole server. +/// +/// b) Comm specfics Configuration. +/// "comm_config" +/// This is defined by the specific communication, each communication can define and implement their own configuration flags. @see +/// IPCSocketServer::Config for an example +/// +/// Example configuration: +/// +// rpc: +// enabled: true +// unix: +// lock_path_name: /tmp/conf_jsonrp +// sock_path_name: /tmp/conf_jsonrpc.sock +// backlog: 5 +// max_retry_on_transient_errors: 64 +/// +/// All communication section should use a root node name "comm_config", @c RPCConfig will return the full node when requested @see +/// get_comm_config_param, then it's up to the communication implementation to parse it. +/// @note By default Unix Domain Socket will be used as a communication. +/// @note By default the enable/disable toggle will set to Enabled. +/// @note By default a comm_config node will be Null. +class RPCConfig +{ +public: + enum class CommType { UNIX = 1 }; + + RPCConfig() = default; + + /// @brief Get the configured specifics for a particular tansport, all nodes under "comm_config" will be return here. + // it's up to the caller to know how to parse this. + /// @return A YAML::Node that contains the passed configuration. + YAML::Node get_comm_config_params() const; + + /// @brief Function that returns the configured communication type. + /// @return a communication type, CommType::UNIX by default. + CommType get_comm_type() const; + + /// @brief Checks if the server was configured to be enabled or disabled. The server should be explicitly disabled by + /// configuration as it is enabled by default. + /// @return true if enable, false if set disabled. + bool is_enabled() const; + + /// @brief Load the configuration from the content of a file. If the file does not exist, the default values will be used. + void load_from_file(std::string const &filePath); + + /// @brief Load configuration from a YAML::Node. This can be used to expose it as public rrc handler. + void load(YAML::Node const ¶ms); + +private: + YAML::Node _commConfig; //!< "comm_config" section of the configuration file. + CommType _selectedCommType{CommType::UNIX}; //!< The selected (by configuration) communication type. 1 by default. + bool _rpcEnabled{true}; //!< holds the configuration toggle value for "rpc_enable" node. Enabled by default. +}; +} // namespace rpc::config diff --git a/mgmt2/rpc/handlers/common/ErrorUtils.h b/mgmt2/rpc/handlers/common/ErrorUtils.h new file mode 100644 index 00000000000..1b17c9e4c40 --- /dev/null +++ b/mgmt2/rpc/handlers/common/ErrorUtils.h @@ -0,0 +1,65 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once +// TODO: we have to rename and split this file.(Errors and Errata) + +#include +#include + +#include "tscore/Errata.h" +#include "tscore/BufferWriter.h" + +namespace rpc::handlers::errors +{ +// High level handler error codes, each particular handler can be fit into one of the +// following categories. +// enum YourOwnHandlerEnum { +// FOO_ERROR = Codes::SOME_CATEGORY, +// }; +// With this we try to avoid error codes collision. You can also use same error Code for all your +// errors. +enum Codes : unsigned int { + CONFIGURATION = 1, + METRIC = 1000, + RECORD = 2000, + SERVER = 3000, + STORAGE = 4000, + PLUGIN = 5000, + // Add more here. Give enough space between jumps. + GENERIC = 30000 +}; + +static constexpr int ERRATA_DEFAULT_ID{1}; + +template +static inline ts::Errata +make_errata(int code, std::string_view fmt, Args &&... args) +{ + std::string text; + return ts::Errata{}.push(ERRATA_DEFAULT_ID, code, ts::bwprint(text, fmt, std::forward(args)...)); +} + +static inline ts::Errata +make_errata(int code, std::string_view text) +{ + return ts::Errata{}.push(ERRATA_DEFAULT_ID, code, text); +} +} // namespace rpc::handlers::errors \ No newline at end of file diff --git a/mgmt2/rpc/handlers/common/RecordsUtils.cc b/mgmt2/rpc/handlers/common/RecordsUtils.cc new file mode 100644 index 00000000000..516cb49fbda --- /dev/null +++ b/mgmt2/rpc/handlers/common/RecordsUtils.cc @@ -0,0 +1,302 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "RecordsUtils.h" + +#include +#include + +#include "convert.h" +#include "records/P_RecCore.h" +#include "tscore/Tokenizer.h" + +namespace +{ // anonymous namespace + +struct RPCRecordErrorCategory : std::error_category { + const char *name() const noexcept override; + std::string message(int ev) const override; +}; + +const char * +RPCRecordErrorCategory::name() const noexcept +{ + return "rpc_handler_record_error"; +} +std::string +RPCRecordErrorCategory::message(int ev) const +{ + switch (static_cast(ev)) { + case rpc::handlers::errors::RecordError::RECORD_NOT_FOUND: + return {"Record not found."}; + case rpc::handlers::errors::RecordError::RECORD_NOT_CONFIG: + return {"Record is not a configuration type."}; + case rpc::handlers::errors::RecordError::RECORD_NOT_METRIC: + return {"Record is not a metric type."}; + case rpc::handlers::errors::RecordError::INVALID_RECORD_NAME: + return {"Invalid Record Name."}; + case rpc::handlers::errors::RecordError::VALIDITY_CHECK_ERROR: + return {"Validity check failed."}; + case rpc::handlers::errors::RecordError::GENERAL_ERROR: + return {"Error reading the record."}; + case rpc::handlers::errors::RecordError::RECORD_WRITE_ERROR: + return {"We could not write the record."}; + case rpc::handlers::errors::RecordError::REQUESTED_TYPE_MISMATCH: + return {"Found record does not match the requested type"}; + case rpc::handlers::errors::RecordError::INVALID_INCOMING_DATA: + return {"Invalid request data provided"}; + default: + return "Record error error " + std::to_string(ev); + } +} + +const RPCRecordErrorCategory rpcRecordErrorCategory{}; +} // anonymous namespace + +namespace rpc::handlers::errors +{ +std::error_code +make_error_code(rpc::handlers::errors::RecordError e) +{ + return {static_cast(e), rpcRecordErrorCategory}; +} +} // namespace rpc::handlers::errors + +namespace +{ +struct Context { + using CbType = std::function; + YAML::Node yaml; + std::error_code ec; + // regex do not need to set the callback. + CbType checkCb; +}; +} // namespace + +namespace rpc::handlers::records::utils +{ +void static get_record_impl(std::string const &name, Context &ctx) +{ + auto yamlConverter = [](const RecRecord *record, void *data) { + auto &ctx = *static_cast(data); + + if (!record) { + ctx.ec = rpc::handlers::errors::RecordError::RECORD_NOT_FOUND; + return; + } + + if (!ctx.checkCb(record->rec_type, ctx.ec)) { + // error_code in the callback will be set. + return; + } + + try { + ctx.yaml = *record; + } catch (std::exception const &ex) { + ctx.ec = rpc::handlers::errors::RecordError::GENERAL_ERROR; + return; + } + }; + + const auto ret = RecLookupRecord(name.c_str(), yamlConverter, &ctx); + + if (ctx.ec) { + // This will be set if the invocation of the callback inside the context have something to report, in this case + // we give this priority of tracking the error back to the caller. + return; + } + + if (ret != REC_ERR_OKAY) { + ctx.ec = rpc::handlers::errors::RecordError::RECORD_NOT_FOUND; + return; + } +} + +void static get_record_regex_impl(std::string const ®ex, unsigned recType, Context &ctx) +{ + // In this case, where we lookup base on a regex, the only validation we need is base on the recType and the ability to be + // converted to a Yaml Node. + auto yamlConverter = [](const RecRecord *record, void *data) { + auto &ctx = *static_cast(data); + + if (!record) { + return; + } + + YAML::Node recordYaml; + + try { + recordYaml = *record; + } catch (std::exception const &ex) { + ctx.ec = rpc::handlers::errors::RecordError::GENERAL_ERROR; + return; + } + + // we have to append the records to the context one. + ctx.yaml.push_back(recordYaml); + }; + + const auto ret = RecLookupMatchingRecords(recType, regex.c_str(), yamlConverter, &ctx); + // if the passed regex didn't match, it will not report any error. We will only get errors when converting + // the record into yaml(so far). + if (ctx.ec) { + return; + } + + if (ret != REC_ERR_OKAY) { + ctx.ec = rpc::handlers::errors::RecordError::GENERAL_ERROR; + return; + } +} + +// This two functions may look similar but they are not. First runs the validation in a different way. +std::tuple +get_yaml_record_regex(std::string const &name, unsigned recType) +{ + Context ctx; + + // librecord API will use the recType to validate the type. + get_record_regex_impl(name, recType, ctx); + + return {ctx.yaml, ctx.ec}; +} + +std::tuple +get_yaml_record(std::string const &name, ValidateRecType check) +{ + Context ctx; + + // Set the validation callback. + ctx.checkCb = check; + + // librecords will use the callback we provide in the ctx.checkCb to run the validation. + get_record_impl(name, ctx); + + return {ctx.yaml, ctx.ec}; +} + +// Basic functions to help setting a record value properly. All this functionality is originally from WebMgmtUtils. +// TODO: we can work out something different. +namespace +{ // anonymous namespace + bool + recordRegexCheck(const char *pattern, const char *value) + { + pcre *regex; + const char *error; + int erroffset; + + regex = pcre_compile(pattern, 0, &error, &erroffset, nullptr); + if (!regex) { + return false; + } else { + int r = pcre_exec(regex, nullptr, value, strlen(value), 0, 0, nullptr, 0); + + pcre_free(regex); + return (r != -1) ? true : false; + } + + return false; // no-op + } + + bool + recordRangeCheck(const char *pattern, const char *value) + { + char *p = const_cast(pattern); + Tokenizer dashTok("-"); + + if (recordRegexCheck("^[0-9]+$", value)) { + while (*p != '[') { + p++; + } // skip to '[' + if (dashTok.Initialize(++p, COPY_TOKS) == 2) { + int l_limit = atoi(dashTok[0]); + int u_limit = atoi(dashTok[1]); + int val = atoi(value); + if (val >= l_limit && val <= u_limit) { + return true; + } + } + } + return false; + } + + bool + recordIPCheck(const char *pattern, const char *value) + { + // regex_t regex; + // int result; + bool check; + const char *range_pattern = R"(\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\]\\\.\[[0-9]+\-[0-9]+\])"; + const char *ip_pattern = "[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9].[0-9]*[0-9]*[0-9]"; + + Tokenizer dotTok1("."); + Tokenizer dotTok2("."); + + check = true; + if (recordRegexCheck(range_pattern, pattern) && recordRegexCheck(ip_pattern, value)) { + if (dotTok1.Initialize(const_cast(pattern), COPY_TOKS) == 4 && + dotTok2.Initialize(const_cast(value), COPY_TOKS) == 4) { + for (int i = 0; i < 4 && check; i++) { + if (!recordRangeCheck(dotTok1[i], dotTok2[i])) { + check = false; + } + } + if (check) { + return true; + } + } + } else if (strcmp(value, "") == 0) { + return true; + } + return false; + } +} // namespace + +bool +recordValidityCheck(const char *value, RecCheckT checkType, const char *pattern) +{ + switch (checkType) { + case RECC_STR: + if (recordRegexCheck(pattern, value)) { + return true; + } + break; + case RECC_INT: + if (recordRangeCheck(pattern, value)) { + return true; + } + break; + case RECC_IP: + if (recordIPCheck(pattern, value)) { + return true; + } + break; + case RECC_NULL: + // skip checking + return true; + default: + // unknown RecordCheckType... + ; + } + + return false; +} + +} // namespace rpc::handlers::records::utils diff --git a/mgmt2/rpc/handlers/common/RecordsUtils.h b/mgmt2/rpc/handlers/common/RecordsUtils.h new file mode 100644 index 00000000000..0127a58d3e2 --- /dev/null +++ b/mgmt2/rpc/handlers/common/RecordsUtils.h @@ -0,0 +1,102 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include + +#include "rpc/handlers/common/convert.h" +#include "rpc/handlers/common/ErrorUtils.h" + +#include "records/I_RecCore.h" +#include "records/P_RecCore.h" +#include "tscore/Diags.h" +#include "tscore/Errata.h" + +#include + +namespace rpc::handlers::errors +{ +enum class RecordError { + RECORD_NOT_FOUND = Codes::RECORD, + RECORD_NOT_CONFIG, + RECORD_NOT_METRIC, + INVALID_RECORD_NAME, + VALIDITY_CHECK_ERROR, + GENERAL_ERROR, + RECORD_WRITE_ERROR, + REQUESTED_TYPE_MISMATCH, + INVALID_INCOMING_DATA +}; +std::error_code make_error_code(rpc::handlers::errors::RecordError e); +} // namespace rpc::handlers::errors + +namespace std +{ +template <> struct is_error_code_enum : true_type { +}; + +} // namespace std + +namespace rpc::handlers::records::utils +{ +// response request constants +inline const std::string RECORD_NAME_REGEX_KEY{"record_name_regex"}; +inline const std::string RECORD_NAME_KEY{"record_name"}; +inline const std::string RECORD_VALUE_KEY{"record_value"}; +inline const std::string RECORD_TYPES_KEY{"rec_types"}; +inline const std::string RECORD_UPDATE_TYPE_KEY{"update_type"}; +inline const std::string ERROR_CODE_KEY{"code"}; +inline const std::string ERROR_MESSAGE_KEY{"message"}; + +using ValidateRecType = std::function; + +/// +/// @brief Get a Record as a YAML node +/// +/// @param name The record name that is being requested. +/// @param check A function @see ValidateRecType that will be used to validate that the record we want meets the expected +/// criteria. ie: record type. Check @c RecLookupRecord API to see how it's called. +/// @return std::tuple +/// +std::tuple get_yaml_record(std::string const &name, ValidateRecType check); + +/// +/// @brief Get a Record as a YAML node using regex as name. +/// +/// @param regex The regex that will be used to lookup records. +/// @param recType The record type we want to match againts the retrieved records. This could be either a single value or a bitwise +/// value. +/// @return std::tuple +/// +std::tuple get_yaml_record_regex(std::string const ®ex, unsigned recType); + +/// +/// @brief Runs a validity check base on the type and the pattern. +/// +/// @param value Value where the validity check should be applied. +/// @param checkType The type of the value. +/// @param pattern The pattern. +/// @return true if the validity was ok, false otherwise. +/// +bool recordValidityCheck(const char *value, RecCheckT checkType, + const char *pattern); // code originally from WebMgmtUtils + +} // namespace rpc::handlers::records::utils diff --git a/mgmt2/rpc/handlers/common/Utils.h b/mgmt2/rpc/handlers/common/Utils.h new file mode 100644 index 00000000000..309f5155f83 --- /dev/null +++ b/mgmt2/rpc/handlers/common/Utils.h @@ -0,0 +1,41 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include + +namespace rpc::handlers::utils +{ +inline bool +is_true_flag(YAML::Node const &node) +{ + if (node.IsNull()) { + return false; + } + bool isTrue{false}; + try { + auto str = node.as(); + isTrue = str == "yes" || str == "true" || str == "1"; + } catch (YAML::Exception const &ex) { + } + return isTrue; +} +} // namespace rpc::handlers::utils \ No newline at end of file diff --git a/mgmt2/rpc/handlers/common/convert.h b/mgmt2/rpc/handlers/common/convert.h new file mode 100644 index 00000000000..d74ec0bee1e --- /dev/null +++ b/mgmt2/rpc/handlers/common/convert.h @@ -0,0 +1,179 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include + +#include + +#include "records/I_RecCore.h" +#include "records/P_RecCore.h" + +#include "shared/overridable_txn_vars.h" + +#include "RecordsUtils.h" + +/// +/// @brief Namespace to group all the names used for key access to the yaml lookup nodes. +/// +/// +namespace constants_rec +{ +static constexpr auto REC{"record"}; + +static constexpr auto NAME{"record_name"}; +static constexpr auto RECORD_TYPE{"record_type"}; +static constexpr auto RECORD_VERSION{"version"}; +static constexpr auto REGISTERED{"registered"}; +static constexpr auto RSB{"raw_stat_block"}; +static constexpr auto ORDER{"order"}; +static constexpr auto ACCESS_TYPE{"access_type"}; +static constexpr auto UPDATE_STATUS{"update_status"}; +static constexpr auto UPDATE_TYPE{"update_type"}; +static constexpr auto CHECK_TYPE{"checktype"}; +static constexpr auto SOURCE{"source"}; +static constexpr auto CHECK_EXPR{"check_expr"}; +static constexpr auto CLASS{"record_class"}; +static constexpr auto OVERRIDABLE{"overridable"}; +static constexpr auto DATA_TYPE{"data_type"}; +static constexpr auto CURRENT_VALUE{"current_value"}; +static constexpr auto DEFAULT_VALUE{"default_value"}; +static constexpr auto CONFIG_META{"config_meta"}; +static constexpr auto STAT_META{"stat_meta"}; + +static constexpr auto PERSIST_TYPE{"persist_type"}; + +} // namespace constants_rec + +namespace YAML +{ +/// +/// @brief specialize convert template class for RecPersistT +/// +template <> struct convert { + static Node + encode(const RecPersistT &type) + { + return Node{static_cast(type)}; + } +}; + +/// +/// @brief specialize convert template class for RecConfigMeta +/// +template <> struct convert { + static Node + encode(const RecConfigMeta &configMeta) + { + Node node; + // TODO: do we really want each specific encode implementation for each enum type? + node[constants_rec::ACCESS_TYPE] = static_cast(configMeta.access_type); + node[constants_rec::UPDATE_STATUS] = static_cast(configMeta.update_required); + node[constants_rec::UPDATE_TYPE] = static_cast(configMeta.update_type); + node[constants_rec::CHECK_TYPE] = static_cast(configMeta.check_type); + node[constants_rec::SOURCE] = static_cast(configMeta.source); + node[constants_rec::CHECK_EXPR] = configMeta.check_expr ? configMeta.check_expr : "null"; + + return node; + } +}; + +/// +/// @brief specialize convert template class for RecStatMeta +/// +template <> struct convert { + static Node + encode(const RecStatMeta &statMeta) + { + // TODO. just make sure that we know which data should be included here. + Node node; + node[constants_rec::PERSIST_TYPE] = statMeta.persist_type; + return node; + } +}; + +/// +/// @brief specialize convert template class for RecRecord +/// +template <> struct convert { + static Node + encode(const RecRecord &record) + { + Node node; + try { + node[constants_rec::NAME] = record.name ? record.name : "null"; + node[constants_rec::RECORD_TYPE] = static_cast(record.data_type); + node[constants_rec::RECORD_VERSION] = record.version; + node[constants_rec::REGISTERED] = record.registered; + node[constants_rec::RSB] = record.rsb_id; + node[constants_rec::ORDER] = record.order; + + if (REC_TYPE_IS_CONFIG(record.rec_type)) { + node[constants_rec::CONFIG_META] = record.config_meta; + } else if (REC_TYPE_IS_STAT(record.rec_type)) { + node[constants_rec::STAT_META] = record.stat_meta; + } + + node[constants_rec::CLASS] = static_cast(record.rec_type); + + if (record.name) { + const auto it = ts::Overridable_Txn_Vars.find(record.name); + node[constants_rec::OVERRIDABLE] = (it == ts::Overridable_Txn_Vars.end()) ? "false" : "true"; + } + + switch (record.data_type) { + case RECD_INT: + node[constants_rec::DATA_TYPE] = "INT"; + node[constants_rec::CURRENT_VALUE] = record.data.rec_int; + node[constants_rec::DEFAULT_VALUE] = record.data_default.rec_int; + break; + case RECD_FLOAT: + node[constants_rec::DATA_TYPE] = "FLOAT"; + node[constants_rec::CURRENT_VALUE] = record.data.rec_float; + node[constants_rec::DEFAULT_VALUE] = record.data_default.rec_float; + break; + case RECD_STRING: + node[constants_rec::DATA_TYPE] = "STRING"; + node[constants_rec::CURRENT_VALUE] = record.data.rec_string ? record.data.rec_string : "null"; + node[constants_rec::DEFAULT_VALUE] = record.data_default.rec_string ? record.data_default.rec_string : "null"; + break; + case RECD_COUNTER: + node[constants_rec::DATA_TYPE] = "COUNTER"; + node[constants_rec::CURRENT_VALUE] = record.data.rec_counter; + node[constants_rec::DEFAULT_VALUE] = record.data_default.rec_counter; + break; + default: + // this is an error, internal we should flag it + break; + } + } catch (std::exception const &e) { + // we create an empty map node, we do not want to have a null. revisit this. + YAML::NodeType::value kind = YAML::NodeType::Map; + node = YAML::Node{kind}; + } + + Node yrecord; + yrecord[constants_rec::REC] = node; + return yrecord; + } +}; + +} // namespace YAML \ No newline at end of file diff --git a/mgmt2/rpc/handlers/config/Configuration.cc b/mgmt2/rpc/handlers/config/Configuration.cc new file mode 100644 index 00000000000..f3c26ef622b --- /dev/null +++ b/mgmt2/rpc/handlers/config/Configuration.cc @@ -0,0 +1,202 @@ +/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "Configuration.h" + +#include +#include +#include + +#include "tscore/BufferWriter.h" +#include "records/I_RecCore.h" +#include "records/P_RecCore.h" +#include "tscore/Diags.h" + +#include "config/FileManager.h" + +#include "rpc/handlers/common/RecordsUtils.h" + +namespace utils = rpc::handlers::records::utils; + +namespace +{ +/// key value pair from each element passed in the set command. +struct SetRecordCmdInfo { + std::string name; + std::string value; +}; +} // namespace + +namespace YAML +{ +template <> struct convert { + static bool + decode(Node const &node, SetRecordCmdInfo &info) + { + if (!node[utils::RECORD_NAME_KEY] || !node[utils::RECORD_VALUE_KEY]) { + return false; + } + + info.name = node[utils::RECORD_NAME_KEY].as(); + info.value = node[utils::RECORD_VALUE_KEY].as(); + return true; + } +}; +} // namespace YAML + +namespace rpc::handlers::config +{ +namespace err = rpc::handlers::errors; +namespace utils = rpc::handlers::records::utils; + +namespace +{ + template + bool + set_data_type(SetRecordCmdInfo const &info) + { + if constexpr (std::is_same_v) { + T val; + try { + val = std::stof(info.value); + } catch (std::exception const &ex) { + return false; + } + // set the value + if (RecSetRecordFloat(info.name.c_str(), val, REC_SOURCE_DEFAULT) != REC_ERR_OKAY) { + return false; + } + } else if constexpr (std::is_same_v) { + T val; + try { + val = std::stoi(info.value); + } catch (std::exception const &ex) { + return false; + } + // set the value + if (RecSetRecordInt(info.name.c_str(), val, REC_SOURCE_DEFAULT) != REC_ERR_OKAY) { + return false; + } + } else if constexpr (std::is_same_v) { + if (RecSetRecordString(info.name.c_str(), const_cast(info.value.c_str()), REC_SOURCE_DEFAULT) != REC_ERR_OKAY) { + return false; + } + } + + // all set. + return true; + } +} // namespace + +ts::Rv +set_config_records(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + + // we need the type and the udpate type for now. + using LookupContext = std::tuple; + + for (auto const &kv : params) { + SetRecordCmdInfo info; + try { + info = kv.as(); + } catch (YAML::Exception const &ex) { + resp.errata().push({err::RecordError::RECORD_NOT_FOUND}); + continue; + } + + LookupContext recordCtx; + + // Get record info first. TODO: we may just want to get the full record and then send it back as a response. + const auto ret = RecLookupRecord( + info.name.c_str(), + [](const RecRecord *record, void *data) { + auto &[dataType, checkType, pattern, updateType] = *static_cast(data); + if (REC_TYPE_IS_CONFIG(record->rec_type)) { + dataType = record->data_type; + checkType = record->config_meta.check_type; + if (record->config_meta.check_expr) { + pattern = record->config_meta.check_expr; + } + updateType = record->config_meta.update_type; + } + }, + &recordCtx); + + // make sure if exist. If not, we stop it and do not keep forward. + if (ret != REC_ERR_OKAY) { + resp.errata().push({err::RecordError::RECORD_NOT_FOUND}); + continue; + } + + // now set the value. + auto const &[dataType, checkType, pattern, updateType] = recordCtx; + + // run the check only if we have something to check against it. + if (pattern != nullptr && utils::recordValidityCheck(info.value.c_str(), checkType, pattern) == false) { + resp.errata().push({err::RecordError::VALIDITY_CHECK_ERROR}); + continue; + } + + bool set_ok{false}; + switch (dataType) { + case RECD_INT: + case RECD_COUNTER: + set_ok = set_data_type(info); + break; + case RECD_FLOAT: + set_ok = set_data_type(info); + break; + case RECD_STRING: + set_ok = set_data_type(info); + break; + default:; + } + + if (set_ok) { + YAML::Node updatedRecord; + updatedRecord[utils::RECORD_NAME_KEY] = info.name; + updatedRecord[utils::RECORD_UPDATE_TYPE_KEY] = std::to_string(updateType); + resp.result().push_back(updatedRecord); + } else { + resp.errata().push({err::RecordError::GENERAL_ERROR}); + continue; + } + } + + return resp; +} + +ts::Rv +reload_config(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + Debug("RPC", "invoke plugin callbacks"); + // if there is any error, report it back. + if (auto err = FileManager::instance().rereadConfig(); err.size()) { + resp = err; + } + // If any callback was register(TSMgmtUpdateRegister) for config notifications, then it will be eventually notify. + FileManager::instance().invokeConfigPluginCallbacks(); + // save config time. + RecSetRecordInt("proxy.node.config.reconfigure_time", time(nullptr), REC_SOURCE_DEFAULT); + // TODO: we may not need this any more + RecSetRecordInt("proxy.node.config.reconfigure_required", 0, REC_SOURCE_DEFAULT); + + return resp; +} +} // namespace rpc::handlers::config diff --git a/mgmt2/rpc/handlers/config/Configuration.h b/mgmt2/rpc/handlers/config/Configuration.h new file mode 100644 index 00000000000..4a3f2f2ce22 --- /dev/null +++ b/mgmt2/rpc/handlers/config/Configuration.h @@ -0,0 +1,30 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "rpc/jsonrpc/JsonRPCManager.h" + +namespace rpc::handlers::config +{ +ts::Rv set_config_records(std::string_view const &id, YAML::Node const ¶ms); +ts::Rv reload_config(std::string_view const &id, YAML::Node const ¶ms); + +} // namespace rpc::handlers::config \ No newline at end of file diff --git a/mgmt2/rpc/handlers/plugins/Plugins.cc b/mgmt2/rpc/handlers/plugins/Plugins.cc new file mode 100644 index 00000000000..285d3a6d84c --- /dev/null +++ b/mgmt2/rpc/handlers/plugins/Plugins.cc @@ -0,0 +1,85 @@ +/* + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "Plugins.h" +#include "rpc/handlers/common/ErrorUtils.h" + +#include "InkAPIInternal.h" + +namespace +{ +const std::string PLUGIN_TAG_KEY{"tag"}; +const std::string PLUGIN_DATA_KEY{"data"}; +static constexpr auto logTag{"rpc.plugins"}; + +struct PluginMsgInfo { + std::string data; + std::string tag; +}; +} // namespace +namespace YAML +{ +template <> struct convert { + static bool + decode(Node const &node, PluginMsgInfo &msg) + { + if (!node[PLUGIN_TAG_KEY] || !node[PLUGIN_DATA_KEY]) { + return false; + } + msg.tag = node[PLUGIN_TAG_KEY].as(); + msg.data = node[PLUGIN_DATA_KEY].as(); + + return true; + } +}; +} // namespace YAML + +namespace rpc::handlers::plugins +{ +namespace err = rpc::handlers::errors; + +ts::Rv +plugin_send_basic_msg(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + try { + // keep the data. + PluginMsgInfo info = params.as(); + + TSPluginMsg msg; + msg.tag = info.tag.c_str(); + msg.data = info.data.data(); + msg.data_size = info.data.size(); + + APIHook *hook = lifecycle_hooks->get(TS_LIFECYCLE_MSG_HOOK); + + while (hook) { + TSPluginMsg tmp(msg); // Just to make sure plugins don't mess this up for others. + hook->invoke(TS_EVENT_LIFECYCLE_MSG, &tmp); + hook = hook->next(); + } + } catch (std::exception const &ex) { + Debug(logTag, "Invalid params %s", ex.what()); + resp = err::make_errata(err::Codes::PLUGIN, "Error parsing the incoming data: {}", ex.what()); + } + + return resp; +} +} // namespace rpc::handlers::plugins diff --git a/mgmt2/rpc/handlers/plugins/Plugins.h b/mgmt2/rpc/handlers/plugins/Plugins.h new file mode 100644 index 00000000000..e82c2d082d1 --- /dev/null +++ b/mgmt2/rpc/handlers/plugins/Plugins.h @@ -0,0 +1,28 @@ +/* + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "rpc/jsonrpc/JsonRPCManager.h" + +namespace rpc::handlers::plugins +{ +ts::Rv plugin_send_basic_msg(std::string_view const &id, YAML::Node const ¶ms); +} // namespace rpc::handlers::plugins diff --git a/mgmt2/rpc/handlers/records/Records.cc b/mgmt2/rpc/handlers/records/Records.cc new file mode 100644 index 00000000000..5264594b957 --- /dev/null +++ b/mgmt2/rpc/handlers/records/Records.cc @@ -0,0 +1,299 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "Records.h" + +#include +#include +#include + +#include "handlers/common/RecordsUtils.h" +// #include "common/yaml/codecs.h" +/// +/// @brief Local definitions to map requests and responsponses(not fully supported yet) to custom structures. All this definitions +/// are used during decoding and encoding of the RPC requests. +/// +namespace utils = rpc::handlers::records::utils; +namespace +{ +const std::string RECORD_LIST_KEY{"recordList"}; +const std::string ERROR_LIST_KEY{"errorList"}; +/// @brief This class maps the incoming rpc record request in general. This should be used to handle all the data around the +/// record requests. +/// +struct RequestRecordElement { + std::string recName; //!< Incoming record name, this is used for a regex as well. + bool isRegex{false}; //!< set to true if the lookup should be done by using a regex instead a full name. + std::vector recTypes; //!< incoming rec_types + + /// @brief test if the requests is intended to use a regex. + bool + is_regex_req() const + { + return isRegex; + } +}; + +/// +/// @brief Class used to wrap non recoverable lookup errors during a lookup call, this errors will then, be pushed inside the +/// errorList nodes. +/// +struct ErrorInfo { + ErrorInfo(int _code) + : code(_code) //!< recordName and message should be set manually, unless it's created from an @c std::error_code + { + } + // Build it from a @c std::error_code + ErrorInfo(std::error_code ec) : code(ec.value()), message(ec.message()) {} + int code; //!< Error code, it's not mandatory to include the message if we have the code instead. The message can be found in + // the documentation if the code is returned. + std::string recordName; //!< record name may not be available in some cases, instead we can use a message. Message will use + // their own field name. + std::string message; +}; + +} // namespace +// using namespace rpc::codec::types; +// YAML Converter for the incoming record request @see RequestRecordElement. Make sure you protect this by try/catch. We may get +// some invalid types. +namespace YAML +{ +// using namespace rpc::codec::types; +template <> struct convert { + static bool + decode(Node const &node, RequestRecordElement &info) + { + if (!node[utils::RECORD_NAME_REGEX_KEY] && !node[utils::RECORD_NAME_KEY]) { + // if we don't get any specific name, seems a bit risky to send them all back. At least some * would be nice. + return false; + } + + // if both are provided, we can't proceed. + if (node[utils::RECORD_NAME_REGEX_KEY] && node[utils::RECORD_NAME_KEY]) { + return false; + } + + // TODO: Add "type" paramater to just say, `config`, `metric`. May be handier. + + if (auto n = node[utils::RECORD_TYPES_KEY]) { + // if it's empty should be ok, will get all of them. + if (n && n.IsSequence()) { + auto const &passedTypes = n.as>(); + for (auto rt : passedTypes) { + switch (rt) { + case RECT_NULL: + case RECT_CONFIG: + case RECT_PROCESS: + case RECT_NODE: + case RECT_LOCAL: + case RECT_PLUGIN: + case RECT_ALL: + info.recTypes.push_back(rt); + break; + default: + // this field allows 1x1 match from the enum. + // we may accept the bitwise being passed as param in the future. + return false; + } + } + } + } + + if (auto n = node[utils::RECORD_NAME_REGEX_KEY]) { + info.recName = n.as(); + info.isRegex = true; + } else { + info.recName = node[utils::RECORD_NAME_KEY].as(); + info.isRegex = false; + } + + return true; + } +}; + +template <> struct convert { + static Node + encode(ErrorInfo const &errorInfo) + { + Node errorInfoNode; + errorInfoNode[utils::ERROR_CODE_KEY] = errorInfo.code; + if (!errorInfo.message.empty()) { + errorInfoNode[utils::ERROR_MESSAGE_KEY] = errorInfo.message; + } + if (!errorInfo.recordName.empty()) { + errorInfoNode[utils::RECORD_NAME_KEY] = errorInfo.recordName; + } + + return errorInfoNode; + } +}; +} // namespace YAML + +namespace +{ +static unsigned +bitwise(std::vector const &values) +{ + unsigned recType = RECT_ALL; + if (values.size() > 0) { + auto it = std::begin(values); + + recType = *it; + ++it; + for (; it != std::end(values); ++it) { + recType |= *it; + } + } + + return recType; +} + +namespace utils = rpc::handlers::records::utils; +namespace err = rpc::handlers::errors; + +static auto +find_record_by_name(RequestRecordElement const &element) +{ + unsigned recType = bitwise(element.recTypes); + + return utils::get_yaml_record(element.recName, [recType](RecT rec_type, std::error_code &ec) { + if ((recType & rec_type) == 0) { + ec = err::RecordError::REQUESTED_TYPE_MISMATCH; + return false; + } + return true; + }); +} + +static auto +find_records_by_regex(RequestRecordElement const &element) +{ + unsigned recType = bitwise(element.recTypes); + + return utils::get_yaml_record_regex(element.recName, recType); +} + +static auto +find_records(RequestRecordElement const &element) +{ + if (element.is_regex_req()) { + return find_records_by_regex(element); + } + return find_record_by_name(element); +} + +} // namespace + +namespace rpc::handlers::records +{ +namespace err = rpc::handlers::errors; + +ts::Rv +lookup_records(std::string_view const &id, YAML::Node const ¶ms) +{ + // TODO: we may want to deal with our own object instead of a node here. + YAML::Node recordList{YAML::NodeType::Sequence}, errorList{YAML::NodeType::Sequence}; + + for (auto &&node : params) { + RequestRecordElement recordElement; + try { + recordElement = node.as(); + } catch (YAML::Exception const &) { + errorList.push_back(ErrorInfo{{err::RecordError::INVALID_INCOMING_DATA}}); + continue; + } + + auto &&[recordNode, error] = find_records(recordElement); + + if (error) { + ErrorInfo ei{error}; + ei.recordName = recordElement.recName; + + errorList.push_back(ei); + continue; + } + + // Regex lookup, will get us back a sequence, of nodes. In this case we will add them one by 1 so we get a list of objects and + // not a sequence inside the result object, this can be changed ofc but for now this is fine. + if (recordNode.IsSequence()) { + for (auto &&n : recordNode) { + recordList.push_back(std::move(n)); + } + } else if (recordNode.IsMap()) { + recordList.push_back(std::move(recordNode)); + } + } + + YAML::Node resp; + // Even if the records/errors are an empty list, we want them in the response. + resp[RECORD_LIST_KEY] = recordList; + resp[ERROR_LIST_KEY] = errorList; + return resp; +} + +ts::Rv +clear_all_metrics_records(std::string_view const &id, YAML::Node const ¶ms) +{ + using namespace rpc::handlers::records::utils; + ts::Rv resp; + if (RecResetStatRecord(RECT_NULL, true) != REC_ERR_OKAY) { + return ts::Errata{rpc::handlers::errors::RecordError::RECORD_WRITE_ERROR}; + } + + return resp; +} + +ts::Rv +clear_metrics_records(std::string_view const &id, YAML::Node const ¶ms) +{ + using namespace rpc::handlers::records::utils; + + YAML::Node resp, errorList; + + for (auto &&element : params) { + RequestRecordElement recordElement; + try { + recordElement = element.as(); + } catch (YAML::Exception const &) { + errorList.push_back(ErrorInfo{{err::RecordError::INVALID_INCOMING_DATA}}); + continue; + } + + if (!recordElement.recName.empty()) { + if (RecResetStatRecord(recordElement.recName.c_str()) != REC_ERR_OKAY) { + // This could be due the fact that the record is already cleared or the metric does not have any significant + // value. + ErrorInfo ei{err::RecordError::RECORD_WRITE_ERROR}; + ei.recordName = recordElement.recName; + errorList.push_back(ei); + } + } else { + errorList.push_back(ErrorInfo{{err::RecordError::INVALID_INCOMING_DATA}}); + continue; + } + } + + if (!errorList.IsNull()) { + resp[ERROR_LIST_KEY] = errorList; + } + + return resp; +} + +} // namespace rpc::handlers::records diff --git a/mgmt2/rpc/handlers/records/Records.h b/mgmt2/rpc/handlers/records/Records.h new file mode 100644 index 00000000000..40fd9696b4b --- /dev/null +++ b/mgmt2/rpc/handlers/records/Records.h @@ -0,0 +1,60 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include "tscore/Errata.h" + +namespace rpc::handlers::records +{ +/// +/// @brief Record lookups. This is a RPC function handler that retrieves a YAML::Node which will contain the result of a records +/// lookup. @see RecRecord. +/// Incoming parameter is expected to be a sequence, params will be converted to a @see RequestRecordElement +/// and the response will be a YAML node that contains the findings base on the query type. @see RequestRecordElement recTypes will +/// lead the search. +/// @param id JSONRPC client's id. +/// @param params lookup_records query structure. +/// @return ts::Rv A node or an error. If ok, the node will hold the @c "recordList" sequence with the findings. In case +/// of any missed search, ie: when paseed types didn't match the found record(s), the particular error will be added to the @c +/// "errorList" field. +/// +ts::Rv lookup_records(std::string_view const &id, YAML::Node const ¶ms); + +/// +/// @brief A RPC function handler that clear all the metrics. +/// +/// @param id JSONRPC client's id. +/// @param params Nothing, this will be ignored. +/// @return ts::Rv An empty YAML::Node or the proper Errata with the tracked error. +/// +ts::Rv clear_all_metrics_records(std::string_view const &id, YAML::Node const &); + +/// +/// @brief A RPC function handler that clear a specific set of metrics. +/// The @c "errorList" field will only be set if there is any error cleaning a specific metric. +/// @param id JSONRPC client's id. +/// @param params A list of records to update. @see RequestRecordElement +/// @return ts::Rv A YAML::Node or the proper Errata with the tracked error. +/// +ts::Rv clear_metrics_records(std::string_view const &id, YAML::Node const ¶ms); +} // namespace rpc::handlers::records diff --git a/mgmt2/rpc/handlers/server/Server.cc b/mgmt2/rpc/handlers/server/Server.cc new file mode 100644 index 00000000000..982bdba8fc0 --- /dev/null +++ b/mgmt2/rpc/handlers/server/Server.cc @@ -0,0 +1,120 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "Server.h" + +#include "P_Cache.h" +#include +#include "rpc/handlers/common/ErrorUtils.h" +#include "rpc/handlers/common/Utils.h" + +namespace rpc::handlers::server +{ +namespace field_names +{ + static constexpr auto NEW_CONNECTIONS{"no_new_connections"}; +} // namespace field_names + +struct DrainInfo { + bool noNewConnections{false}; +}; +} // namespace rpc::handlers::server +namespace YAML +{ +template <> struct convert { + static bool + decode(const Node &node, rpc::handlers::server::DrainInfo &rhs) + { + namespace field = rpc::handlers::server::field_names; + namespace utils = rpc::handlers::utils; + if (!node.IsMap()) { + return false; + } + // optional + if (auto n = node[field::NEW_CONNECTIONS]; utils::is_true_flag(n)) { + rhs.noNewConnections = true; + } + return true; + } +}; +} // namespace YAML + +namespace rpc::handlers::server +{ +namespace err = rpc::handlers::errors; + +static bool +is_server_draining() +{ + RecInt draining = 0; + if (RecGetRecordInt("proxy.node.config.draining", &draining) != REC_ERR_OKAY) { + return false; + } + return draining != 0; +} + +static void inline set_server_drain(bool drain) +{ + TSSystemState::drain(drain); + RecSetRecordInt("proxy.node.config.draining", TSSystemState::is_draining() ? 1 : 0, REC_SOURCE_DEFAULT); +} + +ts::Rv +server_start_drain(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + try { + if (!params.IsNull()) { + DrainInfo di = params.as(); + Debug("rpc.server", "draining - No new connections %s", (di.noNewConnections ? "yes" : "no")); + // TODO: no new connections flag - implement with the right metric / unimplemented in traffic_ctl + } + + if (!is_server_draining()) { + set_server_drain(true); + } else { + resp.errata().push(err::make_errata(err::Codes::SERVER, "Server already draining.")); + } + } catch (std::exception const &ex) { + Debug("rpc.handler.server", "Got an error DrainInfo decoding: %s", ex.what()); + resp.errata().push(err::make_errata(err::Codes::SERVER, "Error found during server drain: {}", ex.what())); + } + return resp; +} + +ts::Rv +server_stop_drain(std::string_view const &id, [[maybe_unused]] YAML::Node const ¶ms) +{ + ts::Rv resp; + if (is_server_draining()) { + set_server_drain(false); + } else { + resp.errata().push(err::make_errata(err::Codes::SERVER, "Server is not draining.")); + } + + return resp; +} + +void +server_shutdown(YAML::Node const &) +{ + sync_cache_dir_on_shutdown(); +} +} // namespace rpc::handlers::server diff --git a/mgmt2/rpc/handlers/server/Server.h b/mgmt2/rpc/handlers/server/Server.h new file mode 100644 index 00000000000..b71d87a1fcd --- /dev/null +++ b/mgmt2/rpc/handlers/server/Server.h @@ -0,0 +1,30 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "rpc/jsonrpc/JsonRPCManager.h" + +namespace rpc::handlers::server +{ +ts::Rv server_start_drain(std::string_view const &id, YAML::Node const ¶ms); +ts::Rv server_stop_drain(std::string_view const &id, YAML::Node const &); +void server_shutdown(YAML::Node const &); +} // namespace rpc::handlers::server diff --git a/mgmt2/rpc/handlers/storage/Storage.cc b/mgmt2/rpc/handlers/storage/Storage.cc new file mode 100644 index 00000000000..34d6fb25583 --- /dev/null +++ b/mgmt2/rpc/handlers/storage/Storage.cc @@ -0,0 +1,102 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "Storage.h" +#include "tscore/BufferWriter.h" +#include "rpc/handlers/common/ErrorUtils.h" +#include "P_Cache.h" + +namespace rpc::handlers::storage::field_names +{ +static constexpr auto PATH{"path"}; +static constexpr auto STATUS{"status"}; +static constexpr auto ERRORS{"error_count"}; +} // namespace rpc::handlers::storage::field_names + +namespace YAML +{ +template <> struct convert { + static Node + encode(CacheDisk const &cdisk) + { + namespace field = rpc::handlers::storage::field_names; + Node node; + try { + node[field::PATH] = cdisk.path; + node[field::STATUS] = (cdisk.online ? "online" : "offline"); + node[field::ERRORS] = cdisk.num_errors; + } catch (std::exception const &e) { + return node; + } + + Node cacheDiskNode; + cacheDiskNode["cachedisk"] = node; + return cacheDiskNode; + } +}; + +} // namespace YAML + +namespace rpc::handlers::storage +{ +namespace err = rpc::handlers::errors; + +ts::Rv +set_storage_offline(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + + for (auto &&it : params) { + std::string device = it.as(); + CacheDisk *d = cacheProcessor.find_by_path(device.c_str(), (device.size())); + + if (d) { + Debug("rpc.server", "Marking %s offline", device.c_str()); + + YAML::Node n; + auto ret = cacheProcessor.mark_storage_offline(d, /* admin */ true); + n["path"] = device; + n["has_online_storage_left"] = ret ? "true" : "false"; + resp.result().push_back(std::move(n)); + } else { + resp.errata().push(err::make_errata(err::Codes::STORAGE, "Passed device:'{}' does not match any defined storage", device)); + } + } + return resp; +} + +ts::Rv +get_storage_status(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + + for (auto &&it : params) { + std::string device = it.as(); + CacheDisk *d = cacheProcessor.find_by_path(device.c_str(), static_cast(device.size())); + + if (d) { + resp.result().push_back(*d); + } else { + resp.errata().push(err::make_errata(err::Codes::STORAGE, "Passed device:'{}' does not match any defined storage", device)); + } + } + return resp; +} +} // namespace rpc::handlers::storage diff --git a/mgmt2/rpc/handlers/storage/Storage.h b/mgmt2/rpc/handlers/storage/Storage.h new file mode 100644 index 00000000000..1b9eba30763 --- /dev/null +++ b/mgmt2/rpc/handlers/storage/Storage.h @@ -0,0 +1,29 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include "rpc/jsonrpc/JsonRPCManager.h" + +namespace rpc::handlers::storage +{ +ts::Rv set_storage_offline(std::string_view const &id, YAML::Node const ¶ms); +ts::Rv get_storage_status(std::string_view const &id, YAML::Node const ¶ms); +} // namespace rpc::handlers::storage diff --git a/mgmt2/rpc/jsonrpc/Defs.h b/mgmt2/rpc/jsonrpc/Defs.h new file mode 100644 index 00000000000..9c321f6f1bf --- /dev/null +++ b/mgmt2/rpc/jsonrpc/Defs.h @@ -0,0 +1,182 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include +#include + +#include + +#include "rpc/jsonrpc/error/RPCError.h" + +// This file contains all the internal types used by the RPC engine to deal with all the messages +// While we use yamlcpp for parsing, internally we model the request/response on our wrappers (RPCRequest, RPCResponse) +namespace rpc::specs +{ +const std::string JSONRPC_VERSION{"2.0"}; + +/// @brief This class encapsulate the registered handler call data. +/// It contains the YAML::Node that will contain the response of a call and if any error, will also encapsulate the error from the +/// call. +/// @see MethodHandler +class RPCHandlerResponse +{ +public: + YAML::Node result; //!< The response from the registered handler. + ts::Errata errata; //!< The error response from the registered handler. +}; + +struct RPCResponseInfo { + RPCResponseInfo(std::optional const &id_) : id(id_) {} + RPCResponseInfo() = default; + + RPCHandlerResponse callResult; + std::error_code rpcError; + std::optional id; +}; + +/// +/// @brief Class that contains all the request information. +/// This class maps the jsonrpc protocol for a request. It can be used for Methods and Notifications. +/// Notifications will not use the id, this is the main reason why is a std::optional<>. +/// +struct RPCRequestInfo { + RPCRequestInfo() = default; + RPCRequestInfo(std::string const &version, std::string const &mid) : jsonrpc(version), id(mid) {} + std::string jsonrpc; //!< JsonRPC version ( we only allow 2.0 ). @see yamlcpp_json_decoder + std::string method; //!< incoming method name. + std::optional id; //!< incoming request if (only used for method calls.) + YAML::Node params; //!< incoming parameter structure. + + /// Convenience functions that checks for the type of request. If contains id then it should be handle as method call, otherwise + /// will be a notification. + bool + is_notification() const + { + return !id.has_value(); + } + bool + is_method() const + { + return !this->is_notification(); + } +}; + +template class RPCMessage; +using RPCRequest = RPCMessage>; +using RPCResponse = RPCMessage; + +/// +/// @brief Class that reprecent a RPC message, it could be either the request or the response. +/// Requests @see RPCRequest are represented by a vector of pairs, which contains, the request @see RPCRequestInfo and an associated +/// error_code in case that the request fails. The main reason of this to be a pair is that as per the protocol specs we need to +/// respond every message(if it's a method), so for cases like a batch request where you may have some of the request fail, then the +/// error will be attached to the response. The order is not important +/// +/// Responses @see RPCResponse models pretty much the same structure except that there is no error associated with it. The @see +/// RPCResponseInfo could be the error. +/// +/// @tparam Message The type of the RPCMessage, either a pair of @see RPCRequestInfo, std::error_code. Or @see RPCResponseInfo +/// +template class RPCMessage +{ + static_assert(!std::is_same_v || !std::is_same_v, + "Ups, only RPCRequest or RPCResponse"); + + using MessageList = std::vector; + /// @brief to keep track of internal message data. + struct Metadata { + Metadata() = default; + Metadata(bool batch) : isBatch(batch) {} + /// @brief Used to mark the incoming request and response base on the former's format. If we want to respond with the same + /// format as the incoming request, then this should be used. base on the request's format. + enum class MsgFormat { + UNKNOWN = 0, //!< Default value. + JSON, //!< If messages arrives as JSON + YAML //!< If messages arrives as YAML + }; + MsgFormat msgFormat{MsgFormat::UNKNOWN}; + bool isBatch{false}; + }; + +public: + RPCMessage() {} + RPCMessage(bool isBatch) : _metadata{isBatch} {} + /// + /// @brief + /// + /// @tparam Msg + /// @param msg + /// + template + void + add_message(Msg &&msg) + { + _elements.push_back(std::forward(msg)); + } + + const MessageList & + get_messages() const + { + return _elements; + } + + bool + is_notification() const noexcept + { + return _elements.size() == 0; + } + + bool + is_batch() const noexcept + { + return _metadata.isBatch; + } + + void + is_batch(bool isBatch) noexcept + { + _metadata.isBatch = isBatch; + } + void + reserve(std::size_t size) + { + _elements.reserve(size); + } + + bool + is_json_format() const noexcept + { + return _metadata.msgFormat == Metadata::MsgFormat::JSON; + } + + bool + is_yaml_format() const noexcept + { + return _metadata.msgFormat == Metadata::MsgFormat::YAML; + } + +private: + MessageList _elements; + Metadata _metadata; +}; + +} // namespace rpc::specs diff --git a/mgmt2/rpc/jsonrpc/JsonRPC.h b/mgmt2/rpc/jsonrpc/JsonRPC.h new file mode 100644 index 00000000000..c885bc40dfa --- /dev/null +++ b/mgmt2/rpc/jsonrpc/JsonRPC.h @@ -0,0 +1,49 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include "JsonRPCManager.h" +#include "rpc/handlers/common/ErrorUtils.h" + +namespace rpc +{ +/// Generic and global JSONRPC service provider info object. It's recommended to use this object when registring your new handler +/// into the rpc system IF the implementor wants the handler to be listed as ATS's handler. +extern RPCRegistryInfo core_ats_rpc_service_provider_handle; +// ----------------------------------------------------------------------------- +/// Set of convenience API function. Users can avoid having to name the whole rps::JsonRPCManager::instance() to use the object. + +/// @see JsonRPCManager::add_method_handler for details +template +inline bool +add_method_handler(const std::string &name, Func &&call, const RPCRegistryInfo *info = nullptr) +{ + return JsonRPCManager::instance().add_method_handler(name, std::forward(call), info); +} + +/// @see JsonRPCManager::add_notification_handler for details +template +inline bool +add_notification_handler(const std::string &name, Func &&call, const RPCRegistryInfo *info = nullptr) +{ + return JsonRPCManager::instance().add_notification_handler(name, std::forward(call), info); +} + +} // namespace rpc diff --git a/mgmt2/rpc/jsonrpc/JsonRPCManager.cc b/mgmt2/rpc/jsonrpc/JsonRPCManager.cc new file mode 100644 index 00000000000..a85745728c9 --- /dev/null +++ b/mgmt2/rpc/jsonrpc/JsonRPCManager.cc @@ -0,0 +1,361 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "JsonRPCManager.h" + +#include +#include +#include +#include +#include + +#include "json/YAMLCodec.h" + +namespace +{ +// RPC service info constants. +const std::string RPC_SERVICE_METHOD_STR{"method"}; +const std::string RPC_SERVICE_NOTIFICATION_STR{"notification"}; +const std::string RPC_SERVICE_NAME_KEY{"name"}; +const std::string RPC_SERVICE_TYPE_KEY{"type"}; +const std::string RPC_SERVICE_PROVIDER_KEY{"provider"}; +const std::string RPC_SERVICE_SCHEMA_KEY{"schema"}; +const std::string RPC_SERVICE_METHODS_KEY{"methods"}; +const std::string RPC_SERVICE_NOTIFICATIONS_KEY{"notifications"}; +const std::string RPC_SERVICE_N_A_STR{"N/A"}; +} // namespace + +namespace rpc +{ +RPCRegistryInfo core_ats_rpc_service_provider_handle = { + "Traffic Server JSONRPC 2.0 API" // Provider's description +}; + +// plugin rpc handling variables. +std::mutex g_rpcHandlingMutex; +std::condition_variable g_rpcHandlingCompletion; +ts::Rv g_rpcHandlerResponseData; +bool g_rpcHandlerProccessingCompleted{false}; + +// jsonrpc log tag. +static constexpr auto logTag = "rpc"; +static constexpr auto logTagMsg = "rpc.msg"; + +// --- Dispatcher +JsonRPCManager::Dispatcher::Dispatcher() +{ + register_service_descriptor_handler(); +} +void +JsonRPCManager::Dispatcher::register_service_descriptor_handler() +{ + // TODO: revisit this. + if (!this->add_handler( + "show_registered_handlers", + [this](std::string_view const &id, const YAML::Node &req) -> ts::Rv { + return show_registered_handlers(id, req); + }, + &core_ats_rpc_service_provider_handle)) { + Warning("Handler already registered."); + } + + if (!this->add_handler( + "get_service_descriptor", + [this](std::string_view const &id, const YAML::Node &req) -> ts::Rv { return get_service_descriptor(id, req); }, + &core_ats_rpc_service_provider_handle)) { + Warning("Handler already registered."); + } +} + +JsonRPCManager::Dispatcher::response_type +JsonRPCManager::Dispatcher::dispatch(specs::RPCRequestInfo const &request) const +{ + std::error_code ec; + auto const &handler = find_handler(request, ec); + + if (ec) { + return {std::nullopt, ec}; + } + + if (request.is_notification()) { + return invoke_notification_handler(handler, request); + } + + // just a method call. + return invoke_method_handler(handler, request); +} + +JsonRPCManager::Dispatcher::InternalHandler const & +JsonRPCManager::Dispatcher::find_handler(specs::RPCRequestInfo const &request, std::error_code &ec) const +{ + static InternalHandler no_handler{}; + + std::lock_guard lock(_mutex); + + auto search = _handlers.find(request.method); + + if (search == std::end(_handlers)) { + // no more checks, no handler either notification or method + ec = error::RPCErrorCode::METHOD_NOT_FOUND; + return no_handler; + } + + // we need to make sure we the request is valid against the internal handler. + if ((request.is_method() && search->second.is_method()) || (request.is_notification() && !search->second.is_method())) { + return search->second; + } + + ec = error::RPCErrorCode::INVALID_REQUEST; + return no_handler; +} + +JsonRPCManager::Dispatcher::response_type +JsonRPCManager::Dispatcher::invoke_method_handler(JsonRPCManager::Dispatcher::InternalHandler const &handler, + specs::RPCRequestInfo const &request) const +{ + specs::RPCResponseInfo response{request.id}; + + try { + auto const &rv = handler.invoke(request); + + if (rv.isOK()) { + response.callResult.result = rv.result(); + } else { + // if we have some errors to log, then include it. + response.callResult.errata = rv.errata(); + } + } catch (std::exception const &e) { + Debug(logTag, "Oops, something happened during the callback invocation: %s", e.what()); + return {std::nullopt, error::RPCErrorCode::ExecutionError}; + } + + return {response, {}}; +} + +JsonRPCManager::Dispatcher::response_type +JsonRPCManager::Dispatcher::invoke_notification_handler(JsonRPCManager::Dispatcher::InternalHandler const &handler, + specs::RPCRequestInfo const ¬ification) const +{ + try { + handler.invoke(notification); + } catch (std::exception const &e) { + Debug(logTag, "Oops, something happened during the callback(notification) invocation: %s", e.what()); + // it's a notification so we do not care much. + } + + return response_type{}; +} + +bool +JsonRPCManager::Dispatcher::remove_handler(std::string const &name) +{ + std::lock_guard lock(_mutex); + auto foundIt = std::find_if(std::begin(_handlers), std::end(_handlers), [&](auto const &p) { return p.first == name; }); + if (foundIt != std::end(_handlers)) { + _handlers.erase(foundIt); + return true; + } + + return false; +} +// --- JsonRPCManager +bool +JsonRPCManager::remove_handler(std::string const &name) +{ + return _dispatcher.remove_handler(name); +} + +static inline specs::RPCResponseInfo +make_error_response(specs::RPCRequestInfo const &req, std::error_code const &ec) +{ + specs::RPCResponseInfo resp; + + // we may have been able to collect the id, if so, use it. + if (req.id) { + resp.id = req.id; + } + + resp.rpcError = ec; + + return resp; +} + +static inline specs::RPCResponseInfo +make_error_response(std::error_code const &ec) +{ + specs::RPCResponseInfo resp; + + resp.rpcError = ec; + return resp; +} + +std::optional +JsonRPCManager::handle_call(std::string const &request) +{ + Debug(logTagMsg, "--> JSONRPC request\n'%s'", request.c_str()); + + std::error_code ec; + try { + // let's decode all the incoming messages into our own types. + specs::RPCRequest const &msg = Decoder::decode(request, ec); + + // If any error happened within the request, they will be kept inside each + // particular request, as they would need to be converted back in a proper error response. + if (ec) { + auto response = make_error_response(ec); + return Encoder::encode(response); + } + + specs::RPCResponse response{msg.is_batch()}; + for (auto const &[req, decode_error] : msg.get_messages()) { + // As per jsonrpc specs we do care about invalid messages as long as they are well-formed, our decode logic will make their + // best to build up a request, if any errors were detected during decoding, we will save the error and make it part of the + // RPCRequest elements for further use. + if (!decode_error) { + // request seems ok and ready to be dispatched. The dispatcher will tell us if the method exist and if so, it will dispatch + // the call and gives us back the response. + auto &&[encodedResponse, ec] = _dispatcher.dispatch(req); + + // On any error, ec will have a value + if (!ec) { + // we only get valid responses if it was a method request, not + // for notifications. + if (encodedResponse) { + response.add_message(*encodedResponse); + } + } else { + // get an error response, we may have the id, so let's try to use it. + response.add_message(make_error_response(req, ec)); + } + + } else { + // If the request was marked as an error(decode error), we still need to send the error back, so we save it. + response.add_message(make_error_response(req, decode_error)); + } + } + + // We will not have a response for notification(s); This could be a batch of notifications only. + std::optional resp; + + if (!response.is_notification()) { + resp = Encoder::encode(response); + Debug(logTagMsg, "<-- JSONRPC Response\n '%s'", (*resp).c_str()); + } + return resp; + } catch (std::exception const &ex) { + ec = error::RPCErrorCode::INTERNAL_ERROR; + } + + return {Encoder::encode(make_error_response(ec))}; +} + +// ---------------------------- InternalHandler --------------------------------- +inline ts::Rv +JsonRPCManager::Dispatcher::InternalHandler::invoke(specs::RPCRequestInfo const &request) const +{ + ts::Rv ret; + std::visit(ts::meta::overloaded{[](std::monostate) -> void { /* no op */ }, + [&request](Notification const &handler) -> void { + // Notification handler call. Ignore response, there is no completion cv check in here basically + // because we do not deal with any response, the callee can just re-schedule the work if + // needed. We fire and forget. + handler.cb(request.params); + }, + [&ret, &request](Method const &handler) -> void { + // Regular Method Handler call, No cond variable check here, this should have not be created by + // a plugin. + ret = handler.cb(*request.id, request.params); + }, + [&ret, &request](PluginMethod const &handler) -> void { + // We call the method handler, we'll lock and wait till the condition_variable + // gets set on the other side. The handler may return immediately with no response being set. + // cond var will give us green to proceed. + handler.cb(*request.id, request.params); + std::unique_lock lock(g_rpcHandlingMutex); + g_rpcHandlingCompletion.wait(lock, []() { return g_rpcHandlerProccessingCompleted; }); + g_rpcHandlerProccessingCompleted = false; + // seems to be done, set the response. As the response data is a ts::Rv this will handle both, + // error and non error cases. + ret = g_rpcHandlerResponseData; + // clean up the shared data. + g_rpcHandlerResponseData.clear(); + lock.unlock(); + }}, + this->_func); + return ret; +} + +inline bool +JsonRPCManager::Dispatcher::InternalHandler::is_method() const +{ + const auto index = static_cast(_func.index()); + switch (index) { + case VariantTypeIndexId::METHOD: + case VariantTypeIndexId::METHOD_FROM_PLUGIN: + return true; + break; + default:; + } + // For now, we say, if not method, it's a notification. + return false; +} + +ts::Rv +JsonRPCManager::Dispatcher::show_registered_handlers(std::string_view const &, const YAML::Node &) +{ + ts::Rv resp; + std::lock_guard lock(_mutex); + for (auto const &[name, handler] : _handlers) { + std::string const &key = handler.is_method() ? RPC_SERVICE_METHODS_KEY : RPC_SERVICE_NOTIFICATIONS_KEY; + resp.result()[key].push_back(name); + } + return resp; +} + +// ----------------------------------------------------------------------------- +// This jsonrpc handler can provides a service descriptor for the RPC +ts::Rv +JsonRPCManager::Dispatcher::get_service_descriptor(std::string_view const &, const YAML::Node &) +{ + YAML::Node rpcService; + std::lock_guard lock(_mutex); + // std::for_each(std::begin(_handlers), std::end(_handlers), [&rpcService](auto const &h) { + for (auto const &[name, handler] : _handlers) { + YAML::Node method; + method[RPC_SERVICE_NAME_KEY] = name; + method[RPC_SERVICE_TYPE_KEY] = handler.is_method() ? RPC_SERVICE_METHOD_STR : RPC_SERVICE_NOTIFICATION_STR; + /* most of this information will be eventually populated from the RPCRegistryInfo object */ + auto regInfo = handler.get_reg_info(); + std::string provider; + if (regInfo && regInfo->provider.size()) { + provider = std::string{regInfo->provider}; + } else { + provider = RPC_SERVICE_N_A_STR; + } + method[RPC_SERVICE_PROVIDER_KEY] = provider; + YAML::Node schema{YAML::NodeType::Map}; // no schema for now, but we have a placeholder for it. Schema should provide + // description and all the details about the call + method[RPC_SERVICE_SCHEMA_KEY] = std::move(schema); + rpcService[RPC_SERVICE_METHODS_KEY].push_back(method); + } + + return rpcService; +} +} // namespace rpc diff --git a/mgmt2/rpc/jsonrpc/JsonRPCManager.h b/mgmt2/rpc/jsonrpc/JsonRPCManager.h new file mode 100644 index 00000000000..f2a100e9177 --- /dev/null +++ b/mgmt2/rpc/jsonrpc/JsonRPCManager.h @@ -0,0 +1,287 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tscore/Errata.h" +#include "tscore/Diags.h" +#include "tscpp/util/ts_meta.h" +#include "ts/apidefs.h" + +#include "Defs.h" + +namespace rpc +{ +// forward +namespace json_codecs +{ + class yamlcpp_json_decoder; + class yamlcpp_json_encoder; +} // namespace json_codecs + +struct RPCRegistryInfo { + std::string_view provider; +}; +/// +/// @brief JSONRPC registration and JSONRPC invocation logic https://www.jsonrpc.org/specification +/// doc TBC +class JsonRPCManager +{ +private: + /// @note In case we want to change the codecs and use another library, we just need to follow the same signatures @see + /// yamlcpp_json_decoder and @see yamlcpp_json_encoder. + // We use the yamlcpp library by default. + using Decoder = json_codecs::yamlcpp_json_decoder; + using Encoder = json_codecs::yamlcpp_json_encoder; + +public: + // Possible RPC method signatures. + using MethodHandlerSignature = std::function(std::string_view const &, const YAML::Node &)>; + using PluginMethodHandlerSignature = std::function; + using NotificationHandlerSignature = std::function; + + /// + /// @brief Add new registered method handler to the JSON RPC engine. + /// + /// @tparam Func The callback function type. See @c MethodHandlerSignature + /// @param name Name to be exposed by the RPC Engine, this should match the incoming request. i.e: If you register 'get_stats' + /// then the incoming jsonrpc call should have this very same name in the 'method' field. .. {...'method': + /// 'get_stats'...} . + /// @param call The function handler. + /// @param info RPCRegistryInfo pointer. + /// @return bool Boolean flag. true if the callback was successfully added, false otherwise + /// + template bool add_method_handler(const std::string &name, Func &&call, const RPCRegistryInfo *info); + + /// + /// @brief Add new registered notification handler to the JSON RPC engine. + /// + /// @tparam Func The callback function type. See @c NotificationHandlerSignature + /// @param name Name to be exposed by the RPC Engine. + /// @param call The callback function that needs handler. + /// @param info RPCRegistryInfo pointer. + /// @return bool Boolean flag. true if the callback was successfully added, false otherwise + /// + template bool add_notification_handler(const std::string &name, Func &&call, const RPCRegistryInfo *info); + + /// + /// @brief This function handles the incoming jsonrpc request and dispatch the associated registered handler. + /// + /// @param jsonString The incoming jsonrpc 2.0 message. \link https://www.jsonrpc.org/specification + /// @return std::optional For methods, a valid jsonrpc 2.0 json string will be passed back. Notifications will not + /// contain any json back. + /// + std::optional handle_call(std::string const &jsonString); + + /// + /// @brief Get the instance of the whole RPC engine. + /// + /// @return JsonRPCManager& The JsonRPCManager protocol implementation object. + /// + static JsonRPCManager & + instance() + { + static JsonRPCManager rpc; + return rpc; + } + +protected: // For unit test. + JsonRPCManager() = default; + JsonRPCManager(JsonRPCManager const &) = delete; + JsonRPCManager(JsonRPCManager &&) = delete; + JsonRPCManager &operator=(JsonRPCManager const &) = delete; + JsonRPCManager &operator=(JsonRPCManager &&) = delete; + + /// + /// @brief Remove handler from the registered method handlers. Test only. + /// + /// @param name Method name. + /// @return true If all is good. + /// @return false If we could not remove it. + /// + bool remove_handler(std::string const &name); + friend bool test_remove_handler(std::string const &name); + +private: + /// + /// @brief Internal class that holds and handles the dispatch logic. + /// + /// It holds methods and notifications as well as provides the mechanism to call each particular handler. + /// + /// Design notes: + /// + /// This class holds a std::unordered_map as a main table for all the callbacks. + /// The @c InternalHandler wraps a std::variant with the supported handler types, depending on each handler type the invocation + /// varies. All handlers gets call synchronously with the difference that for Plugin handlers (see @c PluginMethod) we will wait + /// for the response to be set, plugins are provided with an API to deal with different responses(success or error), plugins do + /// not require to respond to the callback with a response, see @c PluginMethodHandlerSignature . + /// @c FunctionWrapper class holds the actual @c std::function object, this class is needed to make easy to handle equal + /// signatures inside a @c std::variant + class Dispatcher + { + using response_type = std::pair< + std::optional, + std::error_code>; ///< The response type used internally, notifications won't fill in the optional response. @c ec will be set + + /// + /// @brief Class that wraps the actual std::function. + /// + /// @tparam T Handler signature See @c MethodHandlerSignature @c PluginMethodHandlerSignature @c NotificationHandlerSignature + /// + template struct FunctionWrapper { + FunctionWrapper(T &&t) : cb(std::forward(t)) {} + + T cb; ///< Function handler (std::function) + }; + + struct InternalHandler; ///< fw declaration + + public: + Dispatcher(); + /// Add a method handler to the internal container + /// @return True if was successfully added, False otherwise. + template + bool add_handler(std::string const &name, Handler &&handler, const RPCRegistryInfo *info); + + /// Find and call the request's callback. If any error occurs, the return type will have the specific error. + /// For notifications the @c RPCResponseInfo will not be set as part of the response. @c response_type + response_type dispatch(specs::RPCRequestInfo const &request) const; + + /// Find a particular registered handler(method) by its associated name. + /// @return A pair. The handler itself and a boolean flag indicating that the handler was found. If not found, second will + /// be false and the handler null. + InternalHandler const &find_handler(specs::RPCRequestInfo const &request, std::error_code &ec) const; + /// Removes a method handler. Unit test mainly. + bool remove_handler(std::string const &name); + + // JSONRPC API - here for now. + ts::Rv show_registered_handlers(std::string_view const &, const YAML::Node &); + ts::Rv get_service_descriptor(std::string_view const &, const YAML::Node &); + + // Supported handler endpoint types. + using Method = FunctionWrapper; + using PluginMethod = FunctionWrapper; + // Plugins and non plugins handlers have no difference from the point of view of the RPC manager, we call and we do not expect + // for the work to be finished. Notifications have no response at all. + using Notification = FunctionWrapper; + + private: + /// Register our own api into the RPC manager. This will expose the methods and notifications registered in the RPC + void register_service_descriptor_handler(); + + // Functions to deal with the handler invocation. + response_type invoke_method_handler(InternalHandler const &handler, specs::RPCRequestInfo const &request) const; + response_type invoke_notification_handler(InternalHandler const &handler, specs::RPCRequestInfo const &request) const; + + /// + /// @brief Class that wraps callable objects of any RPC specific type. If provided, this class also holds a valid registry + /// information + /// + /// This class holds the actual callable object from one of our supported types, this helps us to + /// simplify the logic to insert and fetch callable objects from our container. + struct InternalHandler { + InternalHandler() = default; + InternalHandler(const RPCRegistryInfo *info) : _regInfo(info) {} + /// Sets the handler. + template void set_callback(F &&t); + explicit operator bool() const; + bool operator!() const; + /// Invoke the actual handler callback. + ts::Rv invoke(specs::RPCRequestInfo const &request) const; + /// Check if the handler was registered as method. + bool is_method() const; + + /// Returns the internal registry info. + const RPCRegistryInfo * + get_reg_info() const + { + return _regInfo; + } + + private: + // We need to keep this match with the order of types in the _func variant. This will help us to identify the holding type. + enum class VariantTypeIndexId : std::size_t { NOTIFICATION = 1, METHOD = 2, METHOD_FROM_PLUGIN = 3 }; + // We support these three for now. This can easily be extended to support other signatures. + // that's one of the main points of the InternalHandler + std::variant _func; + const RPCRegistryInfo *_regInfo; ///< Can hold internal information about the handler, this could be null as it is optional. + ///< This pointer can eventually holds important information about the call. + }; + // We will keep all the handlers wrapped inside the InternalHandler class, this will help us + // to have a single container for all the types(method, notification & plugin method(cond var)). + std::unordered_map _handlers; ///< Registered handler container. + mutable std::mutex _mutex; ///< insert/find/delete mutex. + }; + + Dispatcher _dispatcher; ///< Internal handler container and dispatcher logic object. +}; + +// ------------------------------ JsonRPCManager ------------------------------- +template +bool +JsonRPCManager::add_method_handler(const std::string &name, Handler &&call, const RPCRegistryInfo *info) +{ + return _dispatcher.add_handler(name, std::forward(call), info); +} + +template +bool +JsonRPCManager::add_notification_handler(const std::string &name, Handler &&call, const RPCRegistryInfo *info) +{ + return _dispatcher.add_handler(name, std::forward(call), info); +} + +// ----------------------------- InternalHandler ------------------------------------ +template +void +JsonRPCManager::Dispatcher::InternalHandler::set_callback(F &&f) +{ + // T would be one of the handler endpoint types. + _func = T{std::forward(f)}; +} +inline JsonRPCManager::Dispatcher::InternalHandler::operator bool() const +{ + return _func.index() != 0; +} +bool inline JsonRPCManager::Dispatcher::InternalHandler::operator!() const +{ + return _func.index() == 0; +} + +// ----------------------------- Dispatcher ------------------------------------ +template +bool +JsonRPCManager::Dispatcher::add_handler(std::string const &name, Handler &&handler, const RPCRegistryInfo *info) +{ + std::lock_guard lock(_mutex); + InternalHandler call{info}; + call.set_callback(std::forward(handler)); + return _handlers.emplace(name, std::move(call)).second; +} + +} // namespace rpc diff --git a/mgmt2/rpc/jsonrpc/error/RPCError.cc b/mgmt2/rpc/jsonrpc/error/RPCError.cc new file mode 100644 index 00000000000..8cbe96f497e --- /dev/null +++ b/mgmt2/rpc/jsonrpc/error/RPCError.cc @@ -0,0 +1,93 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "RPCError.h" + +#include +#include // TODO: remove + +namespace +{ // anonymous namespace + +struct RPCErrorCategory : std::error_category { + const char *name() const noexcept override; + std::string message(int ev) const override; +}; + +const char * +RPCErrorCategory::name() const noexcept +{ + return "rpc_msg"; +} + +std::string +RPCErrorCategory::message(int ev) const +{ + using namespace rpc::error; + switch (static_cast(ev)) { + case RPCErrorCode::INVALID_REQUEST: + return {"Invalid Request"}; + case RPCErrorCode::METHOD_NOT_FOUND: + return {"Method not found"}; + case RPCErrorCode::INVALID_PARAMS: + return {"Invalid params"}; + case RPCErrorCode::INTERNAL_ERROR: + return {"Internal error"}; + case RPCErrorCode::PARSE_ERROR: + return {"Parse error"}; + // version + case RPCErrorCode::InvalidVersion: + return {"Invalid version, 2.0 only"}; + case RPCErrorCode::InvalidVersionType: + return {"Invalid version type, should be a string"}; + case RPCErrorCode::MissingVersion: + return {"Missing version field"}; + // method + case RPCErrorCode::InvalidMethodType: + return {"Invalid method type, should be a string"}; + case RPCErrorCode::MissingMethod: + return {"Missing method field"}; + // params + case RPCErrorCode::InvalidParamType: + return {"Invalid params type, should be a structure"}; + // id + case RPCErrorCode::InvalidIdType: + return {"Invalid id type"}; + case RPCErrorCode::NullId: + return {"Use of null as id is discouraged"}; + case RPCErrorCode::ExecutionError: + return {"Error during execution"}; + default: + return "Rpc error " + std::to_string(ev); + } +} + +const RPCErrorCategory rpcErrorCategory{}; +} // anonymous namespace + +namespace rpc::error +{ +std::error_code +make_error_code(rpc::error::RPCErrorCode e) +{ + return {static_cast(e), rpcErrorCategory}; +} + +} // namespace rpc::error \ No newline at end of file diff --git a/mgmt2/rpc/jsonrpc/error/RPCError.h b/mgmt2/rpc/jsonrpc/error/RPCError.h new file mode 100644 index 00000000000..b10c8b91742 --- /dev/null +++ b/mgmt2/rpc/jsonrpc/error/RPCError.h @@ -0,0 +1,72 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace rpc::error +{ +enum class RPCErrorCode { + // for std::error_code to work, we shouldn't define 0. + + // JSONRPC 2.0 protocol defined errors. + INVALID_REQUEST = -32600, + METHOD_NOT_FOUND = -32601, + INVALID_PARAMS = -32602, + INTERNAL_ERROR = -32603, + PARSE_ERROR = -32700, + + // Custom errors. + + // version + InvalidVersion = 1, + InvalidVersionType = 2, + MissingVersion, + // method + InvalidMethodType, + MissingMethod, + + // params + InvalidParamType, + + // id + InvalidIdType, + NullId = 8, + + // execution errors + + // Internal rpc error when executing the method. + ExecutionError +}; +// TODO: force non 0 check +std::error_code make_error_code(rpc::error::RPCErrorCode e); +} // namespace rpc::error + +namespace std +{ +template <> struct is_error_code_enum : true_type { +}; +} // namespace std diff --git a/mgmt2/rpc/jsonrpc/json/YAMLCodec.h b/mgmt2/rpc/jsonrpc/json/YAMLCodec.h new file mode 100644 index 00000000000..2998caf2463 --- /dev/null +++ b/mgmt2/rpc/jsonrpc/json/YAMLCodec.h @@ -0,0 +1,333 @@ +/** + @file YAMLCodec.h + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include "rpc/jsonrpc/error/RPCError.h" +#include "rpc/jsonrpc/Defs.h" + +namespace rpc::json_codecs +{ +/// +/// @note The overall design is to make this classes @c yamlcpp_json_decoder and @c yamlcpp_json_encoder plugables into the Json Rpc +/// encode/decode logic. yamlcpp does not give us all the behavior we need, such as the way it handles the null values. Json needs +/// to use literal null and yamlcpp uses ~. If this becomes a problem, then we may need to change the codec implementation, we just +/// follow the api and it should work with minimum changes. +/// + +/// +/// @brief Implements json to request object decoder. +/// This class converts and validate the incoming string into a request object. +/// +class yamlcpp_json_decoder +{ + /// + /// @brief Function that decodes and validate the fields base on the JSONRPC 2.0 protocol, All this is based on the specs for each + /// field. + /// + /// @param node @see YAML::Node that contains all the fields. + /// @return std::pair It returns a pair of request and an the error reporting. + /// + static std::pair + decode_and_validate(YAML::Node const &node) noexcept + { + specs::RPCRequestInfo request; + if (!node.IsDefined() || (node.Type() != YAML::NodeType::Map) || (node.size() == 0)) { + // We only care about structures with elements. + return {request, error::RPCErrorCode::INVALID_REQUEST}; + } + + try { + // try the id first so we can use for a possible error. + // All this may be obsolete if we decided to accept only strings. + if (auto id = node["id"]) { + if (id.IsNull()) { + // if it's present, it should be valid. + return {request, error::RPCErrorCode::NullId}; + } + + try { + request.id = id.as(); + } catch (YAML::Exception const &) { + return {request, error::RPCErrorCode::InvalidIdType}; + } + } // else -> it's fine, could be a notification. + // version + if (auto version = node["jsonrpc"]) { + try { + request.jsonrpc = version.as(); + + if (request.jsonrpc != specs::JSONRPC_VERSION) { + return {request, error::RPCErrorCode::InvalidVersion}; + } + } catch (YAML::Exception const &ex) { + return {request, error::RPCErrorCode::InvalidVersionType}; + } + } else { + return {request, error::RPCErrorCode::MissingVersion}; + } + // method + if (auto method = node["method"]) { + try { + request.method = method.as(); + } catch (YAML::Exception const &ex) { + return {request, error::RPCErrorCode::InvalidMethodType}; + } + } else { + return {request, error::RPCErrorCode::MissingMethod}; + } + // params + if (auto params = node["params"]) { + // TODO: check schema. + switch (params.Type()) { + case YAML::NodeType::Map: + case YAML::NodeType::Sequence: + break; + default: + return {request, error::RPCErrorCode::InvalidParamType}; + } + request.params = std ::move(params); + } + // else -> params can be omitted + } catch (std::exception const &e) { + // we want to keep the request as we will respond with a message. + return {request, error::RPCErrorCode::PARSE_ERROR}; + } + // TODO We may want to extend the error handling and inform the user if there is more than one invalid field in the request, + // so far we notify only the first one, we can use the data field to add more errors in it. ts::Errata + return {request, {/*ok*/}}; + } + +public: + /// + /// @brief Decode a string, either json or yaml into a @see specs::RPCRequest . @c ec will report the error if occurs @see + /// RPCErrorCode + /// + /// @param request The string request, this should be either json or yaml. + /// @param ec Output value, The error reporting. + /// @return specs::RPCRequest A valid rpc response object if no errors. + /// + static specs::RPCRequest + decode(std::string const &request, std::error_code &ec) noexcept + { + specs::RPCRequest msg; + try { + YAML::Node node = YAML::Load(request); + switch (node.Type()) { + case YAML::NodeType::Map: { // 4 + // single element + msg.add_message(decode_and_validate(node)); + } break; + case YAML::NodeType::Sequence: { // 3 + // In case we get [] which is valid sequence but invalid jsonrpc message. + if (node.size() > 0) { + // it's a batch + msg.is_batch(true); + msg.reserve(node.size()); + + for (auto &&n : node) { + msg.add_message(decode_and_validate(n)); + } + } else { + // Valid json but invalid base on jsonrpc specs, ie: []. + ec = error::RPCErrorCode::INVALID_REQUEST; + } + } break; + default: + // Only Sequences or Objects are valid. + ec = error::RPCErrorCode::INVALID_REQUEST; + break; + } + } catch (YAML::Exception const &e) { + ec = error::RPCErrorCode::PARSE_ERROR; + } + return msg; + } +}; + +/// +/// @brief Implements request to string encoder. +/// This class converts a request(including errors) into a json string. +/// +class yamlcpp_json_encoder +{ + /// + /// @brief Encode the ID if present. + /// If the id is not present which could be interpret as null(should not happen), it will not be set into the emitter, it will be + /// ignored. This is due the way that yamlcpp deals with the null, which instead of the literal null, it uses ~ + static void + encode_id(const std::optional &id, YAML::Emitter &json) + { + // workaround, we should find a better way, we should be able to use literal null if needed + if (id) { + json << YAML::Key << "id" << YAML::Value << *id; + } + // We do not insert null as it will break the json, we need literal null and not ~ (as per yaml) + // json << YAML::Null; + } + + /// + /// @brief Function to encode an error. + /// Error could be from two sources, presence of @c std::error_code means a high level and the @c ts::Errata a callee . Both will + /// be written into the passed @c out YAML::Emitter. This is mainly a convenience class for the other two encode_* functions. + /// + /// @param error std::error_code High level, main error. + /// @param errata the Errata from the callee + /// @param json output parameter. YAML::Emitter. + /// + static void + encode_error(std::error_code error, ts::Errata const &errata, YAML::Emitter &json) + { + json << YAML::Key << "error"; + json << YAML::BeginMap; + json << YAML::Key << "code" << YAML::Value << error.value(); + json << YAML::Key << "message" << YAML::Value << error.message(); + if (!errata.isOK()) { + json << YAML::Key << "data"; + json << YAML::BeginSeq; + for (auto const &err : errata) { + json << YAML::BeginMap; + json << YAML::Key << "code" << YAML::Value << err.getCode(); + json << YAML::Key << "message" << YAML::Value << err.text(); + json << YAML::EndMap; + } + json << YAML::EndSeq; + } + json << YAML::EndMap; + } + + /// Convenience functions to call encode_error. + static void + encode_error(std::error_code error, YAML::Emitter &json) + { + ts::Errata errata{}; + encode_error(error, errata, json); + } + /// Convenience functions to call encode_error. + static void + encode_error(ts::Errata const &errata, YAML::Emitter &json) + { + encode_error({error::RPCErrorCode::ExecutionError}, errata, json); + } + + static void + encode_error_from_callee(ts::Errata const &errata, YAML::Emitter &json) + { + if (!errata.isOK()) { + json << YAML::Key << "errors"; + json << YAML::BeginSeq; + for (auto const &err : errata) { + json << YAML::BeginMap; + json << YAML::Key << "code" << YAML::Value << err.getCode(); + json << YAML::Key << "message" << YAML::Value << err.text(); + json << YAML::EndMap; + } + json << YAML::EndSeq; + } + } + /// + /// @brief Function to encode a single response(no batch) into an emitter. + /// + /// @param resp Response object + /// @param json output yaml emitter + /// + static void + encode(const specs::RPCResponseInfo &resp, YAML::Emitter &json) + { + json << YAML::BeginMap; + json << YAML::Key << "jsonrpc" << YAML::Value << specs::JSONRPC_VERSION; + + // Important! As per specs, errors have preference over the result, we ignore result if error was set. + + if (resp.rpcError) { + // internal library detected error: Decoding, etc. + encode_error(resp.rpcError, json); + } + // Registered handler error: They have set the error on the response from the registered handler. This uses ExecutionError as + // top error. + else if (!resp.callResult.errata.isOK()) { + encode_error(resp.callResult.errata, json); + } + // A valid response: The registered handler have set the proper result and no error was flagged. + else { + json << YAML::Key << "result" << YAML::Value; + + // Could be the case that the registered handler did not set the result. Make sure it was set before inserting it. + if (!resp.callResult.result.IsNull()) { + json << resp.callResult.result; + } else { + // TODO: do we want to let it return null or by default we put success when there was no error and no result either. Maybe + // empty? + json << "success"; + } + } + + // insert the id. + encode_id(resp.id, json); + json << YAML::EndMap; + } + +public: + /// + /// @brief Convert @see specs::RPCResponseInfo into a std::string. + /// + /// @param resp The rpc object to be converted @see specs::RPCResponseInfo + /// @return std::string The string representation of the response object + /// + static std::string + encode(const specs::RPCResponseInfo &resp) + { + YAML::Emitter json; + json << YAML::DoubleQuoted << YAML::Flow; + encode(resp, json); + + return json.c_str(); + } + + /// + /// @brief Convert @see specs::RPCResponse into a std::string, this handled a batch response. + /// + /// @param response The object to be converted + /// @return std::string the string representation of the response object after being encode. + /// + static std::string + encode(const specs::RPCResponse &response) + { + YAML::Emitter json; + json << YAML::DoubleQuoted << YAML::Flow; + { + if (response.is_batch()) { + json << YAML::BeginSeq; + } + + for (const auto &resp : response.get_messages()) { + encode(resp, json); + } + + if (response.is_batch()) { + json << YAML::EndSeq; + } + } + + return json.c_str(); + } +}; +} // namespace rpc::json_codecs diff --git a/mgmt2/rpc/jsonrpc/unit_tests/test_basic_protocol.cc b/mgmt2/rpc/jsonrpc/unit_tests/test_basic_protocol.cc new file mode 100644 index 00000000000..543394b23a4 --- /dev/null +++ b/mgmt2/rpc/jsonrpc/unit_tests/test_basic_protocol.cc @@ -0,0 +1,589 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include /* catch unit-test framework */ + +#include + +#include "rpc/jsonrpc/JsonRPCManager.h" +#include "rpc/jsonrpc/JsonRPC.h" +#include "rpc/handlers/common/ErrorUtils.h" + +namespace +{ +const int ErratId{1}; + +// Not using the singleton logic. +struct JsonRpcUnitTest : rpc::JsonRPCManager { + JsonRpcUnitTest() : JsonRPCManager() {} + using base = JsonRPCManager; + bool + remove_handler(std::string const &name) + { + return rpc::JsonRPCManager::remove_handler(name); + } + template + bool + add_notification_handler(const std::string &name, Func &&call) + { + return base::add_notification_handler(name, std::forward(call), nullptr); + } + template + bool + add_method_handler(const std::string &name, Func &&call) + { + return base::add_method_handler(name, std::forward(call), nullptr); + } +}; + +enum class TestErrors { ERR1 = 9999, ERR2 }; +inline ts::Rv +test_callback_ok_or_error(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv resp; + + // play with the req.id if needed. + if (YAML::Node n = params["return_error"]) { + auto yesOrNo = n.as(); + if (yesOrNo == "yes") { + // Can we just have a helper for this? + // resp.errata.push(static_cast(TestErrors::ERR1), "Just an error message to add more meaning to the failure"); + resp.errata().push(ErratId, static_cast(TestErrors::ERR1), "Just an error message to add more meaning to the failure"); + } else { + resp.result()["ran"] = "ok"; + } + } + return resp; +} + +static int notificationCallCount{0}; +inline void +test_nofitication(YAML::Node const ¶ms) +{ + notificationCallCount++; +} +} // namespace +TEST_CASE("Multiple Registrations - methods", "[methods]") +{ + JsonRpcUnitTest rpc; + SECTION("More than one method") + { + REQUIRE(rpc.add_method_handler("test_callback_ok_or_error", &test_callback_ok_or_error)); + REQUIRE(rpc.add_method_handler("test_callback_ok_or_error", &test_callback_ok_or_error) == false); + } +} + +TEST_CASE("Multiple Registrations - notifications", "[notifications]") +{ + JsonRpcUnitTest rpc; + SECTION("inserting several notifications with the same name") + { + REQUIRE(rpc.add_notification_handler("test_nofitication", &test_nofitication)); + REQUIRE(rpc.add_notification_handler("test_nofitication", &test_nofitication) == false); + } +} + +TEST_CASE("Register/call method", "[method]") +{ + JsonRpcUnitTest rpc; + + SECTION("Registring the method") + { + REQUIRE(rpc.add_method_handler("test_callback_ok_or_error", &test_callback_ok_or_error)); + + SECTION("Calling the method") + { + const auto json = rpc.handle_call( + R"({"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "no"}, "id": "13"})"); + REQUIRE(json); + const std::string_view expected = R"({"jsonrpc": "2.0", "result": {"ran": "ok"}, "id": "13"})"; + REQUIRE(*json == expected); + } + } +} + +// VALID RESPONSES WITH CUSTOM ERRORS +TEST_CASE("Register/call method - respond with errors (data field)", "[method][error.data]") +{ + JsonRpcUnitTest rpc; + + SECTION("Registring the method") + { + REQUIRE(rpc.add_method_handler("test_callback_ok_or_error", &test_callback_ok_or_error)); + + SECTION("Calling the method") + { + const auto json = rpc.handle_call( + R"({"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "yes"}, "id": "14"})"); + REQUIRE(json); + const std::string_view expected = + R"({"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "message": "Just an error message to add more meaning to the failure"}]}, "id": "14"})"; + REQUIRE(*json == expected); + } + } +} + +TEST_CASE("Register/call notification", "[notifications]") +{ + JsonRpcUnitTest rpc; + + SECTION("Registring the notification") + { + REQUIRE(rpc.add_notification_handler("test_nofitication", &test_nofitication)); + + SECTION("Calling the notification") + { + rpc.handle_call(R"({"jsonrpc": "2.0", "method": "test_nofitication", "params": {"json": "rpc"}})"); + REQUIRE(notificationCallCount == 1); + notificationCallCount = 0; // for further use. + } + } +} + +TEST_CASE("Basic test, batch calls", "[methods][notifications]") +{ + JsonRpcUnitTest rpc; + + SECTION("inserting multiple functions, mixed method and notifications.") + { + const auto f1_added = rpc.add_method_handler("test_callback_ok_or_error", &test_callback_ok_or_error); + const bool f2_added = rpc.add_notification_handler("test_nofitication", &test_nofitication); + + REQUIRE(f1_added); + REQUIRE(f2_added); + + SECTION("we call in batch, two functions and one notification") + { + const auto resp1 = rpc.handle_call( + R"([{"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "no"}, "id": "13"} + ,{"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "yes"}, "id": "14"} + ,{"jsonrpc": "2.0", "method": "test_nofitication", "params": {"name": "damian"}}])"); + + REQUIRE(resp1); + const std::string_view expected = + R"([{"jsonrpc": "2.0", "result": {"ran": "ok"}, "id": "13"}, {"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution", "data": [{"code": 9999, "message": "Just an error message to add more meaning to the failure"}]}, "id": "14"}])"; + REQUIRE(*resp1 == expected); + } + } +} + +TEST_CASE("Single registered notification", "[notifications]") +{ + notificationCallCount = 0; + JsonRpcUnitTest rpc; + SECTION("Single notification, only notifications in the batch.") + { + REQUIRE(rpc.add_notification_handler("test_nofitication", &test_nofitication)); + + SECTION("Call the notifications in batch") + { + const auto should_be_no_response = rpc.handle_call( + R"([{"jsonrpc": "2.0", "method": "test_nofitication", "params": {"name": "JSON"}}, + {"jsonrpc": "2.0", "method": "test_nofitication", "params": {"name": "RPC"}}, + {"jsonrpc": "2.0", "method": "test_nofitication", "params": {"name": "2.0"}}])"); + + REQUIRE(!should_be_no_response); + REQUIRE(notificationCallCount == 3); + } + } +} + +TEST_CASE("Valid json but invalid messages", "[errors]") +{ + JsonRpcUnitTest rpc; + + SECTION("Valid json but invalid protocol {}") + { + const auto resp = rpc.handle_call(R"({})"); + REQUIRE(resp); + std::string_view expected = R"({"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}})"; + REQUIRE(*resp == expected); + } + + SECTION("Valid json but invalid protocol [{},{}] - batch response") + { + const auto resp = rpc.handle_call(R"([{},{}])"); + REQUIRE(resp); + std::string_view expected = + R"([{"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}}, {"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}}])"; + REQUIRE(*resp == expected); + } +} + +TEST_CASE("Invalid json messages", "[errors][invalid json]") +{ + JsonRpcUnitTest rpc; + SECTION("Invalid json in an attempt of batch") + { + const auto resp = rpc.handle_call( + R"([{"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "no"}, "id": "13"} + ,{"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "yes + ,{"jsonrpc": "2.0", "method": "test_nofitication", "params": {"name": "damian"}}])"); + + REQUIRE(resp); + + std::string_view expected = R"({"jsonrpc": "2.0", "error": {"code": -32700, "message": "Parse error"}})"; + REQUIRE(*resp == expected); + } +} + +TEST_CASE("Invalid parameters base on the jsonrpc 2.0 protocol", "[protocol]") +{ + JsonRpcUnitTest rpc; + + REQUIRE(rpc.add_method_handler("test_callback_ok_or_error", &test_callback_ok_or_error)); + REQUIRE(rpc.add_notification_handler("test_nofitication", &test_nofitication)); + + SECTION("version field") + { + SECTION("number instead of a string") + { + // THIS WILL FAIL BASE ON THE YAMLCPP WAY TO TEST TYPES. We can get the number first, which will be ok and then fail, but + // seems not the right way to do it. ok for now. + [[maybe_unused]] const auto resp = + rpc.handle_call(R"({"jsonrpc": 2.0, "method": "test_callback_ok_or_error", "params": {"return_error": "no"}, "id": "13"})"); + + // REQUIRE(resp); + + [[maybe_unused]] const std::string_view expected = + R"({"jsonrpc": "2.0", "error": {"code": 2, "message": "Invalid version type, should be a string"}, "id": "13"})"; + // REQUIRE(*resp == expected); + } + + SECTION("2.8 instead of 2.0") + { + const auto resp = rpc.handle_call( + R"({"jsonrpc": "2.8", "method": "test_callback_ok_or_error", "params": {"return_error": "no"}, "id": "15"})"); + + REQUIRE(resp); + const std::string_view expected = + R"({"jsonrpc": "2.0", "error": {"code": 1, "message": "Invalid version, 2.0 only"}, "id": "15"})"; + REQUIRE(*resp == expected); + } + } + SECTION("method field") + { + SECTION("using a number") + { + // THIS WILL FAIL BASE ON THE YAMLCPP WAY TO TEST TYPES, there is no explicit way to test for a particular type, we can get + // the number first, then as it should not be converted we get the string instead, this seems rather not the best way to do + // it. ok for now. + [[maybe_unused]] const auto resp = + rpc.handle_call(R"({"jsonrpc": "2.0", "method": 123, "params": {"return_error": "no"}, "id": "14"})"); + + // REQUIRE(resp); + [[maybe_unused]] const std::string_view expected = + R"({"jsonrpc": "2.0", "error": {"code": 4, "message": "Invalid method type, should be a string"}, "id": "14"})"; + // REQUIRE(*resp == expected); + } + } + SECTION("param field") + { + SECTION("Invalidtype") + { + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": 13, "id": "13"})"); + + REQUIRE(resp); + const std::string_view expected = + R"({"jsonrpc": "2.0", "error": {"code": 6, "message": "Invalid params type, should be a structure"}, "id": "13"})"; + REQUIRE(*resp == expected); + } + } + + SECTION("id field") + { + SECTION("null id") + { + const auto resp = rpc.handle_call( + R"({"jsonrpc": "2.0", "method": "test_callback_ok_or_error", "params": {"return_error": "no"}, "id": null})"); + REQUIRE(resp); + const std::string_view expected = + R"({"jsonrpc": "2.0", "error": {"code": 8, "message": "Use of null as id is discouraged"}})"; + REQUIRE(*resp == expected); + } + } +} + +TEST_CASE("Basic test with member functions(add, remove)", "[basic][member_functions]") +{ + struct TestMemberFunctionCall { + TestMemberFunctionCall() {} + bool + register_member_function_as_callback(JsonRpcUnitTest &rpc) + { + return rpc.add_method_handler( + "member_function", + [this](std::string_view const &id, const YAML::Node &req) -> ts::Rv { return test(id, req); }); + } + ts::Rv + test(std::string_view const &id, const YAML::Node &req) + { + ts::Rv resp; + resp.result() = "grand!"; + return resp; + } + + // TODO: test notification as well. + }; + + SECTION("A RPC object and a custom class") + { + JsonRpcUnitTest rpc; + TestMemberFunctionCall tmfc; + + REQUIRE(tmfc.register_member_function_as_callback(rpc) == true); + SECTION("call the member function") + { + const auto response = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "member_function", "id": "AbC"})"); + REQUIRE(response); + REQUIRE(*response == R"({"jsonrpc": "2.0", "result": "grand!", "id": "AbC"})"); + } + + SECTION("We remove the callback handler") + { + REQUIRE(rpc.remove_handler("member_function") == true); + + const auto response = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "member_function", "id": "AbC"})"); + REQUIRE(response); + REQUIRE(*response == R"({"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": "AbC"})"); + } + } +} + +TEST_CASE("Test Dispatcher rpc method", "[dispatcher]") +{ + JsonRpcUnitTest rpc; + const auto response = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "show_registered_handlers", "id": "AbC"})"); + REQUIRE(*response == + R"({"jsonrpc": "2.0", "result": {"methods": ["get_service_descriptor", "show_registered_handlers"]}, "id": "AbC"})"); +} + +[[maybe_unused]] static ts::Rv +subtract(std::string_view const &id, YAML::Node const &numbers) +{ + ts::Rv res; + + if (numbers.Type() == YAML::NodeType::Sequence) { + auto it = numbers.begin(); + int total = it->as(); + ++it; + for (; it != numbers.end(); ++it) { + total -= it->as(); + } + + res.result() = total; + } else if (numbers.Type() == YAML::NodeType::Map) { + if (numbers["subtrahend"] && numbers["minuend"]) { + int total = numbers["minuend"].as() - numbers["subtrahend"].as(); + res.result() = total; + } + } + return res; +} + +[[maybe_unused]] static ts::Rv +sum(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv res; + int total{0}; + for (auto n : params) { + total += n.as(); + } + res.result() = total; + return res; +} + +[[maybe_unused]] static ts::Rv +get_data(std::string_view const &id, YAML::Node const ¶ms) +{ + ts::Rv res; + res.result().push_back("hello"); + res.result().push_back("5"); + return res; +} + +[[maybe_unused]] static void +update(YAML::Node const ¶ms) +{ +} +[[maybe_unused]] static void +foobar(YAML::Node const ¶ms) +{ +} +[[maybe_unused]] static void +notify_hello(YAML::Node const ¶ms) +{ +} + +// TODO: add tests base on the protocol example doc. +TEST_CASE("Basic tests from the jsonrpc 2.0 doc.") +{ + SECTION("rpc call with positional parameters") + { + JsonRpcUnitTest rpc; + + REQUIRE(rpc.add_method_handler("subtract", &subtract)); + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "subtract", "params": [42, 23], "id": "1"})"); + REQUIRE(resp); + REQUIRE(*resp == R"({"jsonrpc": "2.0", "result": "19", "id": "1"})"); + + const auto resp1 = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "subtract", "params": [23, 42], "id": "1"})"); + REQUIRE(resp1); + REQUIRE(*resp1 == R"({"jsonrpc": "2.0", "result": "-19", "id": "1"})"); + } + + SECTION("rpc call with named parameters") + { + JsonRpcUnitTest rpc; + + REQUIRE(rpc.add_method_handler("subtract", &subtract)); + const auto resp = + rpc.handle_call(R"({"jsonrpc": "2.0", "method": "subtract", "params": {"subtrahend": 23, "minuend": 42}, "id": "3"})"); + REQUIRE(resp); + REQUIRE(*resp == R"({"jsonrpc": "2.0", "result": "19", "id": "3"})"); + + const auto resp1 = + rpc.handle_call(R"({"jsonrpc": "2.0", "method": "subtract", "params": {"minuend": 42, "subtrahend": 23}, "id": "3"})"); + REQUIRE(resp1); + REQUIRE(*resp1 == R"({"jsonrpc": "2.0", "result": "19", "id": "3"})"); + } + + SECTION("A notification") + { + JsonRpcUnitTest rpc; + + REQUIRE(rpc.add_notification_handler("update", &update)); + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "update", "params": [1,2,3,4,5]})"); + REQUIRE(!resp); + + REQUIRE(rpc.add_notification_handler("foobar", &foobar)); + const auto resp1 = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "foobar"})"); + REQUIRE(!resp1); + } + + SECTION("rpc call of non-existent method") + { + JsonRpcUnitTest rpc; + + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "foobar", "id": "1"})"); + REQUIRE(resp); + REQUIRE(*resp == R"({"jsonrpc": "2.0", "error": {"code": -32601, "message": "Method not found"}, "id": "1"})"); + } + SECTION("rpc call with invalid JSON") + { + JsonRpcUnitTest rpc; + + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "foobar, "params": "bar", "baz])"); + REQUIRE(resp); + REQUIRE(*resp == R"({"jsonrpc": "2.0", "error": {"code": -32700, "message": "Parse error"}})"); + } + SECTION("rpc call with invalid Request object") + { + // We do have a custom object here, which will return invalid param object. + // skip it + } + SECTION("rpc call Batch, invalid JSON") + { + JsonRpcUnitTest rpc; + + const auto resp = + rpc.handle_call(R"( {"jsonrpc": "2.0", "method": "sum", "params": [1,2,4], "id": "1"}, {"jsonrpc": "2.0", "method")"); + REQUIRE(resp); + REQUIRE(*resp == R"({"jsonrpc": "2.0", "error": {"code": -32700, "message": "Parse error"}})"); + } + SECTION("rpc call with an empty Array") + { + JsonRpcUnitTest rpc; + const auto resp = rpc.handle_call(R"([])"); + REQUIRE(resp); + std::string_view expected = R"({"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}})"; + REQUIRE(*resp == expected); + } + SECTION("rpc call with an invalid Batch") + { + JsonRpcUnitTest rpc; + const auto resp = rpc.handle_call(R"([1])"); + REQUIRE(resp); + std::string_view expected = R"([{"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}}])"; + REQUIRE(*resp == expected); + } + SECTION("rpc call with invalid Batch") + { + JsonRpcUnitTest rpc; + const auto resp = rpc.handle_call(R"([1,2,3])"); + REQUIRE(resp); + std::string_view expected = + R"([{"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}}, {"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}}, {"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}}])"; + REQUIRE(*resp == expected); + } + + SECTION("rpc call Batch(all notifications") + { + JsonRpcUnitTest rpc; + REQUIRE(rpc.add_notification_handler("notify_hello", ¬ify_hello)); + REQUIRE(rpc.add_notification_handler("notify_sum", ¬ify_hello)); + const auto resp = rpc.handle_call( + R"( [{"jsonrpc": "2.0", "method": "notify_sum", "params": [1,2,4]}, {"jsonrpc": "2.0", "method": "notify_hello", "params": [7]}])"); + REQUIRE(!resp); + } +} + +TEST_CASE("Handle un-handle handler's error", "[throw]") +{ + JsonRpcUnitTest rpc; + SECTION("Basic exception thrown") + { + REQUIRE( + rpc.add_method_handler("oops_i_did_it_again", [](std::string_view const &id, const YAML::Node ¶ms) -> ts::Rv { + throw std::runtime_error("Oops, I did it again"); + })); + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "oops_i_did_it_again", "id": "1"})"); + std::string_view expected = R"({"jsonrpc": "2.0", "error": {"code": 9, "message": "Error during execution"}, "id": "1"})"; + REQUIRE(*resp == expected); + } +} + +TEST_CASE("Call registered method with no ID", "[no-id]") +{ + JsonRpcUnitTest rpc; + SECTION("Basic test, no id on method call") + { + REQUIRE( + rpc.add_method_handler("call_me_with_no_id", [](std::string_view const &id, const YAML::Node ¶ms) -> ts::Rv { + throw std::runtime_error("Oops, I did it again"); + })); + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "call_me_with_no_id"})"); + std::string_view expected = R"({"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}})"; + REQUIRE(*resp == expected); + } +} + +TEST_CASE("Call registered notification with ID", "[notification_and_id]") +{ + JsonRpcUnitTest rpc; + SECTION("Basic test, id on a notification call") + { + REQUIRE(rpc.add_notification_handler( + "call_me_with_id", [](const YAML::Node ¶ms) -> void { throw std::runtime_error("Oops, I did it again"); })); + const auto resp = rpc.handle_call(R"({"jsonrpc": "2.0", "method": "call_me_with_id", "id": "1"})"); + std::string_view expected = R"({"jsonrpc": "2.0", "error": {"code": -32600, "message": "Invalid Request"}, "id": "1"})"; + REQUIRE(*resp == expected); + } +} diff --git a/mgmt2/rpc/jsonrpc/unit_tests/unit_test_main.cc b/mgmt2/rpc/jsonrpc/unit_tests/unit_test_main.cc new file mode 100644 index 00000000000..46915952196 --- /dev/null +++ b/mgmt2/rpc/jsonrpc/unit_tests/unit_test_main.cc @@ -0,0 +1,22 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#define CATCH_CONFIG_MAIN +#include diff --git a/mgmt2/rpc/schema/admin_lookup_records_params_schema.json b/mgmt2/rpc/schema/admin_lookup_records_params_schema.json new file mode 100644 index 00000000000..92aa7ca13da --- /dev/null +++ b/mgmt2/rpc/schema/admin_lookup_records_params_schema.json @@ -0,0 +1,64 @@ +{ + "$schema":"http://json-schema.org/draft-04/schema#", + "title":"Params field RPC Record request definition.", + "description":"This is the definition expected for a RPC record request. This should be used to obtain information regarding a particular record(s) from Traffic Server Licensed under Apache V2 https://www.apache.org/licenses/LICENSE-2.0", + "type":"array", + "items":{ + "type":"object", + "properties":{ + "record_name":{ + "type":"string", + "note":[ + "if requesting a regex, then use record_name_regex instead, only one can be used in the same object." + ] + }, + "record_name_regex":{ + "type":"string" + }, + "rec_types":{ + "type":"array", + "items":{ + "anyOf":[ + { + "type":"integer", + "enum":[ + 1, + 2, + 4, + 16, + 32, + 63 + ] + }, + { + "type":"string", + "enum":[ + "1", + "2", + "4", + "16", + "32", + "63" + ] + } + ], + "note":[ + "Config=1, Process=2, Node=4, Local=16, Plugin=32, All=63." + ] + } + } + }, + "oneOf":[ + { + "required":[ + "record_name_regex" + ] + }, + { + "required":[ + "record_name" + ] + } + ] + } +} \ No newline at end of file diff --git a/mgmt2/rpc/schema/jsonrpc_request_schema.json b/mgmt2/rpc/schema/jsonrpc_request_schema.json new file mode 100644 index 00000000000..4ca5e96b391 --- /dev/null +++ b/mgmt2/rpc/schema/jsonrpc_request_schema.json @@ -0,0 +1,54 @@ +{ + "$schema":"http://json-schema.org/draft-04/schema#", + "title":"JSONRPC 2.0 request schema", + "description":"Data for a JSON RPC 2.0 request, this may contains some exceptions to the standard. Licensed under Apache V2 https://www.apache.org/licenses/LICENSE-2.0", + "oneOf":[ + { + "description":"An individual request", + "$ref":"#/definitions/request" + }, + { + "description":"An array of requests", + "type":"array", + "items":{ + "$ref":"#/definitions/request" + } + } + ], + "definitions":{ + "request":{ + "type":"object", + "required":[ + "jsonrpc", + "method" + ], + "properties":{ + "jsonrpc":{ + "enum":[ + "2.0" + ] + }, + "method":{ + "type":"string" + }, + "id":{ + "type":[ + "string", + "number", + "null" + ], + "note":[ + "While allowed, null should be avoided: http://www.jsonrpc.org/specification#id1", + "While allowed, a number with a fractional part should be avoided: http://www.jsonrpc.org/specification#id2" + ] + }, + "params":{ + "type":[ + "array", + "object" + ] + } + } + } + } + } diff --git a/mgmt2/rpc/schema/jsonrpc_response_schema.json b/mgmt2/rpc/schema/jsonrpc_response_schema.json new file mode 100644 index 00000000000..a851e792899 --- /dev/null +++ b/mgmt2/rpc/schema/jsonrpc_response_schema.json @@ -0,0 +1,104 @@ +{ + "$schema":"http://json-schema.org/draft-04/schema#", + "title":"JSON RPC 2.0 response message", + "description":"Data for a JSON RPC 2.0 response, this may contains some exceptions to the standard which will be noted. Licensed under Apache V2 https://www.apache.org/licenses/LICENSE-2.0", + "oneOf":[ + { + "$ref":"#/definitions/success" + }, + { + "$ref":"#/definitions/error" + }, + { + "type":"array", + "items":{ + "oneOf":[ + { + "$ref":"#/definitions/success" + }, + { + "$ref":"#/definitions/error" + } + ] + } + } + ], + "definitions":{ + "common":{ + "required":[ + "jsonrpc" + ], + "not":{ + "description":"Cannot have result and error at the same time", + "required":[ + "result", + "error" + ] + }, + "type":"object", + "properties":{ + "id":{ + "type":[ + "string", + "integer", + "null" + ], + "note":[ + "Specs specify that this should be mandatory, in this implementation if the server cannot fetch the id on error, the id field will not be set.", + "As per our implementation, the Server will not use literal null" + ] + }, + "jsonrpc":{ + "enum":[ + "2.0" + ] + } + } + }, + "success":{ + "description":"A success. The result member is then required and can be anything.", + "allOf":[ + { + "$ref":"#/definitions/common" + }, + { + "required":[ + "result" + ] + } + ] + }, + "error":{ + "allOf":[ + { + "$ref":"#/definitions/common" + }, + { + "required":[ + "error" + ], + "properties":{ + "error":{ + "type":"object", + "required":[ + "code", + "message" + ], + "properties":{ + "code":{ + "type":"integer" + }, + "message":{ + "type":"string" + }, + "data":{ + "description":"Optional, It will describe a list of events that occurred during the handling." + } + } + } + } + } + ] + } + } +} diff --git a/mgmt2/rpc/schema/success_response_schema.json b/mgmt2/rpc/schema/success_response_schema.json new file mode 100644 index 00000000000..42b05d1029f --- /dev/null +++ b/mgmt2/rpc/schema/success_response_schema.json @@ -0,0 +1,6 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Success response schema", + "type": "string", + "enum":["success"] +} \ No newline at end of file diff --git a/mgmt2/rpc/server/CommBase.cc b/mgmt2/rpc/server/CommBase.cc new file mode 100644 index 00000000000..acec7764c32 --- /dev/null +++ b/mgmt2/rpc/server/CommBase.cc @@ -0,0 +1,63 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "CommBase.h" + +namespace +{ +struct CommInternalErrorCategory : std::error_category { + const char *name() const noexcept override; + std::string message(int ev) const override; +}; + +const char * +CommInternalErrorCategory::name() const noexcept +{ + return "comm_internal_error_category"; +} + +std::string +CommInternalErrorCategory::message(int ev) const +{ + switch (static_cast(ev)) { + case rpc::comm::InternalError::MAX_TRANSIENT_ERRORS_HANDLED: + return {"We've reach the maximun attempt on transient errors."}; + case rpc::comm::InternalError::POLLIN_ERROR: + return {"We haven't got a POLLIN flag back while waiting"}; + case rpc::comm::InternalError::PARTIAL_READ: + return {"No more data to be read, but the buffer contains some invalid? data."}; + case rpc::comm::InternalError::FULL_BUFFER: + return {"Buffer's full."}; + default: + return "Internal Communication Error" + std::to_string(ev); + } +} + +const CommInternalErrorCategory commInternalErrorCategory{}; +} // anonymous namespace + +namespace rpc::comm +{ +std::error_code +make_error_code(rpc::comm::InternalError e) +{ + return {static_cast(e), commInternalErrorCategory}; +} + +} // namespace rpc::comm \ No newline at end of file diff --git a/mgmt2/rpc/server/CommBase.h b/mgmt2/rpc/server/CommBase.h new file mode 100644 index 00000000000..5646477b32d --- /dev/null +++ b/mgmt2/rpc/server/CommBase.h @@ -0,0 +1,47 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include + +#include + +#include "rpc/config/JsonRPCConfig.h" + +namespace rpc::comm +{ +struct BaseCommInterface { + virtual ~BaseCommInterface() {} + virtual bool configure(YAML::Node const ¶ms) = 0; + virtual void run() = 0; + virtual std::error_code init() = 0; + virtual bool stop() = 0; + virtual std::string const &name() const = 0; +}; + +enum class InternalError { MAX_TRANSIENT_ERRORS_HANDLED = 1, POLLIN_ERROR, PARTIAL_READ, FULL_BUFFER }; +std::error_code make_error_code(rpc::comm::InternalError e); + +} // namespace rpc::comm +namespace std +{ +template <> struct is_error_code_enum : true_type { +}; +} // namespace std \ No newline at end of file diff --git a/mgmt2/rpc/server/IPCSocketServer.cc b/mgmt2/rpc/server/IPCSocketServer.cc new file mode 100644 index 00000000000..ffabea9ff22 --- /dev/null +++ b/mgmt2/rpc/server/IPCSocketServer.cc @@ -0,0 +1,443 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "tscore/Diags.h" +#include "tscore/bwf_std_format.h" +#include "records/I_RecProcess.h" + +#include + +#include "rpc/jsonrpc/JsonRPCManager.h" +#include "rpc/server/IPCSocketServer.h" + +namespace +{ +constexpr size_t MAX_REQUEST_BUFFER_SIZE{32000}; + +// Quick check for errors(base on the errno); +bool check_for_transient_errors(); + +// Check poll's return and validate against the passed function. +template +bool +poll_on_socket(Func &&check_poll_return, std::chrono::milliseconds timeout, int fd) +{ + struct pollfd poll_fd; + poll_fd.fd = fd; + poll_fd.events = POLLIN; // when data is ready. + int poll_ret; + do { + poll_ret = poll(&poll_fd, 1, timeout.count()); + } while (check_poll_return(poll_ret)); + + if (!(poll_fd.revents & POLLIN)) { + return false; + } + return true; +} +} // namespace + +namespace rpc::comm +{ +static constexpr auto logTag = "rpc.net"; + +IPCSocketServer::~IPCSocketServer() +{ + unlink(_conf.sockPathName.c_str()); +} + +bool +IPCSocketServer::configure(YAML::Node const ¶ms) +{ + try { + _conf = params.as(); + } catch (YAML::Exception const &ex) { + return false; + } + + return true; +} + +std::error_code +IPCSocketServer::init() +{ + // Need to run some validations on the pathname to avoid issue. Normally this would not be an issue, but some tests may fail on + // this. + if (_conf.sockPathName.empty() || _conf.sockPathName.size() > sizeof _serverAddr.sun_path) { + Debug(logTag, "Invalid unix path name, check the size."); + return std::make_error_code(static_cast(ENAMETOOLONG)); + } + + std::error_code ec; // Flag possible errors. + + if (this->create_socket(ec); ec) { + return ec; + } + + Debug(logTag, "Using %s as socket path.", _conf.sockPathName.c_str()); + _serverAddr.sun_family = AF_UNIX; + std::strncpy(_serverAddr.sun_path, _conf.sockPathName.c_str(), sizeof(_serverAddr.sun_path) - 1); + + if (this->bind(ec); ec) { + this->close(); + return ec; + } + + if (this->listen(ec); ec) { + this->close(); + return ec; + } + + return ec; +} + +bool +IPCSocketServer::poll_for_new_client(std::chrono::milliseconds timeout) const +{ + auto check_poll_return = [&](int pfd) -> bool { + if (!_running.load()) { + return false; + } + if (pfd < 0) { + switch (errno) { + case EINTR: + case EAGAIN: + return true; + default: + return false; + } + } else if (pfd > 0) { + // ready. + return false; + } else { + // time out, try again + return true; + } + }; + + return poll_on_socket(check_poll_return, timeout, this->_socket); +} + +void +IPCSocketServer::run() +{ + _running.store(true); + + ts::LocalBufferWriter bw; + while (_running) { + // poll till socket it's ready. + if (!this->poll_for_new_client()) { + if (_running.load()) { + Warning("ups, we've got an issue."); + } + break; + } + + std::error_code ec; + if (auto fd = this->accept(ec); !ec) { + Client client{fd}; + + if (auto [ok, errStr] = client.read_all(bw); ok) { + const auto json = std::string{bw.data(), bw.size()}; + if (auto response = rpc::JsonRPCManager::instance().handle_call(json); response) { + // seems a valid response. + if (client.write(*response, ec); ec) { + Debug(logTag, "Error sending the response: %s", ec.message().c_str()); + } + } // it was a notification. + } else { + Debug(logTag, "Error detected while reading: %s", errStr.c_str()); + } + } else { + Debug(logTag, "Error while accepting a new connection on the socket: %s", ec.message().c_str()); + } + + bw.reset(); + } + + this->close(); +} + +bool +IPCSocketServer::stop() +{ + _running.store(false); + + this->close(); + + return true; +} + +void +IPCSocketServer::create_socket(std::error_code &ec) +{ + _socket = socket(AF_UNIX, SOCK_STREAM, 0); + + if (_socket < 0) { + ec = std::make_error_code(static_cast(errno)); + } +} + +int +IPCSocketServer::accept(std::error_code &ec) const +{ + int ret = {-1}; + + for (int retries = 0; retries < _conf.maxRetriesOnTransientErrors; retries++) { + ret = ::accept(_socket, 0, 0); + if (ret >= 0) { + return ret; + } + if (!check_for_transient_errors()) { + ec = std::make_error_code(static_cast(errno)); + return ret; + } + } + + if (ret < 0) { + // seems that we have reched the max retries. + ec = InternalError::MAX_TRANSIENT_ERRORS_HANDLED; + } + + return ret; +} + +void +IPCSocketServer::bind(std::error_code &ec) +{ + int lock_fd = open(_conf.lockPathName.c_str(), O_RDONLY | O_CREAT, 0600); + if (lock_fd == -1) { + ec = std::make_error_code(static_cast(errno)); + return; + } + + int ret = flock(lock_fd, LOCK_EX | LOCK_NB); + if (ret != 0) { + ec = std::make_error_code(static_cast(errno)); + return; + } + // TODO: we may be able to use SO_REUSEADDR + + // remove socket file + unlink(_conf.sockPathName.c_str()); + + ret = ::bind(_socket, (struct sockaddr *)&_serverAddr, sizeof(struct sockaddr_un)); + if (ret < 0) { + ec = std::make_error_code(static_cast(errno)); + return; + } + + mode_t mode = _conf.restrictedAccessApi ? 00700 : 00777; + if (chmod(_conf.sockPathName.c_str(), mode) < 0) { + ec = std::make_error_code(static_cast(errno)); + return; + } +} + +void +IPCSocketServer::listen(std::error_code &ec) +{ + if (::listen(_socket, _conf.backlog) < 0) { + ec = std::make_error_code(static_cast(errno)); + return; + } +} + +void +IPCSocketServer::close() +{ + if (_socket > 0) { + ::close(_socket); + _socket = -1; + } +} +//// client + +IPCSocketServer::Client::Client(int fd) : _fd{fd} {} +IPCSocketServer::Client::~Client() +{ + this->close(); +} + +// ---------------- client -------------- +bool +IPCSocketServer::Client::poll_for_data(std::chrono::milliseconds timeout) const +{ + auto check_poll_return = [&](int pfd) -> bool { + if (pfd > 0) { + // something is ready. + return false; + } else if (pfd < 0) { + switch (errno) { + case EINTR: + case EAGAIN: + return true; + default: + return false; + } + } else { // timeout + return false; + } + }; + + return poll_on_socket(check_poll_return, timeout, this->_fd); +} + +void +IPCSocketServer::Client::close() +{ + if (_fd > 0) { + ::close(_fd); + _fd = -1; + } +} + +ssize_t +IPCSocketServer::Client::read(ts::MemSpan span) const +{ + return ::read(_fd, span.data(), span.size()); +} + +std::tuple +IPCSocketServer::Client::read_all(ts::FixedBufferWriter &bw) const +{ + std::string buff; + while (bw.remaining() > 0) { + auto ret = read({bw.auxBuffer(), bw.remaining()}); + if (ret < 0) { + if (check_for_transient_errors()) { + continue; + } else { + return {false, ts::bwprint(buff, "Error reading the socket: {}", ts::bwf::Errno{})}; + } + } + + if (ret == 0) { + if (bw.size()) { + return {false, ts::bwprint(buff, "Peer disconnected after reading {} bytes.", bw.size())}; + } + return {false, ts::bwprint(buff, "Peer disconnected. EOF")}; + } + bw.fill(ret); + if (bw.remaining() > 0) { + using namespace std::chrono_literals; + if (!this->poll_for_data(1ms)) { + return {true, buff}; + } + continue; + } else { + ts::bwprint(buff, "Buffer is full, we hit the limit: {}", bw.capacity()); + break; + } + } + + return {false, buff}; +} + +void +IPCSocketServer::Client::write(std::string const &data, std::error_code &ec) const +{ + if (::write(_fd, data.c_str(), data.size()) < 0) { + ec = std::make_error_code(static_cast(errno)); + } +} +IPCSocketServer::Config::Config() +{ + // Set default values. + std::string rundir{RecConfigReadRuntimeDir()}; + lockPathName = Layout::relative_to(rundir, "jsonrpc20.lock"); + sockPathName = Layout::relative_to(rundir, "jsonrpc20.sock"); +} +} // namespace rpc::comm + +namespace YAML +{ +template <> struct convert { + using config = rpc::comm::IPCSocketServer::Config; + + static bool + decode(const Node &node, config &rhs) + { + // ++ If we configure this, traffic_ctl will not be able to connect. + // ++ This is meant to be used by unit test as you need to set up a + // ++ server. + if (auto n = node[config::LOCK_PATH_NAME_KEY_STR]) { + rhs.lockPathName = n.as(); + } + if (auto n = node[config::SOCK_PATH_NAME_KEY_STR]) { + rhs.sockPathName = n.as(); + } + + if (auto n = node[config::BACKLOG_KEY_STR]) { + rhs.backlog = n.as(); + } + if (auto n = node[config::MAX_RETRY_ON_TR_ERROR_KEY_STR]) { + rhs.maxRetriesOnTransientErrors = n.as(); + } + if (auto n = node[config::RESTRICTED_API]) { + rhs.restrictedAccessApi = n.as(); + } + return true; + } +}; +} // namespace YAML + +namespace +{ +bool +check_for_transient_errors() +{ + switch (errno) { + case EINTR: + case EAGAIN: + +#ifdef ENOMEM + case ENOMEM: +#endif + +#ifdef ENOBUF + case ENOBUF: +#endif + +#if defined(EWOULDBLOCK) && (EWOULDBLOCK != EAGAIN) + case EWOULDBLOCK: +#endif + return true; + default: + return false; + } +} +} // namespace \ No newline at end of file diff --git a/mgmt2/rpc/server/IPCSocketServer.h b/mgmt2/rpc/server/IPCSocketServer.h new file mode 100644 index 00000000000..2817c95f303 --- /dev/null +++ b/mgmt2/rpc/server/IPCSocketServer.h @@ -0,0 +1,138 @@ +/* @file + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +#include +#include +#include + +#include +#include + +#include "tscpp/util/MemSpan.h" +#include "tscore/BufferWriter.h" +#include "tscore/I_Layout.h" + +#include "rpc/server/CommBase.h" +#include "rpc/config/JsonRPCConfig.h" + +namespace rpc::comm +{ +/// +/// @brief Unix Domain Socket implementation that deals with the JSON RPC call handling mechanism. +/// +/// Very basic and straight forward implementation of an unix socket domain. The implementation +/// follows the \link BaseCommInterface. +/// +/// @note The server will keep reading the client's requests till the buffer is full or there is no more data in the wire. +/// Buffer size = 32k +class IPCSocketServer : public BaseCommInterface +{ + /// + /// @brief Connection abstraction class that deals with sending and receiving data from the connected peer. + /// + /// When client goes out of scope it will close the socket. If you want to keep the socket connected, you need to keep + /// the client object around. + struct Client { + /// @param fd Peer's socket. + Client(int fd); + /// Destructor will close the socket(if opened); + ~Client(); + + /// Close the passed socket (if opened); + void close(); + /// Reads from the socket, this function calls the system read function. + /// @return the size of what was read by the read() function. + ssize_t read(ts::MemSpan span) const; + /// Function that reads all the data available in the socket, it will validate the data on every read if there is more than + /// a single chunk. + /// The size of the buffer to be read is not defined in this function, but rather passed in the @c bw parameter. + /// @return A tuple with a boolean flag indicating if the operation did success or not, in case of any error, a text will + /// be added with a description. + std::tuple read_all(ts::FixedBufferWriter &bw) const; + /// Write the the socket with the passed data. + /// @return std::error_code. + void write(std::string const &data, std::error_code &ec) const; + bool is_connected() const; + + private: + /// Wait for data to be ready for reading. + /// @return true if the data is ready, false otherwise. + bool poll_for_data(std::chrono::milliseconds timeout) const; + int _fd; ///< connected peer's socket. + }; + +public: + IPCSocketServer() = default; + virtual ~IPCSocketServer() override; + + /// Configure the local socket. + bool configure(YAML::Node const ¶ms) override; + /// This function will create the socket, bind it and make it listen to the new socket. + /// @return the std::error_code with the collected error(if any) + std::error_code init() override; + + void run() override; + bool stop() override; + + std::string const & + name() const override + { + return _name; + } + +protected: // unit test access + struct Config { + Config(); + static constexpr auto SOCK_PATH_NAME_KEY_STR{"sock_path_name"}; + static constexpr auto LOCK_PATH_NAME_KEY_STR{"lock_path_name"}; + static constexpr auto BACKLOG_KEY_STR{"backlog"}; + static constexpr auto MAX_RETRY_ON_TR_ERROR_KEY_STR{"max_retry_on_transient_errors"}; + static constexpr auto RESTRICTED_API{"restricted_api"}; + // is it safe to call Layout now? + std::string sockPathName; + std::string lockPathName; + + int backlog{5}; + int maxRetriesOnTransientErrors{64}; + bool restrictedAccessApi{ + true}; // This config value will drive the permissions of the jsonrpc socket(either 0700(default) or 0777). + }; + + friend struct YAML::convert; + Config _conf; + +private: + inline static const std::string _name = "Local Socket"; + bool poll_for_new_client(std::chrono::milliseconds timeout = std::chrono::milliseconds(1000)) const; + void create_socket(std::error_code &ec); + + int accept(std::error_code &ec) const; + void bind(std::error_code &ec); + void listen(std::error_code &ec); + void close(); + + std::atomic_bool _running; + + struct sockaddr_un _serverAddr; + int _socket{-1}; +}; +} // namespace rpc::comm diff --git a/mgmt2/rpc/server/RPCServer.cc b/mgmt2/rpc/server/RPCServer.cc new file mode 100644 index 00000000000..2117ee29920 --- /dev/null +++ b/mgmt2/rpc/server/RPCServer.cc @@ -0,0 +1,90 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "RPCServer.h" +#include "rpc/server/IPCSocketServer.h" + +rpc::RPCServer *jsonrpcServer = nullptr; + +namespace rpc +{ +static const auto logTag{"rpc"}; + +RPCServer::RPCServer(config::RPCConfig const &conf) +{ + switch (conf.get_comm_type()) { + case config::RPCConfig::CommType::UNIX: { + _socketImpl = std::make_unique(); + if (!_socketImpl->configure(conf.get_comm_config_params())) { + Debug(logTag, "Unable to configure the socket: Stick to the default configuration."); + } + } break; + default:; + throw std::runtime_error("Unsupported communication type."); + }; + + // make sure we can initialize it. + if (auto ec = _socketImpl->init(); ec) { + throw std::runtime_error(ec.message()); + } +} + +std::string_view +RPCServer::selected_comm_name() const noexcept +{ + return _socketImpl->name(); +} + +RPCServer::~RPCServer() +{ + stop_thread(); +} + +void * /* static */ +RPCServer::run_thread(void *a) +{ + void *ret = a; + if (jsonrpcServer->_init) { + jsonrpcServer->_rpcThread = jsonrpcServer->_init(); + } + jsonrpcServer->_socketImpl->run(); + Debug(logTag, "Socket stopped"); + return ret; +} + +void +RPCServer::start_thread(std::function const &cb_init, std::function const &cb_destroy) +{ + Debug(logTag, "Starting RPC Server on: %s", _socketImpl->name().c_str()); + _init = cb_init; + _destroy = cb_destroy; + + ink_thread_create(&_this_thread, run_thread, nullptr, 0, 0, nullptr); +} + +void +RPCServer::stop_thread() +{ + _socketImpl->stop(); + + ink_thread_join(_this_thread); + _this_thread = ink_thread_null(); + Debug(logTag, "Stopping RPC server on: %s", _socketImpl->name().c_str()); +} +} // namespace rpc diff --git a/mgmt2/rpc/server/RPCServer.h b/mgmt2/rpc/server/RPCServer.h new file mode 100644 index 00000000000..a94e88a1e39 --- /dev/null +++ b/mgmt2/rpc/server/RPCServer.h @@ -0,0 +1,90 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +#include +#include +#include + +#include "tscore/Diags.h" + +#include "tscore/ink_thread.h" +#include "tscore/ink_mutex.h" + +#include "tscore/ink_apidefs.h" +#include +#include "rpc/jsonrpc/JsonRPCManager.h" +#include "rpc/config/JsonRPCConfig.h" + +namespace rpc +{ +namespace comm +{ + struct BaseCommInterface; +} + +/// +/// @brief RPC Server implementation for the JSONRPC Logic. This class holds a transport which implements @see +/// BaseCommInterface +/// Objects of this class can start @see start_thread , stop @see stop_thread the server at any? time. More than one instance of +/// this class can be created as long as they use different transport configuration. +class RPCServer +{ +public: + RPCServer() = default; + /// + /// @brief Construct a new Rpc Server object + /// This function have one main goal, select the transport type base on the configuration and initialize it. + /// + /// @throw std::runtime_error if: + /// 1 - It the configured transport isn't valid for the server to create it, then an exception will be thrown. + /// 2 - The transport layer cannot be initialized. + /// @param conf the configuration object. + /// + RPCServer(config::RPCConfig const &conf); + + /// @brief The destructor will join the thread. + ~RPCServer(); + + /// @brief Returns a descriptive name that was set by the transport. Check @see BaseCommInterface + std::string_view selected_comm_name() const noexcept; + + /// @brief Thread function that runs the transport. + void start_thread(std::function const &cb_init = std::function(), + std::function const &cb_destroy = std::function()); + + /// @brief Function to stop the transport and join the thread to finish. + void stop_thread(); + +private: + /// @brief Actual thread routine. This will start the socket. + static void *run_thread(void *); + + std::function _init; + std::function _destroy; + ink_thread _this_thread{ink_thread_null()}; + TSThread _rpcThread{nullptr}; + + std::thread running_thread; + std::unique_ptr _socketImpl; +}; +} // namespace rpc + +extern rpc::RPCServer *jsonrpcServer; diff --git a/mgmt2/rpc/server/unit_tests/test_rpcserver.cc b/mgmt2/rpc/server/unit_tests/test_rpcserver.cc new file mode 100644 index 00000000000..7a5941faf58 --- /dev/null +++ b/mgmt2/rpc/server/unit_tests/test_rpcserver.cc @@ -0,0 +1,537 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#define CATCH_CONFIG_EXTERNAL_INTERFACES + +#include /* catch unit-test framework */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include "ts/ts.h" + +#include "rpc/jsonrpc/JsonRPC.h" +#include "rpc/server/RPCServer.h" +#include "rpc/server/IPCSocketServer.h" + +#include "shared/rpc/IPCSocketClient.h" +#include "I_EventSystem.h" +#include "tscore/I_Layout.h" +#include "diags.i" + +#define DEFINE_JSONRPC_PROTO_FUNCTION(fn) ts::Rv fn(std::string_view const &id, const YAML::Node ¶ms) + +namespace rpc +{ +bool +test_remove_handler(std::string const &name) +{ + return rpc::JsonRPCManager::instance().remove_handler(name); +} +} // namespace rpc +static const std::string sockPath{"/tmp/jsonrpc20_test.sock"}; +static const std::string lockPath{"/tmp/jsonrpc20_test.lock"}; +static constexpr int default_backlog{5}; +static constexpr int default_maxRetriesOnTransientErrors{64}; +static constexpr auto logTag{"rpc.test.client"}; + +struct RPCServerTestListener : Catch::TestEventListenerBase { + using TestEventListenerBase::TestEventListenerBase; // inherit constructor + ~RPCServerTestListener(); + + // The whole test run starting + void + testRunStarting(Catch::TestRunInfo const &testRunInfo) override + { + Layout::create(); + init_diags("rpc|rpc.test", nullptr); + RecProcessInit(RECM_STAND_ALONE); + + signal(SIGPIPE, SIG_IGN); + + ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION); + eventProcessor.start(2, 1048576); + + // EThread *main_thread = new EThread; + main_thread = std::make_unique(); + main_thread->set_specific(); + + rpc::config::RPCConfig serverConfig; + + auto confStr{R"({"rpc": { "enabled": true, "unix": { "lock_path_name": ")" + lockPath + R"(", "sock_path_name": ")" + sockPath + + R"(", "backlog": 5,"max_retry_on_transient_errors": 64 }}})"}; + YAML::Node configNode = YAML::Load(confStr); + serverConfig.load(configNode["rpc"]); + try { + jsonrpcServer = new rpc::RPCServer(serverConfig); + + jsonrpcServer->start_thread(); + } catch (std::exception const &ex) { + Debug(logTag, "Oops: %s", ex.what()); + } + } + + // The whole test run ending + void + testRunEnded(Catch::TestRunStats const &testRunStats) override + { + // jsonrpcServer->stop_thread(); + // delete main_thread; + if (jsonrpcServer) { + delete jsonrpcServer; + } + } + +private: + // std::unique_ptr jrpcServer; + std::unique_ptr main_thread; +}; +CATCH_REGISTER_LISTENER(RPCServerTestListener) + +RPCServerTestListener::~RPCServerTestListener() {} + +DEFINE_JSONRPC_PROTO_FUNCTION(some_foo) // id, params +{ + ts::Rv resp; + int dur{1}; + try { + dur = params["duration"].as(); + } catch (...) { + } + INFO("Sleeping for " << dur << "s"); + std::this_thread::sleep_for(std::chrono::seconds(dur)); + resp.result()["res"] = "ok"; + resp.result()["duration"] = dur; + + INFO("Done sleeping"); + return resp; +} +namespace +{ +// Handy class to avoid manually disconecting the socket. +// TODO: should it also connect? +struct ScopedLocalSocket : shared::rpc::IPCSocketClient { + using super = shared::rpc::IPCSocketClient; + // TODO, use another path. + ScopedLocalSocket() : IPCSocketClient(sockPath) {} + ~ScopedLocalSocket() { IPCSocketClient::disconnect(); } + + template + void + send_in_chunks(std::string_view data, int disconnect_after_chunk_n = -1) + { + int chunk_number{1}; + auto chunks = chunk(data); + for (auto &&part : chunks) { + if (::write(_sock, part.c_str(), part.size()) < 0) { + Debug(logTag, "error sending message :%s", std ::strerror(errno)); + break; + } + + if (disconnect_after_chunk_n == chunk_number) { + Debug(logTag, "Disconnecting it after chunk %d", chunk_number); + super::disconnect(); + return; + } + ++chunk_number; + } + } + + // basic read, if fail, why it fail is irrelevant in this test. + std::string + read() + { + ts::LocalBufferWriter<32000> bw; + auto ret = super::read_all(bw); + if (ret == ReadStatus::NO_ERROR) { + return {bw.data(), bw.size()}; + } + return {}; + } + // small wrapper function to deal with the bw. + std::string + query(std::string_view msg) + { + ts::LocalBufferWriter<32000> bw; + auto ret = connect().send(msg).read_all(bw); + if (ret == ReadStatus::NO_ERROR) { + return {bw.data(), bw.size()}; + } + + return {}; + } + +private: + template + std::array + chunk_impl(Iter from, Iter to) + { + const std::size_t size = std::distance(from, to); + if (size <= N) { + return {std::string{from, to}}; + } + std::size_t index{0}; + std::array ret; + const std::size_t each_part = size / N; + const std::size_t remainder = size % N; + + for (auto it = from; it != to;) { + if (std::size_t rem = std::distance(it, to); rem == (each_part + remainder)) { + ret[index++] = std::string{it, it + rem}; + break; + } + ret[index++] = std::string{it, it + each_part}; + std::advance(it, each_part); + } + + return ret; + } + + template + auto + chunk(std::string_view v) + { + return chunk_impl(v.begin(), v.end()); + } +}; + +// helper function to send a request and update the promise when the response is done. +// This is to be used in a multithread test. +void +send_request(std::string json, std::promise p) +{ + ScopedLocalSocket rpc_client; + auto resp = rpc_client.query(json); + p.set_value(resp); +} +} // namespace +TEST_CASE("Sending 'concurrent' requests to the rpc server.", "[thread]") +{ + SECTION("A registered handlers") + { + rpc::add_method_handler("some_foo", &some_foo); + rpc::add_method_handler("some_foo2", &some_foo); + + std::promise p1; + std::promise p2; + auto fut1 = p1.get_future(); + auto fut2 = p2.get_future(); + + REQUIRE_NOTHROW([&]() { + // Two different clients, on the same server, as the server is an Unix Domain Socket, it should handle all this + // properly, in any case we just run the basic smoke test for our server. + auto t1 = std::thread(&send_request, R"({"jsonrpc": "2.0", "method": "some_foo", "params": {"duration": 1}, "id": "aBcD"})", + std::move(p1)); + auto t2 = std::thread(&send_request, R"({"jsonrpc": "2.0", "method": "some_foo", "params": {"duration": 1}, "id": "eFgH"})", + std::move(p2)); + // wait to get the promise set. + fut1.wait(); + fut2.wait(); + + // the expected + std::string_view expected1{R"({"jsonrpc": "2.0", "result": {"res": "ok", "duration": "1"}, "id": "aBcD"})"}; + std::string_view expected2{R"({"jsonrpc": "2.0", "result": {"res": "ok", "duration": "1"}, "id": "eFgH"})"}; + + CHECK(fut1.get() == expected1); + CHECK(fut2.get() == expected2); + + t1.join(); + t2.join(); + }()); + } +} + +std::string +random_string(std::string::size_type length) +{ + auto randchar = []() -> char { + const char charset[] = "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + const size_t max_index = (sizeof(charset) - 1); + return charset[rand() % max_index]; + }; + std::string str(length, 0); + std::generate_n(str.begin(), length, randchar); + return str; +} + +DEFINE_JSONRPC_PROTO_FUNCTION(do_nothing) // id, params, resp +{ + ts::Rv resp; + resp.result()["size"] = params["msg"].as().size(); + return resp; +} + +TEST_CASE("Basic message sending to a running server", "[socket]") +{ + REQUIRE(rpc::add_method_handler("do_nothing", &do_nothing)); + SECTION("Basic single request to the rpc server") + { + const int S{500}; + auto json{R"({"jsonrpc": "2.0", "method": "do_nothing", "params": {"msg":")" + random_string(S) + R"("}, "id":"EfGh-1"})"}; + REQUIRE_NOTHROW([&]() { + ScopedLocalSocket rpc_client; + auto resp = rpc_client.query(json); + + REQUIRE(resp == R"({"jsonrpc": "2.0", "result": {"size": ")" + std::to_string(S) + R"("}, "id": "EfGh-1"})"); + }()); + } + REQUIRE(rpc::test_remove_handler("do_nothing")); +} + +TEST_CASE("Sending a message bigger than the internal server's buffer. 32000", "[buffer][error]") +{ + REQUIRE(rpc::add_method_handler("do_nothing", &do_nothing)); + + SECTION("Message larger than the the accepted size.") + { + const int S{32000}; // + the rest of the json message. + auto json{R"({"jsonrpc": "2.0", "method": "do_nothing", "params": {"msg":")" + random_string(S) + R"("}, "id":"EfGh-1"})"}; + REQUIRE_NOTHROW([&]() { + ScopedLocalSocket rpc_client; + auto resp = rpc_client.query(json); + REQUIRE(resp.empty()); + }()); + } + + REQUIRE(rpc::test_remove_handler("do_nothing")); +} + +TEST_CASE("Test with invalid json message", "[socket]") +{ + REQUIRE(rpc::add_method_handler("do_nothing", &do_nothing)); + + SECTION("A rpc server") + { + const int S{10}; + auto json{R"({"jsonrpc": "2.0", "method": "do_nothing", "params": { "msg": ")" + random_string(S) + R"("}, "id": "EfGh})"}; + REQUIRE_NOTHROW([&]() { + ScopedLocalSocket rpc_client; + auto resp = rpc_client.query(json); + + CHECK(resp == R"({"jsonrpc": "2.0", "error": {"code": -32700, "message": "Parse error"}})"); + }()); + } + REQUIRE(rpc::test_remove_handler("do_nothing")); +} + +TEST_CASE("Test with chunks", "[socket][chunks]") +{ + REQUIRE(rpc::add_method_handler("do_nothing", &do_nothing)); + + SECTION("Sending request by chunks") + { + const int S{10}; + auto json{R"({"jsonrpc": "2.0", "method": "do_nothing", "params": { "msg": ")" + random_string(S) + + R"("}, "id": "chunk-parts-3"})"}; + + REQUIRE_NOTHROW([&]() { + ScopedLocalSocket rpc_client; + using namespace std::chrono_literals; + rpc_client.connect(); + rpc_client.send_in_chunks<3>(json); + auto resp = rpc_client.read(); + REQUIRE(resp == R"({"jsonrpc": "2.0", "result": {"size": ")" + std::to_string(S) + R"("}, "id": "chunk-parts-3"})"); + }()); + } + REQUIRE(rpc::test_remove_handler("do_nothing")); +} + +TEST_CASE("Test with chunks - disconnect after second part", "[socket][chunks]") +{ + REQUIRE(rpc::add_method_handler("do_nothing", &do_nothing)); + + SECTION("Sending request by chunks") + { + const int S{4000}; + auto json{R"({"jsonrpc": "2.0", "method": "do_nothing", "params": { "msg": ")" + random_string(S) + + R"("}, "id": "chunk-parts-3-2"})"}; + + REQUIRE_NOTHROW([&]() { + ScopedLocalSocket rpc_client; + using namespace std::chrono_literals; + rpc_client.connect(); + rpc_client.send_in_chunks<3>(json, 2); + // read will fail. + auto resp = rpc_client.read(); + REQUIRE(resp == ""); + }()); + } + REQUIRE(rpc::test_remove_handler("do_nothing")); +} + +TEST_CASE("Test with chunks - incomplete message", "[socket][chunks]") +{ + REQUIRE(rpc::add_method_handler("do_nothing", &do_nothing)); + + SECTION("Sending request by chunks, broken message") + { + const int S{50}; + auto json{R"({"jsonrpc": "2.0", "method": "do_nothing", "params": { "msg": ")" + random_string(S) + + R"("}, "id": "chunk-parts-3)"}; + // ^ missing-> "} + + REQUIRE_NOTHROW([&]() { + ScopedLocalSocket rpc_client; + using namespace std::chrono_literals; + rpc_client.connect(); + rpc_client.send_in_chunks<3>(json); + auto resp = rpc_client.read(); + REQUIRE(resp == R"({"jsonrpc": "2.0", "error": {"code": -32700, "message": "Parse error"}})"); + }()); + } + REQUIRE(rpc::test_remove_handler("do_nothing")); +} + +// Enable toggle +TEST_CASE("Test rpc enable toggle feature - default enabled.", "[default values]") +{ + rpc::config::RPCConfig serverConfig; + REQUIRE(serverConfig.is_enabled() == true); +} + +TEST_CASE("Test rpc enable toggle feature. Enabled by configuration", "[rpc][enabled]") +{ + rpc::config::RPCConfig serverConfig; + + auto confStr{R"({"rpc": {"enabled": true}})"}; + std::cout << "'" << confStr << "'" << std::endl; + YAML::Node configNode = YAML::Load(confStr); + serverConfig.load(configNode["rpc"]); + REQUIRE(serverConfig.is_enabled() == true); +} + +TEST_CASE("Test rpc enable toggle feature. Disabled by configuration", "[rpc][disabled]") +{ + rpc::config::RPCConfig serverConfig; + + auto confStr{R"({"rpc": {"enabled":false}})"}; + + REQUIRE_NOTHROW([&]() { + YAML::Node configNode = YAML::Load(confStr); + serverConfig.load(configNode["rpc"]); + }()); + REQUIRE(serverConfig.is_enabled() == false); +} + +// TEST UDS Server configuration +namespace +{ +namespace trp = rpc::comm; +// This class is defined to get access to the protected config object inside the IPCSocketServer class. +struct LocalSocketTest : public trp::IPCSocketServer { + inline static const std::string _name = "LocalSocketTest"; + bool + configure(YAML::Node const ¶ms) override + { + return trp::IPCSocketServer::configure(params); + } + void + run() override + { + } + std::error_code + init() override + { + return trp::IPCSocketServer::init(); + } + bool + stop() override + { + return true; + } + std::string const & + name() const override + { + return _name; + } + trp::IPCSocketServer::Config const & + get_conf() const + { + return _conf; + } +}; +} // namespace + +TEST_CASE("Test configuration parsing. UDS values", "[string]") +{ + rpc::config::RPCConfig serverConfig; + + auto confStr{R"({"rpc": { "enabled": true, "unix": { "lock_path_name": ")" + lockPath + R"(", "sock_path_name": ")" + sockPath + + R"(", "backlog": 5,"max_retry_on_transient_errors": 64 }}})"}; + YAML::Node configNode = YAML::Load(confStr); + serverConfig.load(configNode["rpc"]); + + REQUIRE(serverConfig.get_comm_type() == rpc::config::RPCConfig::CommType::UNIX); + + auto socket = std::make_unique(); + auto const ret = socket->configure(serverConfig.get_comm_config_params()); + REQUIRE(ret); + REQUIRE(socket->get_conf().backlog == default_backlog); + REQUIRE(socket->get_conf().maxRetriesOnTransientErrors == default_maxRetriesOnTransientErrors); + REQUIRE(socket->get_conf().sockPathName == sockPath); + REQUIRE(socket->get_conf().lockPathName == lockPath); +} + +TEST_CASE("Test configuration parsing from a file. UDS Server", "[file]") +{ + namespace fs = ts::file; + + fs::path sandboxDir = fs::temp_directory_path(); + fs::path configPath = sandboxDir / "jsonrpc.yaml"; + + // define here to later compare. + std::string sockPathName{configPath.string() + "jsonrpc20_test2.sock"}; + std::string lockPathName{configPath.string() + "jsonrpc20_test2.lock"}; + + auto confStr{R"({"rpc": { "enabled": true, "unix": { "lock_path_name": ")" + lockPathName + R"(", "sock_path_name": ")" + + sockPathName + R"(", "backlog": 5,"max_retry_on_transient_errors": 64 }}})"}; + // write the config. + std::ofstream ofs(configPath.string(), std::ofstream::out); + // Yes, we write json into the yaml, remember, YAML is a superset of JSON, yaml parser can handle this. + ofs << confStr; + ofs.close(); + + rpc::config::RPCConfig serverConfig; + // on any error reading the file, default values will be used. + serverConfig.load_from_file(configPath.string()); + + REQUIRE(serverConfig.get_comm_type() == rpc::config::RPCConfig::CommType::UNIX); + + auto socket = std::make_unique(); + auto const &ret = socket->configure(serverConfig.get_comm_config_params()); + REQUIRE(ret); + REQUIRE(socket->get_conf().backlog == 5); + REQUIRE(socket->get_conf().maxRetriesOnTransientErrors == 64); + REQUIRE(socket->get_conf().sockPathName == sockPathName); + REQUIRE(socket->get_conf().lockPathName == lockPathName); + + std::error_code ec; + REQUIRE(fs::remove(sandboxDir, ec)); +} diff --git a/mgmt2/rpc/server/unit_tests/unit_test_main.cc b/mgmt2/rpc/server/unit_tests/unit_test_main.cc new file mode 100644 index 00000000000..76e51e02262 --- /dev/null +++ b/mgmt2/rpc/server/unit_tests/unit_test_main.cc @@ -0,0 +1,22 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#define CATCH_CONFIG_MAIN +#include diff --git a/rc/trafficserver.conf.in b/rc/trafficserver.conf.in index 6d2aef2ab49..ed647760a14 100644 --- a/rc/trafficserver.conf.in +++ b/rc/trafficserver.conf.in @@ -37,4 +37,4 @@ pre-start script fi end script -exec @exp_bindir@/traffic_manager +exec @exp_bindir@/traffic_server diff --git a/rc/trafficserver.in b/rc/trafficserver.in index dc0093ba2e3..02b325bbfdb 100644 --- a/rc/trafficserver.in +++ b/rc/trafficserver.in @@ -109,15 +109,11 @@ TS_ROOT=${TS_ROOT:-$TS_PREFIX} # For standard installations TS_BASE will be empty eval TS_BASE="`echo $TS_ROOT | ${ESED} -e 's;@prefix@$;;'`" -TM_NAME=${TM_NAME:-traffic_manager} TS_NAME=${TS_NAME:-traffic_server} -TM_DAEMON=${TM_DAEMON:-$TS_BASE@exp_bindir@/traffic_manager} -TM_DAEMON_ARGS="" TS_DAEMON=${TS_DAEMON:-$TS_BASE@exp_bindir@/traffic_server} TS_DAEMON_ARGS="" -TL_BINARY=${TL_BINARY:-$TS_BASE@exp_bindir@/traffic_ctl} -TY_BINARY=${TL_BINARY:-$TS_BASE@exp_bindir@/traffic_layout} -TM_PIDFILE=${TM_PIDFILE:-$TS_BASE@exp_runtimedir@/manager.lock} +TC_BINARY=${TC_BINARY:-$TS_BASE@exp_bindir@/traffic_ctl} +TY_BINARY=${TC_BINARY:-$TS_BASE@exp_bindir@/traffic_layout} TS_PIDFILE=${TS_PIDFILE:-$TS_BASE@exp_runtimedir@/server.lock} # number of times to retry check on pid lock file PIDFILE_CHECK_RETRIES=${PIDFILE_CHECK_RETRIES:-30} @@ -206,8 +202,8 @@ forkdaemon() while (( $i < $PIDFILE_CHECK_RETRIES )) do # check for regular file and size greater than 0 - if [[ -f $TM_PIDFILE ]] && [[ -s $TM_PIDFILE ]]; then - success || true + if [[ -f $TS_PIDFILE ]] && [[ -s $TS_PIDFILE ]]; then + success return 0 fi @@ -230,16 +226,16 @@ do_start() # 0 if daemon has been started # 1 if daemon was already running # 2 if daemon could not be started - start-stop-daemon --start --quiet --pidfile $TM_PIDFILE --exec $TM_DAEMON --test > /dev/null \ + start-stop-daemon --start --quiet --pidfile $TS_PIDFILE --exec $TS_DAEMON --test > /dev/null \ || return 1 - start-stop-daemon --start --background --quiet --pidfile $TM_PIDFILE --exec $TM_DAEMON -- \ - $TM_DAEMON_ARGS \ + start-stop-daemon --start --background --quiet --pidfile $TS_PIDFILE --exec $TS_DAEMON -- \ + $TS_DAEMON_ARGS \ || return 2 # Add code here, if necessary, that waits for the process to be ready # to handle requests from services started subsequently which depend # on this one. As a last resort, sleep for some time. sleep 1 - test -f "$TM_PIDFILE" || return 2 + test -f "$TS_PIDFILE" || return 2 } # @@ -252,7 +248,7 @@ do_stop() # 1 if daemon was already stopped # 2 if daemon could not be stopped # other if a failure occurred - start-stop-daemon --stop --quiet --retry=QUIT/30/KILL/5 --pidfile $TM_PIDFILE --name $TM_NAME + start-stop-daemon --stop --quiet --retry=QUIT/30/KILL/5 --pidfile $TS_PIDFILE --name $TS_NAME RETVAL="$?" test "$RETVAL" != 0 && return $RETVAL # Wait for children to finish too if this is a daemon that forks @@ -261,19 +257,14 @@ do_stop() # that waits for the process to drop all resources that could be # needed by services started subsequently. A last resort is to # sleep for some time. - start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $TM_DAEMON - RETVAL="$?" - test "$RETVAL" != 0 && return $RETVAL - # Need to stop the TM and TS also - start-stop-daemon --stop --quiet --oknodo --retry=QUIT/30/KILL/5 --pidfile $TM_PIDFILE --name $TM_NAME + start-stop-daemon --stop --quiet --oknodo --retry=0/30/KILL/5 --exec $TS_DAEMON RETVAL="$?" test "$RETVAL" != 0 && return $RETVAL + # Need to stop the TS start-stop-daemon --stop --quiet --oknodo --retry=QUIT/30/KILL/5 --pidfile $TS_PIDFILE --name $TS_NAME RETVAL="$?" test "$RETVAL" != 0 && return $RETVAL # Many daemons don't delete their pidfiles when they exit. - rm -f $TM_PIDFILE - rm -f $TM_PIDFILE rm -f $TS_PIDFILE return "$RETVAL" } @@ -322,24 +313,24 @@ case "$1" in do_start eend $? elif [ "$DISTRIB_ID" = "fedora" -o "$DISTRIB_ID" = "redhat" ]; then - action "Starting ${TS_PACKAGE_NAME}:" forkdaemon $TM_DAEMON $TM_DAEMON_ARGS + action "Starting ${TS_PACKAGE_NAME}:" forkdaemon $TS_DAEMON $TS_DAEMON_ARGS elif [ "$DISTRIB_ID" = "suse" ]; then echo -n "Starting ${TS_PACKAGE_NAME}" - startproc -p $TM_PIDFILE $TM_DAEMON $TM_DAEMON_ARGS + startproc -p $TS_PIDFILE $TS_DAEMON $TS_DAEMON_ARGS rc_status -v elif [ "$DISTRIB_ID" = "nixos" ]; then echo "Starting ${TS_PACKAGE_NAME}" forkdaemon $TM_DAEMON $TM_DAEMON_ARGS elif [ "$DISTRIB_ID" = "Darwin" ]; then echo "Starting ${TS_PACKAGE_NAME}" - launchctl bsexec / launchctl list $TM_NAME > /dev/null 2>&1 && exit 0 - launchctl bsexec / launchctl submit -l $TM_NAME -p $TM_DAEMON -o $STDOUTLOG -e $STDERRLOG -- $TM_DAEMON_ARGS + launchctl bsexec / launchctl list $TS_NAME > /dev/null 2>&1 && exit 0 + launchctl bsexec / launchctl submit -l $TS_NAME -p $TS_DAEMON -o $STDOUTLOG -e $STDERRLOG -- $TS_DAEMON_ARGS elif [ "$DISTRIB_ID" = "FreeBSD" ]; then echo "Starting ${TS_PACKAGE_NAME}" - name="$TM_NAME" + name="$TS_NAME" command="/usr/sbin/daemon" - command_args="-o $STDOUTLOG $TM_DAEMON $TM_DAEMON_ARGS" - pidfile="$TM_PIDFILE" + command_args="-o $STDOUTLOG $TS_DAEMON $TS_DAEMON_ARGS" + pidfile="$TS_PIDFILE" run_rc_command "$1" else echo "This script needs to be ported to this OS" @@ -355,16 +346,12 @@ case "$1" in test "x$VERBOSE" != "xno" && log_end_msg "$retval" exit "$retval" elif [ "$DISTRIB_ID" = "fedora" -o "$DISTRIB_ID" = "redhat" ]; then - action "Stopping ${TM_NAME}:" killproc -p $TM_PIDFILE -d 35 $TM_DAEMON action "Stopping ${TS_NAME}:" killproc -p $TS_PIDFILE -d 35 $TS_DAEMON elif [ "$DISTRIB_ID" = "gentoo" ]; then ebegin "Stopping ${TS_PACKAGE_NAME}" do_stop eend $? elif [ "$DISTRIB_ID" = "suse" ]; then - echo -n "Stopping ${TM_NAME}" - killproc -p $TM_PIDFILE $TM_DAEMON - rc_status -v echo -n "Stopping ${TS_NAME}" killproc -p $TS_PIDFILE $TS_DAEMON rc_status -v @@ -380,18 +367,15 @@ case "$1" in fi elif [ "$DISTRIB_ID" = "Darwin" ]; then echo "Stopping ${TS_PACKAGE_NAME}" - launchctl bsexec / launchctl list $TM_NAME > /dev/null 2>&1 || exit 0 - echo "Stopping ${TM_NAME}" - launchctl bsexec / launchctl remove ${TM_NAME} - rm -f ${TM_PIDFILE} + launchctl bsexec / launchctl list $TS_NAME > /dev/null 2>&1 || exit 0 echo "Stopping ${TS_NAME}" - kill $(cat $TS_PIDFILE) + launchctl bsexec / launchctl remove ${TS_NAME} rm -f ${TS_PIDFILE} elif [ "$DISTRIB_ID" = "FreeBSD" ]; then echo "Stopping ${TS_PACKAGE_NAME}" - if [ -e "$TM_PIDFILE" ]; then - kill $(cat $TM_PIDFILE) - rm -f ${TM_PIDFILE} + if [ -e "$TS_PIDFILE" ]; then + kill $(cat $TS_PIDFILE) + rm -f ${TS_PIDFILE} fi else echo "This script needs to be ported to this OS" @@ -409,29 +393,29 @@ case "$1" in if [ "$DISTRIB_ID" = "ubuntu" -o "$DISTRIB_ID" = "debian" ] ; then test "x$VERBOSE" != "xno" && log_daemon_msg "Reloading ${TS_PACKAGE_NAME}" "$NAME" retval=0 - $TL_BINARY config reload + $TC_BINARY config reload test "$?" -ne 0 -a "$?" -ne 1 && retval=1 test "x$VERBOSE" != "xno" && log_end_msg "$retval" exit "$retval" elif [ "$DISTRIB_ID" = "fedora" -o "$DISTRIB_ID" = "redhat" ]; then - action "Reloading ${NAME}:" $TL_BINARY config reload + action "Reloading ${NAME}:" $TC_BINARY config reload elif [ "$DISTRIB_ID" = "gentoo" ]; then ebegin "Reloading ${NAME}" - $TL_BINARY config reload + $TC_BINARY config reload eend $? elif [ "$DISTRIB_ID" = "suse" ]; then echo -n "Reloading ${NAME}" - $TL_BINARY config reload + $TC_BINARY config reload rc_status -v elif [ "$DISTRIB_ID" = "nixos" ]; then echo "Reloading ${NAME}" $TL_BINARY config reload elif [ "$DISTRIB_ID" = "Darwin" ]; then echo "Reloading ${NAME}" - $TL_BINARY config reload + $TC_BINARY config reload elif [ "$DISTRIB_ID" = "FreeBSD" ]; then echo "Reloading ${NAME}" - $TL_BINARY config reload + $TC_BINARY config reload else echo "This script needs to be ported to this OS" exit 1 @@ -469,24 +453,28 @@ case "$1" in ;; status) if [ "$DISTRIB_ID" = "fedora" -o "$DISTRIB_ID" = "redhat" ]; then - status -p $TM_PIDFILE $TM_NAME + status -p $TS_PIDFILE $TS_NAME elif [ "$DISTRIB_ID" = "ubuntu" -o "$DISTRIB_ID" = "debian" ] ; then - status_of_proc "$TM_DAEMON" "$TM_NAME" -p "$TM_PIDFILE" && exit 0 || exit $? + status_of_proc "$TS_DAEMON" "$TS_NAME" -p "$TS_PIDFILE" && exit 0 || exit $? elif [ "$DISTRIB_ID" = "suse" ]; then echo -n "Checking for service ${DM}: " - checkproc -p $TM_PIDFILE $TM_NAME + checkproc -p $TS_PIDFILE $TS_NAME rc_status -v elif [ "$DISTRIB_ID" = "Darwin" ]; then /bin/echo -n "${TS_PACKAGE_NAME} is " - launchctl bsexec / launchctl list $TM_NAME > /dev/null 2>&1 + launchctl bsexec / launchctl list $TS_NAME > /dev/null 2>&1 status=$? [ $status -eq 0 ] || /bin/echo -n "not " echo "running." +<<<<<<< HEAD elif [ "$DISTRIB_ID" = "FreeBSD" -o "$DISTRIB_ID" = "gentoo" -o "$DISTRIB_ID" = "nixos" ]; then if pgrep $TM_NAME > /dev/null ; then echo "$TM_NAME running as pid `cat $TM_PIDFILE`" ; else echo "$TM_NAME not running" fi +======= + elif [ "$DISTRIB_ID" = "FreeBSD" -o "$DISTRIB_ID" = "gentoo" ]; then +>>>>>>> 24778a1... JSONRPC - Adapt rc/* files to work with TS instead of TM if pgrep $TS_NAME > /dev/null ; then echo "$TS_NAME running as pid `cat $TS_PIDFILE`"; else echo "$TS_NAME not running" diff --git a/rc/trafficserver.service.in b/rc/trafficserver.service.in index a503a49d8cf..e5ce1ab8cf1 100644 --- a/rc/trafficserver.service.in +++ b/rc/trafficserver.service.in @@ -22,11 +22,14 @@ After=network.target [Service] Type=simple EnvironmentFile=-/etc/sysconfig/trafficserver -ExecStart=@exp_bindir@/traffic_manager $TM_DAEMON_ARGS +ExecStart=@exp_bindir@/traffic_server $TS_DAEMON_ARGS Restart=on-failure RestartSec=5s LimitNOFILE=1000000 -PIDFile=@exp_runtimedir@/manager.lock +ExecStopPost=/bin/sh -c ' \ + export TS_PIDFILE=$(@exp_bindir@/traffic_layout 2>/dev/null | grep RUNTIMEDIR | cut -d: -f2)/server.lock ; \ + /bin/rm -f $TS_PIDFILE ; \ + if [[ $? -ne 0 ]]; then echo "ERROR: Unable to delete PID"; exit 1; fi' TimeoutStopSec=5s ExecReload=@exp_bindir@/traffic_ctl config reload KillMode=process diff --git a/src/Makefile.am b/src/Makefile.am index b6a1b98bd2c..c230c75ceba 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -37,10 +37,18 @@ include traffic_manager/Makefile.inc include traffic_server/Makefile.inc include traffic_logstats/Makefile.inc include traffic_crashlog/Makefile.inc -include traffic_ctl/Makefile.inc include traffic_layout/Makefile.inc include traffic_logcat/Makefile.inc +# Build non jsonrpc traffic_ctl +if BUILD_LEGACY_TC +include traffic_ctl/Makefile.inc +else +include traffic_ctl_jsonrpc/Makefile.inc +endif + + + if ENABLE_QUIC include traffic_quic/Makefile.inc endif diff --git a/src/shared/rpc/IPCSocketClient.cc b/src/shared/rpc/IPCSocketClient.cc new file mode 100644 index 00000000000..7455b8e985c --- /dev/null +++ b/src/shared/rpc/IPCSocketClient.cc @@ -0,0 +1,95 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "shared/rpc/IPCSocketClient.h" +#include +#include + +#include +#include + +namespace shared::rpc +{ +IPCSocketClient::self_reference +IPCSocketClient::connect() +{ + _sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (this->is_closed()) { + std::string text; + ts::bwprint(text, "connect: error creating new socket. Why?: {}\n", std::strerror(errno)); + throw std::runtime_error{text}; + } + _server.sun_family = AF_UNIX; + std::strncpy(_server.sun_path, _path.c_str(), sizeof(_server.sun_path) - 1); + if (::connect(_sock, (struct sockaddr *)&_server, sizeof(struct sockaddr_un)) < 0) { + this->close(); + std::string text; + ts::bwprint(text, "connect: Couldn't open connection with {}. Why?: {}\n", _path, std::strerror(errno)); + throw std::runtime_error{text}; + } + + return *this; +} + +IPCSocketClient::self_reference +IPCSocketClient ::send(std::string_view data) +{ + std::string msg{data}; + if (::write(_sock, msg.c_str(), msg.size()) < 0) { + this->close(); + std::string text; + throw std::runtime_error{ts::bwprint(text, "Error writing on stream socket {}", std ::strerror(errno))}; + } + + return *this; +} + +IPCSocketClient::ReadStatus +IPCSocketClient::read_all(ts::FixedBufferWriter &bw) +{ + if (this->is_closed()) { + // we had a failure. + return {}; + } + ReadStatus readStatus{ReadStatus::UNKNOWN}; + while (bw.remaining()) { + ts::MemSpan span{bw.auxBuffer(), bw.remaining()}; + const ssize_t ret = ::read(_sock, span.data(), span.size()); + if (ret > 0) { + bw.fill(ret); + if (bw.remaining() > 0) { // some space available. + continue; + } else { + // buffer full. + readStatus = ReadStatus::BUFFER_FULL; + break; + } + } else { + if (bw.size()) { + // data was read. + readStatus = ReadStatus::NO_ERROR; + break; + } + readStatus = ReadStatus::STREAM_ERROR; + break; + } + } + return readStatus; +} +} // namespace shared::rpc \ No newline at end of file diff --git a/src/traffic_crashlog/procinfo.cc b/src/traffic_crashlog/procinfo.cc index 1180e031503..db99549d5bd 100644 --- a/src/traffic_crashlog/procinfo.cc +++ b/src/traffic_crashlog/procinfo.cc @@ -141,77 +141,6 @@ crashlog_write_datime(FILE *fp, const crashlog_target &target) return true; } -bool -crashlog_write_backtrace(FILE *fp, const crashlog_target &) -{ - TSString trace = nullptr; - TSMgmtError mgmterr; - - // NOTE: sometimes we can't get a backtrace because the ptrace attach will fail with - // EPERM. I've seen this happen when a debugger is attached, which makes sense, but it - // can also happen without a debugger. Possibly in that case, there is a race with the - // kernel locking the process information? - - if ((mgmterr = TSProxyBacktraceGet(0, &trace)) != TS_ERR_OKAY) { - char *msg = TSGetErrorMessage(mgmterr); - fprintf(fp, "Unable to retrieve backtrace: %s\n", msg); - TSfree(msg); - return false; - } - - fprintf(fp, "%s", trace); - TSfree(trace); - return true; -} - -bool -crashlog_write_records(FILE *fp, const crashlog_target &) -{ - TSMgmtError mgmterr; - TSList list = TSListCreate(); - bool success = false; - - if ((mgmterr = TSRecordGetMatchMlt(".", list)) != TS_ERR_OKAY) { - char *msg = TSGetErrorMessage(mgmterr); - fprintf(fp, "Unable to retrieve Traffic Server records: %s\n", msg); - TSfree(msg); - goto done; - } - - // If the RPC call failed, the list will be empty, so we won't print anything. Otherwise, - // print all the results, freeing them as we go. - for (TSRecordEle *rec_ele = (TSRecordEle *)TSListDequeue(list); rec_ele; rec_ele = (TSRecordEle *)TSListDequeue(list)) { - if (!success) { - success = true; - fprintf(fp, "Traffic Server Configuration Records:\n"); - } - - switch (rec_ele->rec_type) { - case TS_REC_INT: - fprintf(fp, "%s %" PRId64 "\n", rec_ele->rec_name, rec_ele->valueT.int_val); - break; - case TS_REC_COUNTER: - fprintf(fp, "%s %" PRId64 "\n", rec_ele->rec_name, rec_ele->valueT.counter_val); - break; - case TS_REC_FLOAT: - fprintf(fp, "%s %f\n", rec_ele->rec_name, rec_ele->valueT.float_val); - break; - case TS_REC_STRING: - fprintf(fp, "%s %s\n", rec_ele->rec_name, rec_ele->valueT.string_val); - break; - default: - // just skip it ... - break; - } - - TSRecordEleDestroy(rec_ele); - } - -done: - TSListDestroy(list); - return success; -} - bool crashlog_write_siginfo(FILE *fp, const crashlog_target &target) { diff --git a/src/traffic_crashlog/traffic_crashlog.cc b/src/traffic_crashlog/traffic_crashlog.cc index 936ccefda75..8fb86f0cb65 100644 --- a/src/traffic_crashlog/traffic_crashlog.cc +++ b/src/traffic_crashlog/traffic_crashlog.cc @@ -134,7 +134,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) { FILE *fp; char *logname; - TSMgmtError mgmterr; crashlog_target target; pid_t parent = getppid(); @@ -195,13 +194,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) Note("crashlog started, target=%ld, debug=%s syslog=%s, uid=%ld euid=%ld", static_cast(target_pid), debug_mode ? "true" : "false", syslog_mode ? "true" : "false", (long)getuid(), (long)geteuid()); - mgmterr = TSInit(nullptr, (TSInitOptionT)(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); - if (mgmterr != TS_ERR_OKAY) { - char *msg = TSGetErrorMessage(mgmterr); - Warning("failed to initialize management API: %s", msg); - TSfree(msg); - } - ink_zero(target); target.pid = static_cast(target_pid); target.timestamp = timestamp(); @@ -246,9 +238,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) fprintf(fp, "\n"); crashlog_write_registers(fp, target); - fprintf(fp, "\n"); - crashlog_write_backtrace(fp, target); - fprintf(fp, "\n"); crashlog_write_procstatus(fp, target); @@ -258,9 +247,6 @@ main(int /* argc ATS_UNUSED */, const char **argv) fprintf(fp, "\n"); crashlog_write_regions(fp, target); - fprintf(fp, "\n"); - crashlog_write_records(fp, target); - Error("wrote crash log to %s", logname); ats_free(logname); diff --git a/src/traffic_crashlog/traffic_crashlog.h b/src/traffic_crashlog/traffic_crashlog.h index ba17f5a7611..37dded2eb5a 100644 --- a/src/traffic_crashlog/traffic_crashlog.h +++ b/src/traffic_crashlog/traffic_crashlog.h @@ -27,7 +27,6 @@ #include "tscore/ink_memory.h" #include "tscore/Diags.h" #include "tscore/TextBuffer.h" -#include "mgmtapi.h" // ucontext.h is deprecated on Darwin, and we really only need it on Linux, so only // include it if we are planning to use it. diff --git a/src/traffic_ctl_jsonrpc/CtrlCommands.cc b/src/traffic_ctl_jsonrpc/CtrlCommands.cc new file mode 100644 index 00000000000..a71af91d534 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/CtrlCommands.cc @@ -0,0 +1,516 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +#include +#include + +#include "CtrlCommands.h" +#include "jsonrpc/CtrlRPCRequests.h" +#include "jsonrpc/ctrl_yaml_codecs.h" +namespace +{ +/// We use yamlcpp as codec implementation. +using Codec = yamlcpp_json_emitter; + +const std::unordered_map _Fmt_str_to_enum = { + {"pretty", BasePrinter::Options::Format::PRETTY}, {"legacy", BasePrinter::Options::Format::LEGACY}, + {"json", BasePrinter::Options::Format::JSON}, {"req", BasePrinter::Options::Format::DATA_REQ}, + {"resp", BasePrinter::Options::Format::DATA_RESP}, {"all", BasePrinter::Options::Format::DATA_ALL}}; + +BasePrinter::Options::Format +parse_format(ts::Arguments &args) +{ + if (args.get("records")) { + return BasePrinter::Options::Format::RECORDS; + } + + BasePrinter::Options::Format val{BasePrinter::Options::Format::LEGACY}; + + if (auto data = args.get("format"); data) { + ts::TextView fmt{data.value()}; + if ("data" == fmt.prefix(':')) { + fmt.take_prefix_at(':'); + } + if (auto search = _Fmt_str_to_enum.find(fmt); search != std::end(_Fmt_str_to_enum)) { + val = search->second; + } + } + return val; +} + +BasePrinter::Options +parse_print_opts(ts::Arguments &args) +{ + return {parse_format(args)}; +} +} // namespace + +//------------------------------------------------------------------------------------------------------------------------------------ +CtrlCommand::CtrlCommand(ts::Arguments args) : _arguments(args) {} + +void +CtrlCommand::execute() +{ + if (_invoked_func) { + _invoked_func(); + } +} + +std::string +CtrlCommand::invoke_rpc(std::string const &request) +{ + if (_printer->print_req_msg()) { + std::string text; + ts::bwprint(text, "--> {}", request); + _printer->write_debug(std::string_view{text}); + } + if (auto resp = _rpcClient.invoke(request); !resp.empty()) { + // all good. + if (_printer->print_resp_msg()) { + std::string text; + ts::bwprint(text, "<-- {}", resp); + _printer->write_debug(std::string_view{text}); + } + return resp; + } + + return {}; +} + +shared::rpc::JSONRPCResponse +CtrlCommand::invoke_rpc(shared::rpc::ClientRequest const &request) +{ + std::string encodedRequest = Codec::encode(request); + std::string resp = invoke_rpc(encodedRequest); + return Codec::decode(resp); +} + +void +CtrlCommand::invoke_rpc(shared::rpc::ClientRequest const &request, std::string &resp) +{ + std::string encodedRequest = Codec::encode(request); + resp = invoke_rpc(encodedRequest); +} +// ----------------------------------------------------------------------------------------------------------------------------------- +RecordCommand::RecordCommand(ts::Arguments args) : CtrlCommand(args) {} + +void +RecordCommand::execute() +{ + execute_subcommand(); +} +ConfigCommand::ConfigCommand(ts::Arguments args) : RecordCommand(args) +{ + BasePrinter::Options printOpts{parse_print_opts(_arguments)}; + if (args.get("match")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_match(); }; + } else if (args.get("get")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_get(); }; + } else if (args.get("diff")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_diff(); }; + } else if (args.get("describe")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_describe(); }; + } else if (args.get("defaults")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_defaults(); }; + } else if (args.get("set")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_set(); }; + } else if (args.get("status")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_status(); }; + } else if (args.get("reload")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_reload(); }; + } else if (args.get("registry")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { config_show_file_registry(); }; + } else { + // work in here. + } +} + +void +ConfigCommand::execute_subcommand() +{ + if (_invoked_func) { + _invoked_func(); + } +} + +shared::rpc::JSONRPCResponse +RecordCommand::record_fetch(ts::ArgumentData argData, bool isRegex, RecordQueryType recQueryType) +{ + shared::rpc::RecordLookupRequest request; + for (auto &&it : argData) { + request.emplace_rec(it, isRegex, + recQueryType == RecordQueryType::CONFIG ? shared::rpc::CONFIG_REC_TYPES : shared::rpc::METRIC_REC_TYPES); + } + return invoke_rpc(request); +} + +void +ConfigCommand::config_match() +{ + _printer->write_output(record_fetch(_arguments.get("match"), shared::rpc::REGEX, RecordQueryType::CONFIG)); +} + +void +ConfigCommand::config_get() +{ + _printer->write_output(record_fetch(_arguments.get("get"), shared::rpc::NOT_REGEX, RecordQueryType::CONFIG)); +} + +void +ConfigCommand::config_describe() +{ + _printer->write_output(record_fetch(_arguments.get("describe"), shared::rpc::NOT_REGEX, RecordQueryType::CONFIG)); +} +void +ConfigCommand::config_defaults() +{ + const bool configs{true}; + shared::rpc::JSONRPCResponse response = invoke_rpc(GetAllRecordsRequest{configs}); + _printer->write_output(response); +} +void +ConfigCommand::config_diff() +{ + GetAllRecordsRequest request{true}; + shared::rpc::JSONRPCResponse response = invoke_rpc(request); + _printer->write_output(response); +} + +void +ConfigCommand::config_status() +{ + ConfigStatusRequest request; + shared::rpc::JSONRPCResponse response = invoke_rpc(request); + _printer->write_output(response); +} + +void +ConfigCommand::config_set() +{ + auto const &data = _arguments.get("set"); + ConfigSetRecordRequest request{{data[0], data[1]}}; + shared::rpc::JSONRPCResponse response = invoke_rpc(request); + + _printer->write_output(response); +} +void +ConfigCommand::config_reload() +{ + _printer->write_output(invoke_rpc(ConfigReloadRequest{})); +} +void +ConfigCommand::config_show_file_registry() +{ + _printer->write_output(invoke_rpc(ConfigShowFileRegistryRequest{})); +} +//------------------------------------------------------------------------------------------------------------------------------------ +MetricCommand::MetricCommand(ts::Arguments args) : RecordCommand(args) +{ + // auto const fmt = parse_format(_arguments); + BasePrinter::Options printOpts{parse_print_opts(_arguments)}; + if (args.get("match")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { metric_match(); }; + } else if (args.get("get")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { metric_get(); }; + } else if (args.get("describe")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { metric_describe(); }; + } else if (args.get("clear")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { metric_clear(); }; + } else if (args.get("zero")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { metric_zero(); }; + } +} + +void +MetricCommand::execute_subcommand() +{ + if (_invoked_func) { + _invoked_func(); + } +} + +void +MetricCommand::metric_get() +{ + _printer->write_output(record_fetch(_arguments.get("get"), shared::rpc::NOT_REGEX, RecordQueryType::METRIC)); +} + +void +MetricCommand::metric_match() +{ + _printer->write_output(record_fetch(_arguments.get("match"), shared::rpc::REGEX, RecordQueryType::METRIC)); +} + +void +MetricCommand::metric_describe() +{ + _printer->write_output(record_fetch(_arguments.get("describe"), shared::rpc::NOT_REGEX, RecordQueryType::METRIC)); +} + +void +MetricCommand::metric_clear() +{ + [[maybe_unused]] auto const &response = invoke_rpc(ClearAllMetricRequest{}); +} + +void +MetricCommand::metric_zero() +{ + auto records = _arguments.get("zero"); + ClearMetricRequest request{// names + {{std::begin(records), std::end(records)}}}; + + [[maybe_unused]] auto const &response = invoke_rpc(request); +} +//------------------------------------------------------------------------------------------------------------------------------------ +// TODO, let call the super const +HostCommand::HostCommand(ts::Arguments args) : CtrlCommand(args) +{ + BasePrinter::Options printOpts{parse_print_opts(_arguments)}; + if (_arguments.get("status")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { status_get(); }; + } else if (_arguments.get("down")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { status_down(); }; + } else if (_arguments.get("up")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { status_up(); }; + } +} + +void +HostCommand::status_get() +{ + auto const &data = _arguments.get("status"); + HostGetStatusRequest request; + for (auto it : data) { + std::string name = std::string{HostGetStatusRequest::STATUS_PREFIX} + "." + it; + request.emplace_rec(name, shared::rpc::NOT_REGEX, shared::rpc::METRIC_REC_TYPES); + } + auto response = invoke_rpc(request); + + _printer->write_output(response); +} + +void +HostCommand::status_down() +{ + auto hosts = _arguments.get("down"); + HostSetStatusRequest request{ + {HostSetStatusRequest::Params::Op::DOWN, {std::begin(hosts), std::end(hosts)}, _arguments.get("reason").value(), "0"}}; + auto response = invoke_rpc(request); + _printer->write_output(response); +} + +void +HostCommand::status_up() +{ + auto hosts = _arguments.get("up"); + HostSetStatusRequest request{ + {HostSetStatusRequest::Params::Op::UP, {std::begin(hosts), std::end(hosts)}, _arguments.get("reason").value(), "0"}}; + + auto response = invoke_rpc(request); + _printer->write_output(response); +} +//------------------------------------------------------------------------------------------------------------------------------------ +PluginCommand::PluginCommand(ts::Arguments args) : CtrlCommand(args) +{ + if (_arguments.get("msg")) { + _invoked_func = [&]() { plugin_msg(); }; + } + _printer = std::make_unique(parse_print_opts(_arguments)); +} + +void +PluginCommand::plugin_msg() +{ + auto msgs = _arguments.get("msg"); + BasicPluginMessageRequest::Params params; + params.tag = msgs[0]; + if (msgs.size() > 1) { + // have a value + params.str = msgs[1]; + } + BasicPluginMessageRequest request{params}; + auto response = invoke_rpc(request); +} +//------------------------------------------------------------------------------------------------------------------------------------ +DirectRPCCommand::DirectRPCCommand(ts::Arguments args) : CtrlCommand(args) +{ + BasePrinter::Options printOpts{parse_print_opts(_arguments)}; + + if (_arguments.get("get-api")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { get_rpc_api(); }; + return; + } else if (_arguments.get("file")) { + _invoked_func = [&]() { from_file_request(); }; + } else if (_arguments.get("input")) { + _invoked_func = [&]() { read_from_input(); }; + } + + _printer = std::make_unique(printOpts); +} + +bool +DirectRPCCommand::validate_input(std::string const &in) const +{ + // validate the input + YAML::Node content = YAML::Load(in); + if (content.Type() != YAML::NodeType::Map && content.Type() != YAML::NodeType::Sequence) { + return false; + } + + return true; +} + +void +DirectRPCCommand::from_file_request() +{ + // TODO: remove all the output messages from here if possible + auto filenames = _arguments.get("file"); + for (auto &&filename : filenames) { + std::string text; + // run some basic validation on the passed files, they should + try { + std::ifstream file(filename); + std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + + if (!validate_input(content)) { + _printer->write_output( + ts::bwprint(text, "Content not accepted. expecting a valid sequence or structure. {} skipped.\n", filename)); + continue; + } + std::string const &response = invoke_rpc(content); + if (_printer->is_json_format()) { + // as we have the raw json in here, we cna just directly print it + _printer->write_output(response); + } else { + _printer->write_output(ts::bwprint(text, "\n[ {} ]\n --> \n{}\n", filename, content)); + _printer->write_output(ts::bwprint(text, "<--\n{}\n", response)); + } + + } catch (std::exception const &ex) { + _printer->write_output(ts::bwprint(text, "Error found: {}\n", ex.what())); + } + } +} + +void +DirectRPCCommand::get_rpc_api() +{ + auto response = invoke_rpc(ShowRegisterHandlersRequest{}); + _printer->write_output(response); +} + +void +DirectRPCCommand::read_from_input() +{ + // TODO: remove all the output messages from here if possible + std::string text; + + try { + _printer->write_output(">> Ctrl-D to fire the request. Ctrl-C to exit\n"); + std::cin >> std::noskipws; + // read cin. + std::string content((std::istreambuf_iterator(std::cin)), std::istreambuf_iterator()); + if (!_arguments.get("raw") && !validate_input(content)) { + _printer->write_output(ts::bwprint(text, "Content not accepted. expecting a valid sequence or structure\n")); + return; + } + std::string const &response = invoke_rpc(content); + _printer->write_output("--> Request sent.\n"); + _printer->write_output(ts::bwprint(text, "\n<-- {}\n", response)); + } catch (std::exception const &ex) { + _printer->write_output(ts::bwprint(text, "Error found: {}\n", ex.what())); + } +} + +//------------------------------------------------------------------------------------------------------------------------------------ +ServerCommand::ServerCommand(ts::Arguments args) : CtrlCommand(args) +{ + BasePrinter::Options printOpts{parse_print_opts(_arguments)}; + if (_arguments.get("drain")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { server_drain(); }; + } +} + +void +ServerCommand::server_drain() +{ + shared::rpc::JSONRPCResponse response; + // TODO, can call_request take a && ?? if needed in the cmd just pass by ref. + + if (_arguments.get("undo")) { + response = invoke_rpc(ServerStopDrainRequest{}); + } else { + bool newConn = _arguments.get("no-new-connection"); + ServerStartDrainRequest request{{newConn}}; + response = invoke_rpc(request); + } + + _printer->write_output(response); +} +// //------------------------------------------------------------------------------------------------------------------------------------ +StorageCommand::StorageCommand(ts::Arguments args) : CtrlCommand(args) +{ + BasePrinter::Options printOpts{parse_print_opts(_arguments)}; + if (_arguments.get("status")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { get_storage_status(); }; + } else if (_arguments.get("offline")) { + _printer = std::make_unique(printOpts); + _invoked_func = [&]() { set_storage_offline(); }; + } +} + +void +StorageCommand::get_storage_status() +{ + auto disks = _arguments.get("status"); + GetStorageDeviceStatusRequest request{{{std::begin(disks), std::end(disks)}}}; + auto response = invoke_rpc(request); + _printer->write_output(response); +} +void +StorageCommand::set_storage_offline() +{ + auto disks = _arguments.get("offline"); + SetStorageDeviceOfflineRequest request{{{std::begin(disks), std::end(disks)}}}; + auto response = invoke_rpc(request); + _printer->write_output(response); +} +// //------------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/traffic_ctl_jsonrpc/CtrlCommands.h b/src/traffic_ctl_jsonrpc/CtrlCommands.h new file mode 100644 index 00000000000..6144dedd709 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/CtrlCommands.h @@ -0,0 +1,187 @@ +/** +@section license License + +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#pragma once + +#include + +#include "tscore/ArgParser.h" + +#include "shared/rpc/RPCClient.h" +#include "CtrlPrinters.h" + +// ---------------------------------------------------------------------------------------------------------------------------------- +/// +/// @brief Base Control Command class. +/// This class should be used as a base class for every new command or group of commands that are related. +/// The base class will provide the client communication through the @c invoke_call member function. Arguments that were +/// parsed by the traffic_ctl are available as a member to all the derived classes. +class CtrlCommand +{ +public: + virtual ~CtrlCommand() = default; + + /// @brief This object will hold the arguments for now. + CtrlCommand(ts::Arguments args); + + /// @brief Main execution point for a particular command. This function will invoke @c _invoked_func which should be set + /// by the derived class. In case you do not want the @c _invoked_func to be called directly, you should override this + /// member function and call it yourself. @c RecordCommand does it and forwards the call to his childrens. + /// If @c _invoked_func is not properly set, the function will not be called. + virtual void execute(); + +protected: + /// @brief Invoke the remote server. This is the very basic function which does not play or interact with any codec. Request + /// and message should be already en|de coded. + /// @param request A string representation of the json/yaml request. + /// @return a string with the json/yaml response. + /// @note This function does print the raw string if requested by the "--format". No printer involved, standard output. + std::string invoke_rpc(std::string const &request); + + /// @brief Function that calls the rpc server. This function takes a json objects and uses the defined coded to convert them to a + /// string. This function will call invoke_rpc(string) overload. + /// @param A Client request. + /// @return A server response. + shared::rpc::JSONRPCResponse invoke_rpc(shared::rpc::ClientRequest const &request); + /// @brief Function that calls the rpc server. The response will not be decoded, it will be a raw string. + void invoke_rpc(shared::rpc::ClientRequest const &request, std::string &bw); + + ts::Arguments _arguments; //!< parsed traffic_ctl arguments. + std::unique_ptr _printer; //!< Specific output formatter. This should be created by the derived class. + + /// @brief The whole design is that the command will execute the @c _invoked_func once invoked. This function ptr should be + /// set by the appropriated derived class base on the passed parameters. The derived class have the option to override + /// the execute() function and do something else. Check @c RecordCommand as an example. + std::function _invoked_func; //!< Actual function that the command will execute. + +private: + shared::rpc::RPCClient _rpcClient; //!< RPC socket client implementation. +}; + +// ----------------------------------------------------------------------------------------------------------------------------------- +/// +/// @brief Record Command Implementation +/// Used as base class for any command that needs to access to a TS record. +/// If deriving from this class, make sure you implement @c execute_subcommand() and call the _invoked_func yourself. +class RecordCommand : public CtrlCommand +{ +public: + virtual ~RecordCommand() = default; + /// @brief RecordCommand constructor. + RecordCommand(ts::Arguments args); + /// @brief We will override this function as we want to call execute_subcommand() in the derived class. + void execute(); + +protected: + /// @brief Handy enum to hold which kind of records we are requesting. + enum class RecordQueryType { CONFIG = 0, METRIC }; + /// @brief Function to fetch record from the rpc server. + /// @param argData argument's data. + /// @param isRegex if the request should be done by regex or name. + /// @param recQueryType Config or Metric. + shared::rpc::JSONRPCResponse record_fetch(ts::ArgumentData argData, bool isRegex, RecordQueryType recQueryType); + + /// @brief To be override + virtual void + execute_subcommand() + { + } +}; +// ----------------------------------------------------------------------------------------------------------------------------------- +class ConfigCommand : public RecordCommand +{ + void config_match(); + void config_get(); + void config_describe(); + void config_defaults(); + void config_diff(); + void config_status(); + void config_set(); + void config_reload(); + void config_show_file_registry(); + +public: + ConfigCommand(ts::Arguments args); + void execute_subcommand(); +}; +// ----------------------------------------------------------------------------------------------------------------------------------- +class MetricCommand : public RecordCommand +{ + void metric_get(); + void metric_match(); + void metric_describe(); + void metric_clear(); + void metric_zero(); + +public: + MetricCommand(ts::Arguments args); + void execute_subcommand(); +}; +// ----------------------------------------------------------------------------------------------------------------------------------- +class HostCommand : public CtrlCommand +{ +public: + HostCommand(ts::Arguments args); + +private: + void status_get(); + void status_down(); + void status_up(); +}; +// ----------------------------------------------------------------------------------------------------------------------------------- +class PluginCommand : public CtrlCommand +{ +public: + PluginCommand(ts::Arguments args); + +private: + void plugin_msg(); +}; +// ----------------------------------------------------------------------------------------------------------------------------------- +class DirectRPCCommand : public CtrlCommand +{ +public: + DirectRPCCommand(ts::Arguments args); + +private: + void from_file_request(); + void get_rpc_api(); + void read_from_input(); + /// run a YAML validation on the input. + bool validate_input(std::string const &in) const; +}; +// ----------------------------------------------------------------------------------------------------------------------------------- +class ServerCommand : public CtrlCommand +{ +public: + ServerCommand(ts::Arguments args); + +private: + void server_drain(); +}; +// +// ----------------------------------------------------------------------------------------------------------------------------------- +struct StorageCommand : public CtrlCommand { + StorageCommand(ts::Arguments args); + +private: + void get_storage_status(); + void set_storage_offline(); +}; +// ----------------------------------------------------------------------------------------------------------------------------------- diff --git a/src/traffic_ctl_jsonrpc/CtrlPrinters.cc b/src/traffic_ctl_jsonrpc/CtrlPrinters.cc new file mode 100644 index 00000000000..6351926a14c --- /dev/null +++ b/src/traffic_ctl_jsonrpc/CtrlPrinters.cc @@ -0,0 +1,366 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#include "CtrlPrinters.h" + +#include +#include +#include + +#include "jsonrpc/ctrl_yaml_codecs.h" +#include "tscpp/util/ts_meta.h" +#include +#include "PrintUtils.h" + +//------------------------------------------------------------------------------------------------------------------------------------ + +namespace +{ +void +print_record_error_list(std::vector const &errors) +{ + if (errors.size()) { + std::cout << "------------ Errors ----------\n"; + auto iter = std::begin(errors); + if (iter != std::end(errors)) { + std::cout << *iter; + } + ++iter; + for (auto err = iter; err != std::end(errors); ++err) { + std::cout << "--\n"; + std::cout << *err; + } + } +} + +} // namespace +void +BasePrinter::write_output(shared::rpc::JSONRPCResponse const &response) +{ + // If json, then we print the full message, either ok or error. + if (this->is_json_format()) { + write_output_json(response.fullMsg); + return; + } + + if (response.is_error() && this->is_pretty_format()) { + // we print the error in this case. Already formatted. + std::cout << response.error.as(); + return; + } + + if (!response.result.IsNull()) { + // on you! + // Found convinient to let the derived class deal with the specifics. + write_output(response.result); + } +} + +void +BasePrinter::write_output(std::string_view output) const +{ + std::cout << output << '\n'; +} + +void +BasePrinter::write_debug(std::string_view output) const +{ + std::cout << output << '\n'; +} +void +BasePrinter::write_output_json(YAML::Node const &node) const +{ + YAML::Emitter out; + out << YAML::DoubleQuoted << YAML::Flow; + out << node; + write_output(std::string_view{out.c_str()}); +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +RecordPrinter::write_output(YAML::Node const &result) +{ + auto response = result.as(); + if (is_legacy_format()) { + write_output_legacy(response); + } else { + write_output_pretty(response); + } +} +void +RecordPrinter::write_output_legacy(shared::rpc::RecordLookUpResponse const &response) +{ + std::string text; + for (auto &&recordInfo : response.recordList) { + if (!recordInfo.registered) { + std::cout << recordInfo.name + << ": Unrecognized configuration value. Record is a configuration name/value but is not registered\n"; + continue; + } + if (!_printAsRecords) { + std::cout << recordInfo.name << ": " << recordInfo.currentValue << '\n'; + } else { + std::cout << ts::bwprint(text, "{} {} {} {} # default: {}\n", rec_labelof(recordInfo.rclass), recordInfo.name, + recordInfo.dataType, recordInfo.currentValue, recordInfo.defaultValue); + } + } + // we print errors if found. + print_record_error_list(response.errorList); +} +void +RecordPrinter::write_output_pretty(shared::rpc::RecordLookUpResponse const &response) +{ + write_output_legacy(response); +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +MetricRecordPrinter::write_output(YAML::Node const &result) +{ + auto response = result.as(); + for (auto &&recordInfo : response.recordList) { + std::cout << recordInfo.name << " " << recordInfo.currentValue << '\n'; + } +} +//------------------------------------------------------------------------------------------------------------------------------------ + +void +DiffConfigPrinter::write_output(YAML::Node const &result) +{ + std::string text; + auto response = result.as(); + for (auto &&recordInfo : response.recordList) { + auto const ¤tValue = recordInfo.currentValue; + auto const &defaultValue = recordInfo.defaultValue; + const bool hasChanged = (currentValue != defaultValue); + if (hasChanged) { + if (!_printAsRecords) { + std::cout << ts::bwprint(text, "{} has changed\n", recordInfo.name); + std::cout << ts::bwprint(text, "\tCurrent Value: {}\n", currentValue); + std::cout << ts::bwprint(text, "\tDefault Value: {}\n", defaultValue); + } else { + std::cout << ts::bwprint(text, "{} {} {} {} # default: {}\n", rec_labelof(recordInfo.rclass), recordInfo.name, + recordInfo.dataType, recordInfo.currentValue, recordInfo.defaultValue); + } + } + } +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +ConfigReloadPrinter::write_output(YAML::Node const &result) +{ +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +ConfigShowFileRegistryPrinter::write_output(YAML::Node const &result) +{ + if (is_pretty_format()) { + this->write_output_pretty(result); + } else { + if (auto registry = result["config_registry"]) { + write_output_json(registry); + } + } +} + +void +ConfigShowFileRegistryPrinter::write_output_pretty(YAML::Node const &result) +{ + if (auto &®istry = result["config_registry"]) { + for (auto &&element : registry) { + std::cout << "┌ " << element["file_path"] << '\n'; + std::cout << "└┬ Config name: " << element["config_record_name"] << '\n'; + std::cout << " ├ Parent config: " << element["parent_config"] << '\n'; + std::cout << " ├ Root access needed: " << element["root_access_needed"] << '\n'; + std::cout << " â”” Is required: " << element["is_required"] << '\n'; + } + } +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +ConfigSetPrinter::write_output(YAML::Node const &result) +{ + // we match the legacy format, the only one supported for now. + static const std::unordered_map Update_Type_To_String_Message = { + {"0", "Set {}"}, // UNDEFINED + {"1", "Set {}, please wait 10 seconds for traffic server to sync configuration, restart is not required"}, // DYNAMIC + {"2", "Set {}, restart required"}, // RESTART_TS + {"3", "Set {}, restart required"} // RESTART TM, we take care of this in case we get it from TS. + }; + std::string text; + try { + auto const &response = result.as(); + for (auto &&updatedRec : response.data) { + if (auto search = Update_Type_To_String_Message.find(updatedRec.updateType); + search != std::end(Update_Type_To_String_Message)) { + std::cout << ts::bwprint(text, search->second, updatedRec.recName) << '\n'; + } else { + std::cout << "Oops we don't know how to handle the update status for '" << updatedRec.recName << "' [" + << updatedRec.updateType << "]\n"; + } + } + } catch (std::exception const &ex) { + std::cout << ts::bwprint(text, "Unexpected error found {}", ex.what()); + } +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +RecordDescribePrinter::write_output(YAML::Node const &result) +{ + auto const &response = result.as(); + if (is_legacy_format()) { + write_output_legacy(response); + } else { + write_output_pretty(response); + } +} + +void +RecordDescribePrinter::write_output_legacy(shared::rpc::RecordLookUpResponse const &response) +{ + std::string text; + for (auto &&recordInfo : response.recordList) { + if (!recordInfo.registered) { + std::cout << recordInfo.name + << ": Unrecognized configuration value. Record is a configuration name/value but is not registered\n"; + continue; + } + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Name", recordInfo.name); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Current Value ", recordInfo.currentValue); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Default Value ", recordInfo.defaultValue); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Record Type ", rec_labelof(recordInfo.rclass)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Data Type ", recordInfo.dataType); + + std::visit(ts::meta::overloaded{ + [&](shared::rpc::RecordLookUpResponse::RecordParamInfo::ConfigMeta const &meta) { + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Access Control ", rec_accessof(meta.accessType)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Update Type ", rec_updateof(meta.updateType)); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Update Status ", meta.updateStatus); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Source ", rec_sourceof(meta.source)); + + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Syntax Check ", meta.checkExpr); + }, + [&](shared::rpc::RecordLookUpResponse::RecordParamInfo::StatMeta const &meta) { + // This may not be what we want, as for a metric we may not need to print all the same info. In that case + // just create a new printer for this. + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Persist Type ", meta.persistType); + }, + }, + recordInfo.meta); + + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Overridable", (recordInfo.overridable ? "yes" : "no")); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Version ", recordInfo.version); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Order ", recordInfo.order); + std::cout << ts::bwprint(text, "{:16s}: {}\n", "Raw Stat Block ", recordInfo.rsb); + } + + // also print errors. + print_record_error_list(response.errorList); +} + +void +RecordDescribePrinter::write_output_pretty(shared::rpc::RecordLookUpResponse const &response) +{ + // we default for legacy. + write_output_legacy(response); +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +GetHostStatusPrinter::write_output(YAML::Node const &result) +{ + auto response = result.as(); + for (auto &&recordInfo : response.recordList) { + std::cout << recordInfo.name << " " << recordInfo.currentValue << '\n'; + } + for (auto &&e : response.errorList) { + std::cout << "Failed to fetch " << e.recordName << '\n'; + } +} + +//------------------------------------------------------------------------------------------------------------------------------------ +void +SetHostStatusPrinter::write_output(YAML::Node const &result) +{ + // do nothing. +} +//------------------------------------------------------------------------------------------------------------------------------------ + +void +CacheDiskStoragePrinter::write_output(YAML::Node const &result) +{ + // do nothing. + if (!is_legacy_format()) { + write_output_pretty(result); + } +} +void +CacheDiskStoragePrinter::write_output_pretty(YAML::Node const &result) +{ + auto my_print = [](auto const &disk) { + std::cout << "Device: " << disk.path << '\n'; + std::cout << "Status: " << disk.status << '\n'; + std::cout << "Error Count: " << disk.errorCount << '\n'; + }; + + auto const &resp = result.as(); + auto iter = std::begin(resp.data); + my_print(*iter); + ++iter; + for (; iter != std::end(resp.data); ++iter) { + std::cout << "---\n"; + my_print(*iter); + } +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +CacheDiskStorageOfflinePrinter::write_output(YAML::Node const &result) +{ + if (!is_legacy_format()) { + write_output_pretty(result); + } +} +void +CacheDiskStorageOfflinePrinter::write_output_pretty(YAML::Node const &result) +{ + for (auto &&item : result) { + if (auto n = item["has_online_storage_left"]) { + bool any_left = n.as(); + if (!any_left) { + std::cout << "No more online storage left" << helper::try_extract(n, "path") << '\n'; + } + } + } +} +//------------------------------------------------------------------------------------------------------------------------------------ +void +RPCAPIPrinter::write_output(YAML::Node const &result) +{ + if (auto methods = result["methods"]) { + std::cout << "Methods:\n"; + for (auto &&m : methods) { + std::cout << "- " << m.as() << '\n'; + } + } + if (auto notifications = result["notifications"]) { + std::cout << "Notifications:\n"; + for (auto &&m : notifications) { + std::cout << "- " << m.as() << '\n'; + } + } +} +//------------------------------------------------------------------------------------------------------------------------------------ +//--------------------------------------------------------------------------------------------------------------------------------- diff --git a/src/traffic_ctl_jsonrpc/CtrlPrinters.h b/src/traffic_ctl_jsonrpc/CtrlPrinters.h new file mode 100644 index 00000000000..5b7d6297d95 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/CtrlPrinters.h @@ -0,0 +1,257 @@ +/** @file + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ +#pragma once + +#include +#include + +#include "shared/rpc/RPCRequests.h" + +//------------------------------------------------------------------------------------------------------------------------------------ +/// +/// Base class that implements the basic output format. +/// +/// Every command will print out specific details depending on the nature of the message. This base class models the basic API. +/// @c _format member should be set when the object is created, this member should be used to decide the way we want to generate +/// the output and possibly(TODO) where we want it(stdout, stderr, etc.). If no output is needed GenericPrinter can be used which is +/// muted. +class BasePrinter +{ +public: + /// This enum maps the --format flag coming from traffic_ctl. (also --records is included here, see comments down below.) + struct Options { + enum class Format { + LEGACY = 0, // Legacy format, mimics the old traffic_ctl output + PRETTY, // Enhanced printing messages. (in case you would like to generate them) + JSON, // Json formatting + RECORDS, // only valid for configs, but it's handy to have it here. + DATA_REQ, // Print json request + default format + DATA_RESP, // Print json response + default format + DATA_ALL // Print json request and response + default format + }; + Options() = default; + Options(Format fmt) : _format(fmt) {} + Format _format{Format::LEGACY}; //!< selected(passed) format. + }; + + /// Printer constructor. Needs the format as it will be used by derived classes. + BasePrinter(Options opts) : _printOpt(opts) {} + + BasePrinter() = default; + virtual ~BasePrinter() = default; + + /// + /// Function that will generate the expected output based on the response result. + /// + /// If the response contains any high level error, it will be print and the the specific derived class @c write_output() will not + /// be called. + /// @param response the server response. + /// + void write_output(shared::rpc::JSONRPCResponse const &response); + + /// + /// Write output based on the response values. + /// + /// Implement this one so you deal with the expected output, @c _format will be already set to the right + /// selected type so you can decide what to print. + /// + /// @param result jsonrpc result structure. No format specified by us, it's the one specified by the actual jsonrpc + /// response. + /// + virtual void write_output(YAML::Node const &result) = 0; + + virtual void write_output(std::string_view output) const; + virtual void write_debug(std::string_view output) const; + + /// Format getters. + Options::Format get_format() const; + bool print_req_msg() const; + bool print_resp_msg() const; + bool is_json_format() const; + bool is_legacy_format() const; + bool is_records_format() const; + bool is_pretty_format() const; + +protected: + void write_output_json(YAML::Node const &node) const; + Options _printOpt; +}; + +inline BasePrinter::Options::Format +BasePrinter::get_format() const +{ + return _printOpt._format; +} + +inline bool +BasePrinter::print_req_msg() const +{ + return get_format() == Options::Format::DATA_ALL || get_format() == Options::Format::DATA_REQ; +} + +inline bool +BasePrinter::print_resp_msg() const +{ + return get_format() == Options::Format::DATA_ALL || get_format() == Options::Format::DATA_RESP; +} + +inline bool +BasePrinter::is_json_format() const +{ + return get_format() == Options::Format::JSON; +} + +inline bool +BasePrinter::is_legacy_format() const +{ + return get_format() == Options::Format::LEGACY; +} +inline bool +BasePrinter::is_records_format() const +{ + return get_format() == Options::Format::RECORDS; +} +inline bool +BasePrinter::is_pretty_format() const +{ + return get_format() == Options::Format::PRETTY; +} +//------------------------------------------------------------------------------------------------------------------------------------ +class GenericPrinter : public BasePrinter +{ + void + write_output(YAML::Node const &result) override + { + /* muted */ + } + +public: + GenericPrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class RecordPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + void write_output_legacy(shared::rpc::RecordLookUpResponse const &result); + void write_output_pretty(shared::rpc::RecordLookUpResponse const &result); + +public: + RecordPrinter(Options opt) : BasePrinter(opt) { _printAsRecords = is_records_format(); } + +protected: + bool _printAsRecords{false}; +}; + +class MetricRecordPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + +public: + MetricRecordPrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class DiffConfigPrinter : public RecordPrinter +{ + void write_output(YAML::Node const &result) override; + void write_output_pretty(YAML::Node const &result); + +public: + DiffConfigPrinter(BasePrinter::Options opt) : RecordPrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class ConfigReloadPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + void write_output_pretty(YAML::Node const &result); + +public: + ConfigReloadPrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class ConfigShowFileRegistryPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + void write_output_pretty(YAML::Node const &result); + +public: + using BasePrinter::BasePrinter; +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class ConfigSetPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + +public: + using BasePrinter::BasePrinter; +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class RecordDescribePrinter : public BasePrinter +{ + void write_output_legacy(shared::rpc::RecordLookUpResponse const &result); + void write_output_pretty(shared::rpc::RecordLookUpResponse const &result); + void write_output(YAML::Node const &result) override; + +public: + RecordDescribePrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class GetHostStatusPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + +public: + GetHostStatusPrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class SetHostStatusPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + +public: + SetHostStatusPrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class CacheDiskStoragePrinter : public BasePrinter +{ + void write_output_pretty(YAML::Node const &result); + void write_output(YAML::Node const &result) override; + +public: + CacheDiskStoragePrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class CacheDiskStorageOfflinePrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + void write_output_pretty(YAML::Node const &result); + +public: + CacheDiskStorageOfflinePrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +class RPCAPIPrinter : public BasePrinter +{ + void write_output(YAML::Node const &result) override; + +public: + RPCAPIPrinter(BasePrinter::Options opt) : BasePrinter(opt) {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/traffic_ctl_jsonrpc/Makefile.inc b/src/traffic_ctl_jsonrpc/Makefile.inc new file mode 100644 index 00000000000..bb47533cca0 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/Makefile.inc @@ -0,0 +1,37 @@ +# +# Makefile.am for the Enterprise Management module. +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bin_PROGRAMS += traffic_ctl_jsonrpc/traffic_ctl + +traffic_ctl_jsonrpc_traffic_ctl_CPPFLAGS = \ + $(AM_CPPFLAGS) \ + $(TS_INCLUDES) \ + -I$(abs_top_srcdir)/mgmt2 \ + -I$(abs_top_srcdir)/include \ + @YAMLCPP_INCLUDES@ + +traffic_ctl_jsonrpc_traffic_ctl_SOURCES = \ + traffic_ctl_jsonrpc/traffic_ctl.cc \ + traffic_ctl_jsonrpc/CtrlPrinters.cc \ + traffic_ctl_jsonrpc/CtrlCommands.cc \ + shared/rpc/IPCSocketClient.cc + +traffic_ctl_jsonrpc_traffic_ctl_LDADD = \ + $(top_builddir)/src/tscore/libtscore.la \ + @HWLOC_LIBS@ @YAMLCPP_LIBS@ diff --git a/src/traffic_ctl_jsonrpc/PrintUtils.h b/src/traffic_ctl_jsonrpc/PrintUtils.h new file mode 100644 index 00000000000..9233b9baebb --- /dev/null +++ b/src/traffic_ctl_jsonrpc/PrintUtils.h @@ -0,0 +1,94 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +// Record access control, indexed by RecAccessT. +static const char * +rec_accessof(int rec_access) +{ + switch (rec_access) { + case 1: + return "no access"; + case 2: + return "read only"; + case 0: /* fallthrough */ + default: + return "default"; + } +} +static const char * +rec_updateof(int rec_updatetype) +{ + switch (rec_updatetype) { + case 1: + return "dynamic, no restart"; + case 2: + return "static, restart traffic_server"; + case 3: + return "static, restart traffic_manager"; + case 0: /* fallthrough */ + default: + return "none"; + } +} + +[[maybe_unused]] static const char * +rec_checkof(int rec_checktype) +{ + switch (rec_checktype) { + case 1: + return "string matching a regular expression"; + case 2: + return "integer with a specified range"; + case 3: + return "IP address"; + case 0: /* fallthrough */ + default: + return "none"; + } +} +static const char * +rec_labelof(int rec_class) +{ + switch (rec_class) { + case 1: + return "CONFIG"; + case 16: + return "LOCAL"; + default: + return "unknown"; + } +} +static const char * +rec_sourceof(int rec_source) +{ + switch (rec_source) { + case 1: + return "built in default"; + case 3: + return "administratively set"; + case 2: + return "plugin default"; + case 4: + return "environment"; + default: + return "unknown"; + } +} \ No newline at end of file diff --git a/src/traffic_ctl_jsonrpc/jsonrpc/CtrlRPCRequests.h b/src/traffic_ctl_jsonrpc/jsonrpc/CtrlRPCRequests.h new file mode 100644 index 00000000000..c3d9740e247 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/jsonrpc/CtrlRPCRequests.h @@ -0,0 +1,253 @@ +/** + @section license License + + Internal traffic_ctl request/responses definitions. + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +// We base on the common client types. +#include "shared/rpc/RPCRequests.h" + +/// This file defines all the traffic_ctl API client request and responses objects needed to model the jsonrpc messages used in the +/// TS JSONRPC Node API. + +/// +/// @brief Models the record request message to fetch all records by type. +/// +struct GetAllRecordsRequest : shared::rpc::RecordLookupRequest { + using super = shared::rpc::RecordLookupRequest; + GetAllRecordsRequest(bool const configs) : super() + { + super::emplace_rec(".*", shared::rpc::REGEX, (configs ? shared::rpc::CONFIG_REC_TYPES : shared::rpc::METRIC_REC_TYPES)); + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +/// +/// @brief Models the config reload request. No params are needed. +/// +struct ConfigReloadRequest : shared::rpc::ClientRequest { + std::string + get_method() const + { + return "admin_config_reload"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +/// +/// @brief To fetch config file registry from the RPC node. +/// +struct ConfigShowFileRegistryRequest : shared::rpc::ClientRequest { + std::string + get_method() const + { + return "filemanager.get_files_registry"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +/// +/// @brief Models the clear 'all' metrics request. +/// +struct ClearAllMetricRequest : shared::rpc::ClientRequest { + std::string + get_method() const + { + return "admin_clear_all_metrics_records"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +/// +/// @brief Models the clear metrics request. +/// +struct ClearMetricRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + struct Params { + std::vector names; //!< client expects a list of record names. + }; + ClearMetricRequest(Params p) { super::params = p; } + std::string + get_method() const + { + return "admin_clear_metrics_records"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct ConfigSetRecordRequest : shared::rpc::ClientRequest { + struct Params { + std::string recName; + std::string recValue; + }; + using super = shared::rpc::ClientRequest; + ConfigSetRecordRequest(Params d) { super::params.push_back(d); } + std::string + get_method() const + { + return "admin_config_set_records"; + } +}; +struct ConfigSetRecordResponse { + struct UpdatedRec { + std::string recName; + std::string updateType; + }; + std::vector data; +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct HostSetStatusRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + struct Params { + enum class Op : short { + UP = 1, + DOWN, + }; + Op op; + std::vector hosts; + std::string reason; + std::string time{"0"}; + }; + + HostSetStatusRequest(Params p) { super::params = p; } + std::string + get_method() const + { + return "admin_host_set_status"; + } +}; + +struct HostGetStatusRequest : shared::rpc::RecordLookupRequest { + static constexpr auto STATUS_PREFIX = "proxy.process.host_status"; + using super = shared::rpc::RecordLookupRequest; + HostGetStatusRequest() : super() {} +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct BasicPluginMessageRequest : shared::rpc::ClientRequest { + using super = BasicPluginMessageRequest; + struct Params { + std::string tag; + std::string str; + }; + BasicPluginMessageRequest(Params p) { super::params = p; } + std::string + get_method() const + { + return "admin_plugin_send_basic_msg"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct ServerStartDrainRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + struct Params { + bool waitForNewConnections{false}; + }; + ServerStartDrainRequest(Params p) + { + super::method = "admin_server_start_drain"; + super::params = p; + } + std::string + get_method() const + { + return "admin_server_start_drain"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct ServerStopDrainRequest : shared::rpc::ClientRequest { + using super = ServerStopDrainRequest; + std::string + get_method() const + { + return "admin_server_start_drain"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct SetStorageDeviceOfflineRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + struct Params { + std::vector names; + }; + SetStorageDeviceOfflineRequest(Params p) { super::params = p; } + std::string + get_method() const + { + return "admin_storage_set_device_offline"; + } +}; + +//------------------------------------------------------------------------------------------------------------------------------------ +struct GetStorageDeviceStatusRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + struct Params { + std::vector names; + }; + GetStorageDeviceStatusRequest(Params p) { super::params = p; } + std::string + get_method() const + { + return "admin_storage_get_device_status"; + } +}; + +struct DeviceStatusInfoResponse { + struct CacheDisk { + CacheDisk(std::string p, std::string s, int e) : path(std::move(p)), status(std::move(s)), errorCount(e) {} + std::string path; + std::string status; + int errorCount; + }; + std::vector data; +}; +//------------------------------------------------------------------------------------------------------------------------------------ +struct ShowRegisterHandlersRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + std::string + get_method() const + { + return "show_registered_handlers"; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +// We expect the method to be passed, this request is used to create dynamic requests by using (traffic_ctl rpc invoke "func_name") +struct CustomizableRequest : shared::rpc::ClientRequest { + using super = shared::rpc::ClientRequest; + CustomizableRequest(std::string const &methodName) { super::method = methodName; } + std::string + get_method() const + { + return super::method; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +/// +/// @brief Config status request mapping class. +/// +/// There is no interaction between the traffic_ctl and this class so all the variables are defined in this +/// class. +/// +struct ConfigStatusRequest : shared::rpc::RecordLookupRequest { + using super = shared::rpc::RecordLookupRequest; + ConfigStatusRequest() : super() + { + static const std::array statusFieldsNames = { + "proxy.process.version.server.long", "proxy.node.restarts.proxy.start_time", + "proxy.node.config.reconfigure_time", "proxy.node.config.reconfigure_required", + "proxy.node.config.restart_required.proxy", "proxy.node.config.restart_required.manager"}; + for (auto &&recordName : statusFieldsNames) { + super::emplace_rec(recordName, shared::rpc::NOT_REGEX, shared::rpc::METRIC_REC_TYPES); + } + } +}; diff --git a/src/traffic_ctl_jsonrpc/jsonrpc/ctrl_yaml_codecs.h b/src/traffic_ctl_jsonrpc/jsonrpc/ctrl_yaml_codecs.h new file mode 100644 index 00000000000..41f81874d27 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/jsonrpc/ctrl_yaml_codecs.h @@ -0,0 +1,161 @@ +/** + @section license License + + traffic_ctl yaml codecs. + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +#pragma once + +// base yaml codecs. +#include "shared/rpc/yaml_codecs.h" + +#include "CtrlRPCRequests.h" + +// traffic_ctl jsonrpc request/response YAML codec implementation. + +namespace YAML +{ +template <> struct convert { + static Node + encode(ConfigSetRecordRequest::Params const ¶ms) + { + Node node; + node["record_name"] = params.recName; + node["record_value"] = params.recValue; + + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(HostSetStatusRequest::Params::Op const &op) + { + using Op = HostSetStatusRequest::Params::Op; + switch (op) { + case Op::UP: + return Node("up"); + case Op::DOWN: + return Node("down"); + } + return Node("unknown"); + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(HostSetStatusRequest::Params const ¶ms) + { + Node node; + node["operation"] = params.op; + node["host"] = params.hosts; // list + node["reason"] = params.reason; + node["time"] = params.time; + + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(ClearMetricRequest::Params const ¶ms) + { + Node node; + for (auto name : params.names) { + Node n; + n["record_name"] = name; + node.push_back(n); + } + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(BasicPluginMessageRequest::Params const ¶ms) + { + Node node; + node["tag"] = params.tag; + node["data"] = params.str; + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(ServerStartDrainRequest::Params const ¶ms) + { + Node node; + node["no_new_connections"] = params.waitForNewConnections; + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(SetStorageDeviceOfflineRequest::Params const ¶ms) + { + Node node; + for (auto &&path : params.names) { + node.push_back(path); + } + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static Node + encode(GetStorageDeviceStatusRequest::Params const ¶ms) + { + Node node; + for (auto &&path : params.names) { + node.push_back(path); + } + return node; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, DeviceStatusInfoResponse &info) + { + for (auto &&item : node) { + Node disk = item["cachedisk"]; + info.data.emplace_back(helper::try_extract(disk, "path"), // path + helper::try_extract(disk, "status"), // status + helper::try_extract(disk, "error_count") // err count + + ); + } + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +template <> struct convert { + static bool + decode(Node const &node, ConfigSetRecordResponse &info) + { + for (auto &&item : node) { + info.data.push_back( + {helper::try_extract(item, "record_name"), helper::try_extract(item, "update_type")}); + } + return true; + } +}; +//------------------------------------------------------------------------------------------------------------------------------------ +} // namespace YAML \ No newline at end of file diff --git a/src/traffic_ctl_jsonrpc/traffic_ctl.cc b/src/traffic_ctl_jsonrpc/traffic_ctl.cc new file mode 100644 index 00000000000..7d7bd4b3bc1 --- /dev/null +++ b/src/traffic_ctl_jsonrpc/traffic_ctl.cc @@ -0,0 +1,192 @@ +/** @file + + traffic_ctl + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +#include + +#include "tscore/I_Layout.h" +#include "tscore/runroot.h" +#include "tscore/ArgParser.h" + +#include "CtrlCommands.h" + +constexpr int CTRL_EX_OK = 0; +constexpr int CTRL_EX_ERROR = 2; +constexpr int CTRL_EX_UNIMPLEMENTED = 3; + +int status_code{CTRL_EX_OK}; + +int +main(int argc, const char **argv) +{ + ts::ArgParser parser; + + std::shared_ptr command; + + auto CtrlUnimplementedCommand = [](std::string_view cmd) { + std::cout << "Command " << cmd << " unimplemented.\n"; + status_code = CTRL_EX_UNIMPLEMENTED; + }; + + parser.add_description("Apache Traffic Server RPC CLI"); + parser.add_global_usage("traffic_ctl [OPTIONS] CMD [ARGS ...]"); + parser.require_commands(); + + parser.add_option("--debug", "", "Enable debugging output - unimplemented") + .add_option("--version", "-V", "Print version string") + .add_option("--help", "-h", "Print usage information") + .add_option("--run-root", "", "using TS_RUNROOT as sandbox", "TS_RUNROOT", 1) + .add_option("--format", "-f", "Use a specific output format {legacy|pretty|json|data:{req|resp|all}}", "", 1, "legacy", + "format"); + + auto &config_command = parser.add_command("config", "Manipulate configuration records").require_commands(); + auto &metric_command = parser.add_command("metric", "Manipulate performance metrics").require_commands(); + auto &server_command = parser.add_command("server", "Stop, restart and examine the server").require_commands(); + auto &storage_command = parser.add_command("storage", "Manipulate cache storage").require_commands(); + auto &plugin_command = parser.add_command("plugin", "Interact with plugins").require_commands(); + auto &host_command = parser.add_command("host", "Interact with host status").require_commands(); + auto &direct_rpc_command = parser.add_command("rpc", "Interact with the rpc api").require_commands(); + + // config commands + config_command.add_command("defaults", "Show default information configuration values", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config defaults [OPTIONS]") + .add_option("--records", "", "Emit output in records.config format"); + config_command + .add_command("describe", "Show detailed information about configuration values", "", MORE_THAN_ONE_ARG_N, + [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config describe RECORD [RECORD ...]"); + config_command.add_command("diff", "Show non-default configuration values", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config diff [OPTIONS]") + .add_option("--records", "", "Emit output in records.config format"); + config_command.add_command("get", "Get one or more configuration values", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config get [OPTIONS] RECORD [RECORD ...]") + .add_option("--records", "", "Emit output in records.config format"); + config_command + .add_command("match", "Get configuration matching a regular expression", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config match [OPTIONS] REGEX [REGEX ...]") + .add_option("--records", "", "Emit output in records.config format"); + config_command.add_command("reload", "Request a configuration reload", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config reload"); + config_command.add_command("status", "Check the configuration status", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config status"); + config_command.add_command("set", "Set a configuration value", "", 2, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config set RECORD VALUE"); + + config_command.add_command("registry", "Show configuration file registry", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl config registry"); + // host commands + host_command.add_command("status", "Get one or more host statuses", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl host status HOST [HOST ...]"); + host_command.add_command("down", "Set down one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl host down HOST [OPTIONS]") + .add_option("--time", "-I", "number of seconds that a host is marked down", "", 1, "0") + .add_option("--reason", "", "reason for marking the host down, one of 'manual|active|local", "", 1, "manual"); + host_command.add_command("up", "Set up one or more host(s)", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl host up METRIC value") + .add_option("--reason", "", "reason for marking the host up, one of 'manual|active|local", "", 1, "manual"); + + // metric commands + metric_command.add_command("get", "Get one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl metric get METRIC [METRIC ...]"); + metric_command.add_command("clear", "Clear all metric values", [&]() { command->execute(); }); + metric_command.add_command("describe", "Show detailed information about one or more metric values", "", MORE_THAN_ONE_ARG_N, + [&]() { command->execute(); }); // not implemented + metric_command.add_command("match", "Get metrics matching a regular expression", "", MORE_THAN_ZERO_ARG_N, + [&]() { command->execute(); }); + metric_command.add_command("monitor", "Display the value of a metric over time", "", MORE_THAN_ZERO_ARG_N, + [&]() { CtrlUnimplementedCommand("monitor"); }); // not implemented + metric_command.add_command("zero", "Clear one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }); + + // plugin command + plugin_command.add_command("msg", "Send message to plugins - a TAG and the message DATA", "", 2, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl plugin msg TAG DATA"); + + // server commands + server_command.add_command("backtrace", "Show a full stack trace of the traffic_server process", + [&]() { CtrlUnimplementedCommand("backtrace"); }); + server_command.add_command("restart", "Restart Traffic Server", [&]() { CtrlUnimplementedCommand("restart"); }) + .add_example_usage("traffic_ctl server restart [OPTIONS]") + .add_option("--drain", "", "Wait for client connections to drain before restarting"); + server_command.add_command("start", "Start the proxy", [&]() { CtrlUnimplementedCommand("start"); }) + .add_example_usage("traffic_ctl server start [OPTIONS]") + .add_option("--clear-cache", "", "Clear the disk cache on startup") + .add_option("--clear-hostdb", "", "Clear the DNS cache on startup"); + server_command.add_command("status", "Show the proxy status", [&]() { CtrlUnimplementedCommand("status"); }) + .add_example_usage("traffic_ctl server status"); + server_command.add_command("stop", "Stop the proxy", [&]() { CtrlUnimplementedCommand("stop"); }) + .add_example_usage("traffic_ctl server stop [OPTIONS]") + .add_option("--drain", "", "Wait for client connections to drain before stopping"); + server_command.add_command("drain", "Drain the requests", [&]() { command->execute(); }) + .add_example_usage("traffic_ctl server drain [OPTIONS]") + .add_option("--no-new-connection", "-N", "Wait for new connections down to threshold before starting draining") + .add_option("--undo", "-U", "Recover server from the drain mode"); + + // storage commands + storage_command + .add_command("offline", "Take one or more storage volumes offline", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); }) + .add_example_usage("storage offline DEVICE [DEVICE ...]"); + storage_command.add_command("status", "Show the storage configuration", "", MORE_THAN_ONE_ARG_N, + [&]() { command->execute(); }); // not implemented + + // direct rpc commands, handy for debug and trouble shooting + direct_rpc_command + .add_command("file", "Send direct JSONRPC request to the server from a passed file(s)", "", MORE_THAN_ONE_ARG_N, + [&]() { command->execute(); }) + .add_example_usage("traffic_ctl rpc file request.yaml"); + direct_rpc_command.add_command("get-api", "Request full API from server", "", 0, [&]() { command->execute(); }) + .add_example_usage("traffic_ctl rpc get-api"); + direct_rpc_command + .add_command("input", "Read from standard input. Ctrl-D to send the request", "", 0, [&]() { command->execute(); }) + .add_option("--raw", "-r", + "No json/yaml parse validation will take place, the raw content will be directly send to the server.", "", 0, "", + "raw") + .add_example_usage("traffic_ctl rpc input "); + + try { + auto args = parser.parse(argv); + argparser_runroot_handler(args.get("run-root").value(), argv[0]); + Layout::create(); + + if (args.get("config")) { + command = std::make_shared(args); + } else if (args.get("metric")) { + command = std::make_shared(args); + } else if (args.get("server")) { + command = std::make_shared(args); + } else if (args.get("storage")) { + command = std::make_shared(args); + } else if (args.get("plugin")) { + command = std::make_shared(args); + } else if (args.get("host")) { + command = std::make_shared(args); + } else if (args.get("rpc")) { + command = std::make_shared(args); + } + // Execute + args.invoke(); + } catch (std::exception const &ex) { + status_code = CTRL_EX_ERROR; + std::cerr << "Error found.\n" << ex.what() << '\n'; + } + + return status_code; +} diff --git a/src/traffic_server/HostStatus.cc b/src/traffic_server/HostStatus.cc index 0449aeb0a67..cb273137013 100644 --- a/src/traffic_server/HostStatus.cc +++ b/src/traffic_server/HostStatus.cc @@ -23,6 +23,11 @@ #include "HostStatus.h" #include "ProcessManager.h" +#include "tscore/BufferWriter.h" +#include "rpc/jsonrpc/JsonRPC.h" + +ts::Rv server_set_status(std::string_view const &id, YAML::Node const ¶ms); + inline void getStatName(std::string &stat_name, const std::string_view name) { @@ -207,6 +212,8 @@ HostStatus::HostStatus() ink_rwlock_init(&host_status_rwlock); pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_UP, &mgmt_host_status_up_callback); pmgmt->registerMgmtCallback(MGMT_EVENT_HOST_STATUS_DOWN, &mgmt_host_status_down_callback); + + rpc::add_method_handler("admin_host_set_status", &server_set_status, &rpc::core_ats_rpc_service_provider_handle); } HostStatus::~HostStatus() @@ -449,3 +456,91 @@ HostStatus::getHostStat(std::string &stat_name, char *buf, unsigned int buf_len) { return RecGetRecordString(stat_name.c_str(), buf, buf_len, true); } + +namespace +{ +struct HostCmdInfo { + TSHostStatus type{TSHostStatus::TS_HOST_STATUS_INIT}; + unsigned int reasonType{0}; + std::vector hosts; + int time{0}; +}; + +} // namespace + +namespace YAML +{ +template <> struct convert { + static bool + decode(const Node &node, HostCmdInfo &rhs) + { + if (auto n = node["operation"]) { + auto const &str = n.as(); + if (str == "up") { + rhs.type = TSHostStatus::TS_HOST_STATUS_UP; + } else if (str == "down") { + rhs.type = TSHostStatus::TS_HOST_STATUS_DOWN; + } else { + // unknown. + return false; + } + } else { + return false; + } + + if (auto n = node["host"]; n.IsSequence() && n.size()) { + for (auto &&it : n) { + rhs.hosts.push_back(it.as()); + } + } else { + return false; + } + + if (auto n = node["reason"]) { + auto reasonStr = n.as(); + rhs.reasonType = Reason::getReason(reasonStr.c_str()); + } // manual by default. + + if (auto n = node["time"]) { + rhs.time = std::stoi(n.as()); + if (rhs.time < 0) { + return false; + } + } else { + return false; + } + + return true; + } +}; +} // namespace YAML + +ts::Rv +server_set_status(std::string_view const &id, YAML::Node const ¶ms) +{ + namespace err = rpc::handlers::errors; + ts::Rv resp; + try { + if (!params.IsNull()) { + auto cmdInfo = params.as(); + + for (auto const &name : cmdInfo.hosts) { + HostStatus &hs = HostStatus::instance(); + std::string statName = stat_prefix + name; + char buf[1024] = {0}; + if (hs.getHostStat(statName, buf, 1024) == REC_ERR_FAIL) { + hs.createHostStat(name.c_str()); + } + Debug("host_statuses", "marking server %s : %s", name.c_str(), + (cmdInfo.type == TSHostStatus::TS_HOST_STATUS_UP ? "up" : "down")); + hs.setHostStatus(name.c_str(), cmdInfo.type, cmdInfo.time, cmdInfo.reasonType); + } + } else { + resp.errata().push(err::make_errata(err::Codes::SERVER, "Invalid input parameters, null")); + } + } catch (std::exception const &ex) { + Debug("host_statuses", "Got an error HostCmdInfo decoding: %s", ex.what()); + resp.errata().push(err::make_errata(err::Codes::SERVER, "Error found during host status set: {}", ex.what())); + } + return resp; +} diff --git a/src/traffic_server/Makefile.inc b/src/traffic_server/Makefile.inc index 1472ede40f0..f544dd7d0ef 100644 --- a/src/traffic_server/Makefile.inc +++ b/src/traffic_server/Makefile.inc @@ -32,11 +32,13 @@ traffic_server_traffic_server_CPPFLAGS = \ -I$(abs_top_srcdir)/proxy/http/remap \ -I$(abs_top_srcdir)/proxy/hdrs \ -I$(abs_top_srcdir)/proxy/shared \ + -I$(abs_top_srcdir)/mgmt2 \ -I$(abs_top_srcdir)/mgmt \ -I$(abs_top_srcdir)/mgmt/utils \ $(TS_INCLUDES) \ @OPENSSL_INCLUDES@ \ - @BORINGOCSP_INCLUDES@ + @BORINGOCSP_INCLUDES@ \ + @YAMLCPP_INCLUDES@ traffic_server_traffic_server_LDFLAGS = \ $(AM_LDFLAGS) \ @@ -53,6 +55,8 @@ traffic_server_traffic_server_SOURCES = \ traffic_server/InkAPI.cc \ traffic_server/InkIOCoreAPI.cc \ traffic_server/SocksProxy.cc \ + traffic_server/RpcAdminPubHandlers.cc \ + traffic_server/RpcAdminPubHandlers.h \ shared/overridable_txn_vars.cc \ traffic_server/traffic_server.cc @@ -82,6 +86,10 @@ traffic_server_traffic_server_LDADD = \ $(top_builddir)/iocore/net/libinknet.a \ $(top_builddir)/lib/records/librecords_p.a \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ + $(top_builddir)/mgmt2/rpc/libjsonrpc_server.la \ + $(top_builddir)/mgmt2/rpc/libjsonrpc_protocol.la \ + $(top_builddir)/mgmt2/config/libconfigmanager.la \ + $(top_builddir)/mgmt2/rpc/librpcpublichandlers.la \ @HWLOC_LIBS@ \ @LIBPCRE@ \ @LIBRESOLV@ \ diff --git a/src/traffic_server/RpcAdminPubHandlers.cc b/src/traffic_server/RpcAdminPubHandlers.cc new file mode 100644 index 00000000000..32422ffed36 --- /dev/null +++ b/src/traffic_server/RpcAdminPubHandlers.cc @@ -0,0 +1,63 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#include "rpc/jsonrpc/JsonRPC.h" + +// Admin API Implementation headers. +#include "rpc/handlers/config/Configuration.h" +#include "rpc/handlers/records/Records.h" +#include "rpc/handlers/storage/Storage.h" +#include "rpc/handlers/server/Server.h" +#include "rpc/handlers/plugins/Plugins.h" + +#include "RpcAdminPubHandlers.h" +namespace rpc::admin +{ +void +register_admin_jsonrpc_handlers() +{ + // Config + using namespace rpc::handlers::config; + rpc::add_method_handler("admin_config_set_records", &set_config_records, &core_ats_rpc_service_provider_handle); + rpc::add_method_handler("admin_config_reload", &reload_config, &core_ats_rpc_service_provider_handle); + + // Records + using namespace rpc::handlers::records; + rpc::add_method_handler("admin_lookup_records", &lookup_records, &core_ats_rpc_service_provider_handle); + rpc::add_method_handler("admin_clear_all_metrics_records", &clear_all_metrics_records, &core_ats_rpc_service_provider_handle); + rpc::add_method_handler("admin_clear_metrics_records", &clear_metrics_records, &core_ats_rpc_service_provider_handle); + + // plugin + using namespace rpc::handlers::plugins; + rpc::add_method_handler("admin_plugin_send_basic_msg", &plugin_send_basic_msg, &core_ats_rpc_service_provider_handle); + + // server + using namespace rpc::handlers::server; + rpc::add_method_handler("admin_server_start_drain", &server_start_drain, &core_ats_rpc_service_provider_handle); + rpc::add_method_handler("admin_server_stop_drain", &server_stop_drain, &core_ats_rpc_service_provider_handle); + rpc::add_notification_handler("admin_server_shutdown", &server_shutdown, &core_ats_rpc_service_provider_handle); + rpc::add_notification_handler("admin_server_restart", &server_shutdown, &core_ats_rpc_service_provider_handle); + + // storage + using namespace rpc::handlers::storage; + rpc::add_method_handler("admin_storage_set_device_offline", &set_storage_offline, &core_ats_rpc_service_provider_handle); + rpc::add_method_handler("admin_storage_get_device_status", &get_storage_status, &core_ats_rpc_service_provider_handle); +} +} // namespace rpc::admin \ No newline at end of file diff --git a/src/traffic_server/RpcAdminPubHandlers.h b/src/traffic_server/RpcAdminPubHandlers.h new file mode 100644 index 00000000000..ff29abcdf57 --- /dev/null +++ b/src/traffic_server/RpcAdminPubHandlers.h @@ -0,0 +1,27 @@ +/** + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +#pragma once + +namespace rpc::admin +{ +/// Initialize and register all public handler that will be exposed to the JSON RPC server. +void register_admin_jsonrpc_handlers(); +} // namespace rpc::admin \ No newline at end of file diff --git a/src/traffic_server/traffic_server.cc b/src/traffic_server/traffic_server.cc index c9520300f7d..ddd6fd85339 100644 --- a/src/traffic_server/traffic_server.cc +++ b/src/traffic_server/traffic_server.cc @@ -104,6 +104,15 @@ extern "C" int plock(int); #include "P_SSLSNI.h" #include "P_SSLClientUtils.h" +// Mgmt Admin public handlers +#include "RpcAdminPubHandlers.h" + +// Json Rpc stuffs +#include "rpc/jsonrpc/JsonRPCManager.h" +#include "rpc/server/RPCServer.h" + +#include "config/FileManager.h" + #if TS_USE_QUIC == 1 #include "Http3.h" #include "Http3Config.h" @@ -248,6 +257,12 @@ struct AutoStopCont : public Continuation { } pmgmt->stop(); + + // if the jsonrpc feature was disabled, the object will not be created. + if (jsonrpcServer != nullptr) { + jsonrpcServer->stop_thread(); + } + TSSystemState::shut_down_event_system(); delete this; return EVENT_CONT; @@ -389,7 +404,15 @@ class TrackerContinuation : public Continuation class DiagsLogContinuation : public Continuation { public: - DiagsLogContinuation() : Continuation(new_ProxyMutex()) { SET_HANDLER(&DiagsLogContinuation::periodic); } + DiagsLogContinuation() : Continuation(new_ProxyMutex()) + { + SET_HANDLER(&DiagsLogContinuation::periodic); + + char *configured_traffic_out_name(REC_ConfigReadString("proxy.config.output.logfile")); + traffic_out_name = std::string(configured_traffic_out_name); + ats_free(configured_traffic_out_name); + } + int periodic(int /* event ATS_UNUSED */, Event * /* e ATS_UNUSED */) { @@ -402,16 +425,33 @@ class DiagsLogContinuation : public Continuation // to send a notification from TS to TM, informing TM that outputlog has // been rolled. It is much easier sending a notification (in the form // of SIGUSR2) from TM -> TS. - int diags_log_roll_int = (int)REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_interval_sec"); - int diags_log_roll_size = (int)REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_size_mb"); - int diags_log_roll_enable = (int)REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_enabled"); - diags->config_roll_diagslog((RollingEnabledValues)diags_log_roll_enable, diags_log_roll_int, diags_log_roll_size); + int diags_log_roll_int = static_cast(REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_interval_sec")); + int diags_log_roll_size = static_cast(REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_size_mb")); + int diags_log_roll_enable = static_cast(REC_ConfigReadInteger("proxy.config.diags.logfile.rolling_enabled")); + diags->config_roll_diagslog(static_cast(diags_log_roll_enable), diags_log_roll_int, diags_log_roll_size); if (diags->should_roll_diagslog()) { Note("Rolled %s", diags_log_filename); } + + // If we are using the JSONRPC service, then there is no traffic_manager + // and we are responsible for rolling traffic.out. + if (jsonrpcServer != nullptr) { + int output_log_roll_int = static_cast(REC_ConfigReadInteger("proxy.config.output.logfile.rolling_interval_sec")); + int output_log_roll_size = static_cast(REC_ConfigReadInteger("proxy.config.output.logfile.rolling_size_mb")); + int output_log_roll_enable = static_cast(REC_ConfigReadInteger("proxy.config.output.logfile.rolling_enabled")); + diags->config_roll_outputlog(static_cast(output_log_roll_enable), output_log_roll_int, + output_log_roll_size); + + if (diags->should_roll_outputlog()) { + Note("Rolled %s", traffic_out_name.c_str()); + } + } return EVENT_CONT; } + +private: + std::string traffic_out_name; }; class MemoryLimit : public Continuation @@ -700,6 +740,45 @@ initialize_process_manager() RECP_NON_PERSISTENT); } +extern void initializeRegistry(); + +static void +initialize_file_manager() +{ + initializeRegistry(); +} + +std::tuple +initialize_jsonrpc_server() +{ + std::tuple ok{true, {}}; + auto filePath = RecConfigReadConfigPath("proxy.config.jsonrpc.filename", ts::filename::JSONRPC); + + auto serverConfig = rpc::config::RPCConfig{}; + serverConfig.load_from_file(filePath); + if (!serverConfig.is_enabled()) { + Debug("rpc.init", "JSONRPC Disabled"); + return ok; + } + + // create and start the server. + try { + jsonrpcServer = new rpc::RPCServer{serverConfig}; + jsonrpcServer->start_thread(TSThreadInit, TSThreadDestroy); + } catch (std::exception const &ex) { + // Only the constructor throws, so if we are here there should be no + // jsonrpcServer object. + ink_assert(jsonrpcServer == nullptr); + std::string msg; + return {false, ts::bwprint(msg, "Server failed: '{}'", ex.what())}; + } + // Register admin handlers. + rpc::admin::register_admin_jsonrpc_handlers(); + Debug("rpc.init", "JSONRPC. Public admin handlers registered."); + + return ok; +} + #define CMD_ERROR -2 // serious error, exit maintenance mode #define CMD_FAILED -1 // error, but recoverable #define CMD_OK 0 // ok, or minor (user) error @@ -1802,6 +1881,9 @@ main(int /* argc ATS_UNUSED */, const char **argv) // Local process manager initialize_process_manager(); + // Initialize file manager for TS. + initialize_file_manager(); + // Set the core limit for the process init_core_size(); init_system(); @@ -1819,6 +1901,8 @@ main(int /* argc ATS_UNUSED */, const char **argv) RecRegisterStatInt(RECT_NODE, "proxy.node.config.restart_required.proxy", 0, RECP_NON_PERSISTENT); RecRegisterStatInt(RECT_NODE, "proxy.node.config.restart_required.manager", 0, RECP_NON_PERSISTENT); RecRegisterStatInt(RECT_NODE, "proxy.node.config.draining", 0, RECP_NON_PERSISTENT); + RecRegisterStatInt(RECT_NODE, "proxy.node.proxy_running", 1, RECP_NON_PERSISTENT); + RecSetRecordInt("proxy.node.restarts.proxy.start_time", time(nullptr), REC_SOURCE_DEFAULT); } // init huge pages @@ -1914,6 +1998,11 @@ main(int /* argc ATS_UNUSED */, const char **argv) } #endif + // JSONRPC server and handlers + if (auto &&[ok, msg] = initialize_jsonrpc_server(); !ok) { + Warning("JSONRPC server could not be started.\n Why?: '%s' ... Continuing without it.", msg.c_str()); + } + // setup callback for tracking remap included files load_remap_file_cb = load_config_file_callback; @@ -2106,7 +2195,7 @@ main(int /* argc ATS_UNUSED */, const char **argv) quic_NetProcessor.start(-1, stacksize); #endif pmgmt->registerPluginCallbacks(global_config_cbs); - + FileManager::instance().registerConfigPluginCallbacks(global_config_cbs); cacheProcessor.afterInitCallbackSet(&CB_After_Cache_Init); cacheProcessor.start(); @@ -2303,12 +2392,15 @@ static void load_ssl_file_callback(const char *ssl_file) { pmgmt->signalConfigFileChild(ts::filename::SSL_MULTICERT, ssl_file); + FileManager::instance().configFileChild(ts::filename::SSL_MULTICERT, ssl_file); } void load_config_file_callback(const char *parent_file, const char *remap_file) { pmgmt->signalConfigFileChild(parent_file, remap_file); + // TODO: for now in both + FileManager::instance().configFileChild(parent_file, remap_file); } static void diff --git a/src/traffic_top/Makefile.inc b/src/traffic_top/Makefile.inc index 7d0a820fb95..0f54cd2d432 100644 --- a/src/traffic_top/Makefile.inc +++ b/src/traffic_top/Makefile.inc @@ -21,34 +21,30 @@ if BUILD_TRAFFIC_TOP bin_PROGRAMS += traffic_top/traffic_top traffic_top_traffic_top_CPPFLAGS = \ - $(AM_CPPFLAGS) \ - $(iocore_include_dirs) \ + $(AM_CPPFLAGS) \ -I$(abs_top_srcdir)/include \ - -I$(abs_top_srcdir)/lib \ - -I$(abs_top_srcdir)/mgmt \ - -I$(abs_top_srcdir)/mgmt/api/include \ $(TS_INCLUDES) \ @CURL_CFLAGS@ \ - @CURSES_CFLAGS@ + @CURSES_CFLAGS@ \ + @YAMLCPP_INCLUDES@ traffic_top_traffic_top_LDFLAGS = \ - $(AM_LDFLAGS) \ + $(AM_LDFLAGS) \ @CURSES_LDFLAGS@ \ - @OPENSSL_LDFLAGS@ + @OPENSSL_LDFLAGS@ \ + @YAMLCPP_LIBS@ + traffic_top_traffic_top_SOURCES = \ - traffic_top/traffic_top.cc + traffic_top/traffic_top.cc \ + shared/rpc/IPCSocketClient.cc traffic_top_traffic_top_LDADD = \ - $(top_builddir)/lib/records/librecords_p.a \ - $(top_builddir)/mgmt/libmgmt_p.la \ - $(top_builddir)/proxy/shared/libUglyLogStubs.a \ $(top_builddir)/iocore/eventsystem/libinkevent.a \ - $(top_builddir)/mgmt/api/libtsmgmt.la \ $(top_builddir)/src/tscore/libtscore.la \ $(top_builddir)/src/tscpp/util/libtscpputil.la \ @CURL_LIBS@ \ @CURSES_LIBS@ \ - @HWLOC_LIBS@ - + @HWLOC_LIBS@ \ + @YAMLCPP_LIBS@ endif diff --git a/src/traffic_top/stats.h b/src/traffic_top/stats.h index 7b508491ff5..e35b56829d2 100644 --- a/src/traffic_top/stats.h +++ b/src/traffic_top/stats.h @@ -20,6 +20,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +#pragma once + #if HAS_CURL #include #endif @@ -30,7 +32,10 @@ #include #include #include -#include "mgmtapi.h" + +#include "shared/rpc/RPCRequests.h" +#include "shared/rpc/RPCClient.h" +#include "shared/rpc/yaml_codecs.h" struct LookupItem { LookupItem(const char *s, const char *n, const int t) : pretty(s), name(n), numerator(""), denominator(""), type(t) {} @@ -57,6 +62,19 @@ const char separator[] = "\": \""; const char end[] = "\",\n"; }; // namespace constant +// Convenient definitions +namespace detail +{ +/// This is a convenience class to abstract the metric params. It makes it less verbose to add a metric info object inside the +/// record lookup object. +struct MetricParam : shared::rpc::RecordLookupRequest::Params { + MetricParam(std::string name) + : // not regex + shared::rpc::RecordLookupRequest::Params{std::move(name), shared::rpc::NOT_REGEX, shared::rpc::METRIC_REC_TYPES} + { + } +}; +} // namespace detail //---------------------------------------------------------------------------- class Stats { @@ -270,7 +288,6 @@ class Stats getStats() { if (_url == "") { - int64_t value = 0; if (_old_stats != nullptr) { delete _old_stats; _old_stats = nullptr; @@ -281,36 +298,22 @@ class Stats gettimeofday(&_time, nullptr); double now = _time.tv_sec + (double)_time.tv_usec / 1000000; + // We will lookup for all the metrics on one single request. + shared::rpc::RecordLookupRequest request; + for (map::const_iterator lookup_it = lookup_table.begin(); lookup_it != lookup_table.end(); ++lookup_it) { const LookupItem &item = lookup_it->second; if (item.type == 1 || item.type == 2 || item.type == 5 || item.type == 8) { - if (strcmp(item.pretty, "Version") == 0) { - // special case for Version information - TSString strValue = nullptr; - if (TSRecordGetString(item.name, &strValue) == TS_ERR_OKAY) { - string key = item.name; - (*_stats)[key] = strValue; - TSfree(strValue); - } else { - fprintf(stderr, "Error getting stat: %s when calling TSRecordGetString() failed: file \"%s\", line %d\n\n", item.name, - __FILE__, __LINE__); - abort(); - } - } else { - if (TSRecordGetInt(item.name, &value) != TS_ERR_OKAY) { - fprintf(stderr, "Error getting stat: %s when calling TSRecordGetInt() failed: file \"%s\", line %d\n\n", item.name, - __FILE__, __LINE__); - abort(); - } - string key = item.name; - char buffer[32]; - sprintf(buffer, "%" PRId64, value); - string foo = buffer; - (*_stats)[key] = foo; - } + // Add records names to the rpc request. + request.emplace_rec(detail::MetricParam{item.name}); } } + // query the rpc node. + if (auto const &error = fetch_and_fill_stats(request, _stats); !error.empty()) { + fprintf(stderr, "Error getting stats from the RPC node:\n%s", error.c_str()); + abort(); + } _old_time = _now; _now = now; _time_diff = _now - _old_time; @@ -382,7 +385,7 @@ class Stats getStat(const string &key, string &value) { map::const_iterator lookup_it = lookup_table.find(key); - assert(lookup_it != lookup_table.end()); + ink_assert(lookup_it != lookup_table.end()); const LookupItem &item = lookup_it->second; map::const_iterator stats_it = _stats->find(item.name); @@ -400,7 +403,7 @@ class Stats value = 0; map::const_iterator lookup_it = lookup_table.find(key); - assert(lookup_it != lookup_table.end()); + ink_assert(lookup_it != lookup_table.end()); const LookupItem &item = lookup_it->second; prettyName = item.pretty; if (overrideType != 0) { @@ -542,6 +545,51 @@ class Stats return std::make_pair(s, i); } + /// Invoke the remote server and fill the responses into the stats map. + std::string + fetch_and_fill_stats(shared::rpc::RecordLookupRequest const &request, std::map *stats) noexcept + { + namespace rpc = shared::rpc; + + if (stats == nullptr) { + return "Invalid stats parameter, it shouldn't be null."; + } + try { + rpc::RPCClient rpcClient; + + // invoke the rpc. + auto const &rpcResponse = rpcClient.invoke<>(request); + + if (!rpcResponse.is_error()) { + auto const &records = rpcResponse.result.as(); + + // we check if we got some specific record error, if any we report it. + if (records.errorList.size()) { + std::stringstream ss; + + for (auto const &err : records.errorList) { + ss << err; + ss << "----\n"; + } + return ss.str(); + } else { + // No records error, so we are good to fill the list + for (auto &&recordInfo : records.recordList) { + (*stats)[recordInfo.name] = recordInfo.currentValue; + } + } + } else { + // something didn't work inside the RPC server. + std::stringstream ss; + ss << rpcResponse.error.as(); + return ss.str(); + } + } catch (std::exception const &ex) { + return {ex.what()}; + } + return {}; // no error + } + map *_stats; map *_old_stats; map lookup_table; diff --git a/src/traffic_top/traffic_top.cc b/src/traffic_top/traffic_top.cc index 2ac6b2a6289..c20c74af220 100644 --- a/src/traffic_top/traffic_top.cc +++ b/src/traffic_top/traffic_top.cc @@ -55,8 +55,7 @@ #include "tscore/I_Layout.h" #include "tscore/ink_args.h" -#include "records/I_RecProcess.h" -#include "RecordsConfig.h" +#include "tscore/I_Version.h" #include "tscore/runroot.h" using namespace std; @@ -411,30 +410,14 @@ main(int argc, const char **argv) runroot_handler(argv); Layout::create(); - RecProcessInit(RECM_STAND_ALONE, nullptr /* diags */); - LibRecordsConfigInit(); - switch (n_file_arguments) { - case 0: { - ats_scoped_str rundir(RecConfigReadRuntimeDir()); - - TSMgmtError err = TSInit(rundir, static_cast(TS_MGMT_OPT_NO_EVENTS | TS_MGMT_OPT_NO_SOCK_TESTS)); - if (err != TS_ERR_OKAY) { - fprintf(stderr, "Error: connecting to local manager: %s\n", TSGetErrorMessage(err)); - exit(1); - } - break; - } - - case 1: + if (n_file_arguments == 1) { #if HAS_CURL url = file_arguments[0]; #else usage(argument_descriptions, countof(argument_descriptions), USAGE); #endif - break; - - default: + } else if (n_file_arguments > 1) { usage(argument_descriptions, countof(argument_descriptions), USAGE); } diff --git a/src/tscore/ArgParser.cc b/src/tscore/ArgParser.cc index b7cb4af0219..ab233f9b417 100644 --- a/src/tscore/ArgParser.cc +++ b/src/tscore/ArgParser.cc @@ -32,6 +32,7 @@ #include std::string global_usage; +std::string description; std::string parser_program_name; std::string default_command; @@ -91,6 +92,10 @@ ArgParser::help_message(std::string_view err) const void ArgParser::Command::help_message(std::string_view err) const { + if (!description.empty()) { + std::cout << description << '\n'; + } + if (!err.empty()) { std::cout << "Error: " << err << std::endl; } @@ -98,6 +103,7 @@ ArgParser::Command::help_message(std::string_view err) const if (global_usage.size() > 0) { std::cout << "\nUsage: " + global_usage << std::endl; } + // output subcommands std::cout << "\nCommands ---------------------- Description -----------------------" << std::endl; std::string prefix = ""; @@ -209,6 +215,11 @@ ArgParser::get_error() const return _error_msg; } +void +ArgParser::add_description(std::string const &descr) +{ + description = descr; +} //=========================== Command class ================================ ArgParser::Command::Command() {} diff --git a/tests/gold_tests/autest-site/jsonrpc.py b/tests/gold_tests/autest-site/jsonrpc.py new file mode 100644 index 00000000000..4bbaa36aa46 --- /dev/null +++ b/tests/gold_tests/autest-site/jsonrpc.py @@ -0,0 +1,283 @@ +''' +JSONRPC Request convenient class helper. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF typing.ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import typing +from collections import OrderedDict +import json +import uuid + + +class BaseRequestType(type): + ''' + Base class for both, request and notifications + ''' + def __getattr__(cls: typing.Callable, name: str) -> typing.Callable: + def attr_handler(*args: typing.Any, **kwargs: typing.Any) -> "Request": + return cls(name, *args, **kwargs) + return attr_handler + + +class Notification(dict, metaclass=BaseRequestType): + ''' + Convenient class to create JSONRPC objects(Notifications) and strings without the need to specify redundant information. + + Examples: + + Notification.foo_bar() = > {"jsonrpc": "2.0", "method": "foo_bar"} + + Notification.foo_bar({"hello"="world"}) => + { + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "hello": "world" + } + } + + Notification.foo_bar(var={"hello":"world"}) => + { + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "var": { + "hello": "world" + } + } + } + Notification.foo_bar(fqdn=["yahoo.com", "trafficserver.org"]) => + { + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "fqdn": ["yahoo.com", "trafficserver.org"] + } + } + + ''' + + def __init__(self, method: str, *args: typing.Any, **kwargs: typing.Any): + super(Notification, self).__init__(jsonrpc='2.0', method=method) + if args and kwargs: + plist = list(args) + plist.append(kwargs) + self.update(params=plist) + elif args: + if isinstance(args, tuple): + # fix this. this is to avoid having params=[[values] which seems invalid. Double check this as [[],[]] would be ok. + self.update(params=args[0]) + else: + self.update(params=list(args)) + elif kwargs: + self.update(params=kwargs) + + # Allow using the dict item as an attribute. + def __getattr__(self, name): + if name in self: + return self[name] + else: + raise AttributeError("No such attribute: " + name) + + def is_notification(self): + return True + + def __str__(self) -> str: + return json.dumps(self) + + +class Request(Notification): + ''' + Convenient class to create JSONRPC objects and strings without the need to specify redundant information like + version or the id. All this will be generated automatically. + + Examples: + + Request.foo_bar() = > {"id": "e9ad55fe-d5a6-11eb-a2fd-fa163e6d2ec5", "jsonrpc": "2.0", "method": "foo_bar"} + + Request.foo_bar({"hello"="world"}) => + { + "id": "850d2998-d5a7-11eb-bebc-fa163e6d2ec5", + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "hello": "world" + } + } + + Request.foo_bar(var={"hello":"world"}) => + { + "id": "850d2e84-d5a7-11eb-bebc-fa163e6d2ec5", + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "var": { + "hello": "world" + } + } + } + Request.foo_bar(fqdn=["yahoo.com", "trafficserver.org"]) => + { + "id": "850d32a8-d5a7-11eb-bebc-fa163e6d2ec5", + "jsonrpc": "2.0", + "method": "foo_bar", + "params": { + "fqdn": ["yahoo.com", "trafficserver.org"] + } + } + + Note: Use full namespace to avoid name collision => jsonrpc.Request, jsonrpc.Response, etc. + ''' + + def __init__(self, method: str, *args: typing.Any, **kwargs: typing.Any): + if 'id' in kwargs: + self.update(id=kwargs.pop('id')) # avoid duplicated + else: + self.update(id=str(uuid.uuid1())) + + super(Request, self).__init__(method, *args, **kwargs) + + def is_notification(self): + return False + + +class BatchRequest(list): + def __init__(self, *args: typing.Union[Request, Notification]): + for r in args: + self.append(r) + + def add_request(self, req: typing.Union[Request, Notification]): + self.append(req) + + def __str__(self) -> str: + return json.dumps(self) + + +class Response(dict): + ''' + Convenient class to help handling jsonrpc responses. This can be source from text directly or by an already parsed test into + a json object. + ''' + + def __init__(self, *arg, **kwargs): + if 'text' in kwargs: + self.__dict__ = json.loads(kwargs['text']) + elif 'json' in kwargs: + self.__dict__ = kwargs['json'] + + def is_error(self) -> bool: + ''' + Check whether the error field is present in the response. ie: + { + "jsonrpc":"2.0", + "error":{...}, + "id":"284e0b86-d03a-11eb-9206-fa163e6d2ec5" + } + ''' + if 'error' in self.__dict__: + return True + return False + + def is_only_success(self) -> bool: + ''' + Some responses may only have set the result as success. This functions checks that the value in the response field only + contains the 'success' string, ie: + + { + "jsonrpc": "2.0", + "result": "success", + "id": "8504569c-d5a7-11eb-bebc-fa163e6d2ec5" + } + ''' + if self.is_ok() and self.result == 'success': + return True + + return False + + def is_ok(self) -> bool: + ''' + No error present in the response, the result field was properly set. + ''' + return 'result' in self.__dict__ + + def error_as_str(self): + ''' + Build up the error string. + { + "jsonrpc":"2.0", + "error":{ + "code":9, + "message":"Error during execution", + "data":[ + { + "code":10001, + "message":"No values provided" + } + ] + }, + "id":"284e0b86-d03a-11eb-9206-fa163e6d2ec5" + } + ''' + if self.is_ok(): + return "no error" + + errStr = "" + errStr += f"\n[code: {self.error['code']}, message: \"{self.error['message']}\"]" + if 'data' in self.error: + errStr += "\n\tAdditional Information:" + for err in self.error['data']: + errStr += f"\n\t - code: {err['code']}, message: \"{err['message']}\"." + return errStr + + def __str__(self) -> str: + return json.dumps(self.__dict__) + + def is_execution_error(self): + ''' + Checks if the provided error is an Execution error(9). + ''' + if self.is_ok(): + return False + + return self.error['code'] == 9 + + def contains_nested_error(self, code=None, msg=None): + if self.is_execution_error(): + for err in self.error['data']: + if code and msg: + return err['code'] == code and err['message'] == msg + elif code and err['code'] == code: + return True + elif msg and err['message'] == msg: + return True + else: + return False + return False + + +def make_response(text): + if text == '': + return None + + s = json.loads(text) + if isinstance(s, dict): + return Response(json=s) + elif isinstance(s, list): + batch = [] + for r in s: + batch.append(Response(json=r)) + return batch diff --git a/tests/gold_tests/autest-site/jsonrpc_client.test.ext b/tests/gold_tests/autest-site/jsonrpc_client.test.ext new file mode 100644 index 00000000000..974e9813d3c --- /dev/null +++ b/tests/gold_tests/autest-site/jsonrpc_client.test.ext @@ -0,0 +1,295 @@ +''' +JSONRPC client test extension. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import jsonrpc +import json +import sys +import typing +from jsonschema import validate +from jsonschema.exceptions import ValidationError +from autest.testers import Tester, tester +import hosts.output as host +from autest.exceptions.killonfailure import KillOnFailureError + + +class SchemaValidator: + ''' + Class that provides some handy schema validation function. It's in a class so some Testers can use this functionality without + exporting the class or method. + ''' + + def validate_request_schema(self, event, file_name, is_request=True, schema_file_name=None, field_schema_file_name=None): + ''' + Perform Schema validation on the JSONRPC params and also to the particular params field. + + file_name: + main json request file, schema check will be applied to the content of this file. + schema_file_name: + main doc schema file, this should only contain the wider JSONRPC 2.0 schema(not including params or result) + is_request: + This lets the function know if it should apply the 'field_schema_file_name' to the 'params' or the 'result' field. + field_schema_file_name: + param or result field schema file. + + ''' + + with open(file_name, 'r') as f: + r = f.read() + rdata = json.loads(r) + + if schema_file_name: + with open(schema_file_name, 'r') as f: + s = f.read() + sdata = json.loads(s) + try: + # validate may throw if invalid schema. + if schema_file_name: + validate(instance=rdata, schema=sdata) + + if field_schema_file_name: + fieldName = 'params' if is_request else 'result' + jsonField = rdata[fieldName] if fieldName in rdata else None + + if jsonField: + with open(field_schema_file_name, 'r') as f: + p = f.read() + psdata = json.loads(p) + validate(instance=jsonField, schema=psdata) + else: + return (False, f"There is no {fieldName} field to validate", "Error found.") + except ValidationError as ve: + event.object.Stop() + return (False, "Check JSONRPC 2.0 schema validation", str(ve)) + + return (True, "Check JSONRPC 2.0 schema validation", "All good") + + +def AddJsonRPCClientRequest(obj, ts, request='', file=None, schema_file_name=None, params_field_schema_file_name=None): + ''' + Function to add a JSONRPC request into a process. This function will internally generate a call to traffic_ctl. + As traffic_ctl can send request by reading from a file, internally this function will create a temporary json file + and will be passed as parameter to traffic_ctl, taking only the output as response (-z). + + Args: + ts: + traffic_server object, this is needed in order to traffic_ctl find the right socket. + + file: The file name used to read the request. + + request: + request should be created by the Request api(jsonrpc.py). ie: + + tr = Test.AddTestRun("Test JSONRPC foo_bar()") + tr.AddJsonRPCClientRequest(ts, Request.foo_bar(fqdn=["yahoo.com", "aol.com", "vz.com"])) + + schema_file_name: + Used to validate the request against a schema file. if empty no request schema + validation will be performed. + + params_field_schema_file_name: + Schema file to validate the params field in the jsonrpc message. + + Validating the response: + + Either by the regular validation mechanism already provided by the Testing framework or by using CustomJSONRPCResponse Tester + which will let you read the response as a dict and play with it. See CustomJSONRPCResponse for more details. + + Errors: + If there is an error in the schema validation, either the params or the whole json message, the test will not run, an exception + will be thrown with the specific error. + + ''' + + fileName = '' + process = obj.Processes.Default + if file is None: + reqFile = tempfile.NamedTemporaryFile(delete=False, dir=process.RunDirectory, suffix=f"_{obj.Name}.json") + fileName = reqFile.name + with open(fileName, "w") as req: + req.write(str(request)) + else: + fileName = file + + command = f"{ts.Variables.BINDIR}/traffic_ctl rpc file {fileName} " + if ts: + command += f" --run-root {ts.Disk.runroot_yaml.Name}" + + command += ' --format json' # we only want the output. + + process.Command = command + process.ReturnCode = 0 + + if schema_file_name != "": + process.SetupEvent.Connect( + Testers.Lambda( + lambda ev: SchemaValidator().validate_request_schema( + ev, + fileName, + True, + schema_file_name, + params_field_schema_file_name))) + return process + + +def AddJsonRPCShowRegisterHandlerRequest(obj, ts): + ''' + Handy function to request all the registered endpoints in the RPC engine. A good way to validate that your new RPC handler + is available through the RPC by calling this function and validating the response. ie: + + tr = Test.AddTestRun("Test registered API - using AddJsonRPCShowRegisterHandlerRequest") + tr.AddJsonRPCShowRegisterHandlerRequest(ts) + + tr.Processes.Default.Streams.stdout = All( + Testers.IncludesExpression('foo_bar', 'Should be listed'), + ) + ''' + return AddJsonRPCClientRequest(obj, ts, jsonrpc.Request.show_registered_handlers()) + + +# Testers +class CustomJSONRPCResponse(Tester): + + ''' + Custom tester that provides the user the ability to be called with the response from the RPC. The registered function will be + called with the jsonrpc.Response(jsonrpc.py). + + Args: + func: + The function that will be called to perform a custom validation of the jsonrpc + message. + + Example: + + tr = Test.AddTestRun("Test update_host_status") + Params = [ + {'name': 'yahoo', 'status': 'up'} + ] + + tr.AddJsonRPCClientRequest(ts, Request.update_host_status(hosts=Params)) + + + def check_no_error_on_response(resp: Response): + # we only check if it's an error. + if resp.is_error(): + return (False, resp.error_as_str()) + return (True, "All good") + + tr.Processes.Default.Streams.stdout = Testers.CustomJSONRPCResponse(check_no_error_on_response) + + ''' + + def __init__(self, + func: typing.Any, + test_value=None, + kill_on_failure: bool = False, + description_group: typing.Optional[str] = None, + description: typing.Optional[str] = None): + if description is None: + description = "Validating JSONRPC 2.0 response" + + super(CustomJSONRPCResponse, self).__init__( + value=func, + test_value=test_value, + kill_on_failure=kill_on_failure, + description_group=description_group, + description=description) + + def test(self, eventinfo, **kw): + + response_text = {} + with open(self._GetContent(eventinfo), "r") as resp: + response_text = resp.read() + + (testPassed, reason) = self.Value(jsonrpc.Response(text=response_text)) + + if testPassed: + self.Result = tester.ResultType.Passed + self.Reason = f"Returned value: {reason}" + host.WriteVerbose( + ["testers.CustomJSONRPCResponse", "testers"], f"tester.ResultType.to_color_string(self.Result) - ", self.Reason) + else: + self.Result = tester.ResultType.Failed + self.Reason = f"Returned value: {reason}" + if self.KillOnFailure: + raise KillOnFailureError + +# Testers + + +class JSONRPCResponseSchemaValidator(Tester, SchemaValidator): + + ''' + Tester for response schema validation. + This class can perform a JSONRPC 2.0 schema validation and also the 'result' field validation if provided. + + schema_file_name: + Main JSONRPC 2.0 schema validation file. + + result_field_schema_file_name: + result field schema validation, this is optional, if not provided the main schema will just check that the result matches + the JSONRPC 2.0 specs. + ''' + + def __init__(self, + schema_file_name, + result_field_schema_file_name=None, + value=None, + test_value=None, + kill_on_failure: bool = False, + description_group: typing.Optional[str] = None, + description: typing.Optional[str] = None): + if description is None: + description = "Validating JSONRPC 2.0 response schema" + self._schema_file_name = schema_file_name + self._result_field_schema_file_name = result_field_schema_file_name + + super(JSONRPCResponseSchemaValidator, self).__init__( + value=value, + test_value=test_value, + kill_on_failure=kill_on_failure, + description_group=description_group, + description=description) + + def test(self, eventinfo, **kw): + response_text = {} + with open(self._GetContent(eventinfo), "r") as resp: + response_text = resp.read() + + (testPassed, reason, cmm) = self.validate_request_schema(eventinfo, self._GetContent( + eventinfo), False, self._schema_file_name, self._result_field_schema_file_name) + + if testPassed: + self.Result = tester.ResultType.Passed + self.Reason = f"Returned value: {reason}" + host.WriteVerbose(["testers.JSONRPCResponseSchemaValidator", "testers"], + f"tester.ResultType.to_color_string(self.Result) - ", self.Reason) + else: + self.Result = tester.ResultType.Failed + self.Reason = f"Returned value: {reason}" + if self.KillOnFailure: + raise KillOnFailureError + + +# Export +AddTester(CustomJSONRPCResponse) +AddTester(JSONRPCResponseSchemaValidator) +ExtendTestRun(AddJsonRPCShowRegisterHandlerRequest, name="AddJsonRPCShowRegisterHandlerRequest") +ExtendTestRun(AddJsonRPCClientRequest, name="AddJsonRPCClientRequest") diff --git a/tests/gold_tests/autest-site/trafficserver.test.ext b/tests/gold_tests/autest-site/trafficserver.test.ext index cca27a59a85..44b968f3774 100755 --- a/tests/gold_tests/autest-site/trafficserver.test.ext +++ b/tests/gold_tests/autest-site/trafficserver.test.ext @@ -40,7 +40,7 @@ default_log_data = { def MakeATSProcess(obj, name, command='traffic_server', select_ports=True, enable_tls=False, enable_cache=True, enable_quic=False, - block_for_debug=False, log_data=default_log_data): + block_for_debug=False, log_data=default_log_data, dump_runroot=False): ##################################### # common locations @@ -300,6 +300,22 @@ def MakeATSProcess(obj, name, command='traffic_server', select_ports=True, tmpname = os.path.join(config_dir, fname) p.Disk.File(tmpname, id=make_id(fname), typename="ats:config") + # The big motivation in exposing this file is that we need to tell the traffic_ctl + # where to find the socket to interact with the TS. traffic_ctl cannot rely only + # in the build layout for unit test. + fname = "runroot.yaml" + tmpname = os.path.join(config_dir, fname) + p.Disk.File(tmpname, id=make_id(fname), typename="ats:config") + + # Expect the instruction to set the variables inside the runroot. This is not mandatory + if dump_runroot: + p.Disk.runroot_yaml.AddLine(f'runtimedir: {runtime_dir}') + + def SetRunRootEnv(self): + self.Env['TS_RUNROOT'] = os.path.join(config_dir, "runroot.yaml") + + AddMethodToInstance(p, SetRunRootEnv) + ########################################################## # set up default ports # get some ports TODO make it so we can hold on to the socket diff --git a/tests/gold_tests/basic/basic-manager.test.py b/tests/gold_tests/basic/basic-manager.test.py index 909ae349d89..c9dddefdb4c 100644 --- a/tests/gold_tests/basic/basic-manager.test.py +++ b/tests/gold_tests/basic/basic-manager.test.py @@ -20,6 +20,8 @@ Test that Trafficserver starts with default configurations. ''' +Test.SkipIf(Condition.true("We do not need to test traffic_manager (JSONRPC)")) + ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) t = Test.AddTestRun("Test traffic server started properly") diff --git a/tests/gold_tests/cache/cache-generation-clear.test.py b/tests/gold_tests/cache/cache-generation-clear.test.py index e066aee4854..df2af0be5c0 100644 --- a/tests/gold_tests/cache/cache-generation-clear.test.py +++ b/tests/gold_tests/cache/cache-generation-clear.test.py @@ -23,7 +23,7 @@ ''' Test.ContinueOnFail = True # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager") +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) # setup some config file for this server ts.Disk.records_config.update({ @@ -31,6 +31,7 @@ 'proxy.config.http.cache.generation': -1, # Start with cache turned off 'proxy.config.config_update_interval_ms': 1, }) + ts.Disk.plugin_config.AddLine('xdebug.so') ts.Disk.remap_config.AddLines([ 'map /default/ http://127.0.0.1/ @plugin=generator.so', @@ -62,7 +63,7 @@ # Call traffic_ctrl to set new generation tr = Test.AddTestRun() -tr.Processes.Default.Command = 'traffic_ctl --debug config set proxy.config.http.cache.generation 77' +tr.Processes.Default.Command = f'traffic_ctl --debug config set proxy.config.http.cache.generation 77 --run-root {ts.Disk.runroot_yaml.Name}' tr.Processes.Default.ForceUseShell = False tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.Env = ts.Env # set the environment for traffic_control to run in diff --git a/tests/gold_tests/continuations/double.test.py b/tests/gold_tests/continuations/double.test.py index ef5875f1a1a..1ed9f86077b 100644 --- a/tests/gold_tests/continuations/double.test.py +++ b/tests/gold_tests/continuations/double.test.py @@ -24,9 +24,12 @@ Test.ContinueOnFail = True # Define default ATS. Disable the cache to simplify the test. -ts = Test.MakeATSProcess("ts", select_ports=True, command="traffic_manager", enable_cache=False) +ts = Test.MakeATSProcess("ts", select_ports=True, command="traffic_server", enable_cache=False, dump_runroot=True) server = Test.MakeOriginServer("server") +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + Test.testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: double_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server diff --git a/tests/gold_tests/continuations/double_h2.test.py b/tests/gold_tests/continuations/double_h2.test.py index 97fe0280595..e825f80fafe 100644 --- a/tests/gold_tests/continuations/double_h2.test.py +++ b/tests/gold_tests/continuations/double_h2.test.py @@ -26,10 +26,14 @@ ) Test.ContinueOnFail = True # Define default ATS. Disable the cache to simplify the test. -ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True, command="traffic_manager", enable_cache=False) +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True, command="traffic_server", enable_cache=False, dump_runroot=True) server = Test.MakeOriginServer("server") server2 = Test.MakeOriginServer("server2") + +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + Test.testName = "" request_header = {"headers": "GET / HTTP/1.1\r\nHost: double_h2.test\r\n\r\n", "timestamp": "1469733493.993", "body": ""} # expected response from the origin server diff --git a/tests/gold_tests/continuations/openclose.test.py b/tests/gold_tests/continuations/openclose.test.py index 178137a6b86..b451464fde1 100644 --- a/tests/gold_tests/continuations/openclose.test.py +++ b/tests/gold_tests/continuations/openclose.test.py @@ -23,7 +23,7 @@ ''' # Define default ATS. Disable the cache to simplify the test. -ts = Test.MakeATSProcess("ts", command="traffic_manager", enable_cache=False) +ts = Test.MakeATSProcess("ts", command="traffic_server", enable_cache=False, dump_runroot=True) server = Test.MakeOriginServer("server") server2 = Test.MakeOriginServer("server2") @@ -71,6 +71,9 @@ server.StartAfter(*ps) tr.StillRunningAfter = ts + +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() # Signal that all the curl processes have completed tr = Test.AddTestRun("Curl Done") tr.DelayStart = 2 # Delaying a couple seconds to make sure the global continuation's lock contention resolves. diff --git a/tests/gold_tests/continuations/openclose_h2.test.py b/tests/gold_tests/continuations/openclose_h2.test.py index add949dd68f..19dcd84a3af 100644 --- a/tests/gold_tests/continuations/openclose_h2.test.py +++ b/tests/gold_tests/continuations/openclose_h2.test.py @@ -27,7 +27,8 @@ ) # Define default ATS. Disable the cache to simplify the test. -ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True, command="traffic_manager", enable_cache=False) +ts = Test.MakeATSProcess("ts", select_ports=True, enable_tls=True, command="traffic_server", enable_cache=False, dump_runroot=True) + server = Test.MakeOriginServer("server") server2 = Test.MakeOriginServer("server2") @@ -84,6 +85,9 @@ server.StartAfter(*ps) tr.StillRunningAfter = ts +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + # Signal that all the curl processes have completed tr = Test.AddTestRun("Curl Done") tr.DelayStart = 2 # Delaying a couple seconds to make sure the global continuation's lock contention resolves. diff --git a/tests/gold_tests/continuations/session_id.test.py b/tests/gold_tests/continuations/session_id.test.py index 9e8f90d761d..4397b901715 100644 --- a/tests/gold_tests/continuations/session_id.test.py +++ b/tests/gold_tests/continuations/session_id.test.py @@ -34,7 +34,7 @@ server.addResponse("sessionfile.log", request_header, response_header) # Configure ATS. Disable the cache to simplify the test. -ts = Test.MakeATSProcess("ts", command="traffic_manager", enable_tls=True, enable_cache=False) +ts = Test.MakeATSProcess("ts", command="traffic_server", enable_tls=True, enable_cache=False) ts.addDefaultSSLFiles() diff --git a/tests/gold_tests/headers/forwarded.test.py b/tests/gold_tests/headers/forwarded.test.py index 5478c5f7c2f..600f1769dc2 100644 --- a/tests/gold_tests/headers/forwarded.test.py +++ b/tests/gold_tests/headers/forwarded.test.py @@ -192,11 +192,12 @@ def TestHttp1_1(host): TestHttp1_1('www.forwarded-connection-std.com') TestHttp1_1('www.forwarded-connection-full.com') -ts2 = Test.MakeATSProcess("ts2", command="traffic_manager", enable_tls=True) +ts2 = Test.MakeATSProcess("ts2", command="traffic_server", enable_tls=True, dump_runroot=True) baselineTsSetup(ts2) ts2.Disk.records_config.update({ + # 'proxy.config.diags.debug.enabled': 1, 'proxy.config.url_remap.pristine_host_hdr': 1, # Retain Host header in original incoming client request. 'proxy.config.http.insert_forwarded': 'by=uuid'}) @@ -204,6 +205,9 @@ def TestHttp1_1(host): 'map https://www.no-oride.com http://127.0.0.1:{0}'.format(server.Variables.Port) ) +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts2.SetRunRootEnv() + # Forwarded header with UUID of 2nd ATS. tr = Test.AddTestRun() # Delay on readiness of our ssl ports diff --git a/tests/gold_tests/ip_allow/ip_allow.test.py b/tests/gold_tests/ip_allow/ip_allow.test.py index f6504cbe5f2..3149bc63f2f 100644 --- a/tests/gold_tests/ip_allow/ip_allow.test.py +++ b/tests/gold_tests/ip_allow/ip_allow.test.py @@ -24,7 +24,7 @@ Test.ContinueOnFail = True # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, enable_tls=True, enable_cache=False) server = Test.MakeOriginServer("server", ssl=True) diff --git a/tests/gold_tests/jsonrpc/json/admin_clear_metrics_records_req.json b/tests/gold_tests/jsonrpc/json/admin_clear_metrics_records_req.json new file mode 100644 index 00000000000..24881bfa93b --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_clear_metrics_records_req.json @@ -0,0 +1,10 @@ +{ + "id":"0ea2672f-b369-4d99-8d46-98c3fadc152d", + "jsonrpc":"2.0", + "method":"admin_clear_metrics_records", + "params":[ + { + "record_name":"$record_name" + } + ] +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_config_reload_req.json b/tests/gold_tests/jsonrpc/json/admin_config_reload_req.json new file mode 100644 index 00000000000..19fda47bb79 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_config_reload_req.json @@ -0,0 +1,5 @@ +{ + "id":"71588e95-4f11-43a9-9c7d-9942e017548c", + "jsonrpc":"2.0", + "method":"admin_config_reload" +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_config_set_records_req.json b/tests/gold_tests/jsonrpc/json/admin_config_set_records_req.json new file mode 100644 index 00000000000..00df6113611 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_config_set_records_req.json @@ -0,0 +1,11 @@ +{ + "id": "a32de1da-08be-11eb-9e1e-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_config_set_records", + "params": [ + { + "record_name": "$record_name", + "record_value": "$record_value" + } + ] +} diff --git a/tests/gold_tests/jsonrpc/json/admin_host_set_status_req.json b/tests/gold_tests/jsonrpc/json/admin_host_set_status_req.json new file mode 100644 index 00000000000..8079159ad77 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_host_set_status_req.json @@ -0,0 +1,11 @@ +{ + "id": "c6d56fba-0cbd-11eb-926d-001fc69cc946", + "jsonrpc": "2.0", + "method": "admin_host_set_status", + "params": { + "operation": "$operation", + "host": ["$host"], + "reason": "manual", + "time": "100" + } +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_1.json b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_1.json new file mode 100644 index 00000000000..7be3e889f1f --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_1.json @@ -0,0 +1,14 @@ +{ + "id":"38a9cc7e-5d41-4415-b74f-487ee1f01217", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"$record_name", + "rec_types":[ + "1", + "16" + ] + } + ] +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_invalid_rec.json b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_invalid_rec.json new file mode 100644 index 00000000000..980796f82a0 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_invalid_rec.json @@ -0,0 +1,14 @@ +{ + "id":"6ec0e08a-c9e4-4d95-b65b-294eacd7f13d", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"some.invalid.record.name", + "rec_types":[ + "1", + "16" + ] + } + ] +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_metric.json b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_metric.json new file mode 100644 index 00000000000..78d2e77d4f4 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_metric.json @@ -0,0 +1,11 @@ +{ + "id":"5dee0f4d-94b9-4132-a6d8-f7924d3a6dac", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "rec_types": [2, 4, 32], + "record_name_regex": "$record_name_regex" + } + ] +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_multiple.json b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_multiple.json new file mode 100644 index 00000000000..c380d5b43e2 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_lookup_records_req_multiple.json @@ -0,0 +1,18 @@ +{ + "id":"38a9cc7e-5d41-4415-b74f-487ee1f01217", + "jsonrpc":"2.0", + "method":"admin_lookup_records", + "params":[ + { + "record_name":"$record_name", + "rec_types":[ + "1", + "16" + ] + }, + { + "rec_types": [987], + "record_name": "$record_name" + } + ] +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_plugin_send_basic_msg_req.json b/tests/gold_tests/jsonrpc/json/admin_plugin_send_basic_msg_req.json new file mode 100644 index 00000000000..0b0bd96c345 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_plugin_send_basic_msg_req.json @@ -0,0 +1,9 @@ +{ + "id":"dd0ac6ad-1afc-4db6-b584-a2a02990940f", + "jsonrpc":"2.0", + "method":"admin_plugin_send_basic_msg", + "params":{ + "tag":"some_tag", + "data":"some_data" + } +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/admin_storage_x_device_status_req.json b/tests/gold_tests/jsonrpc/json/admin_storage_x_device_status_req.json new file mode 100644 index 00000000000..bbc420fccbb --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/admin_storage_x_device_status_req.json @@ -0,0 +1,8 @@ +{ + "id":"9893668b-8c58-477e-a962-39904247557b", + "jsonrpc":"2.0", + "method":"$method", + "params":[ + "$device" + ] +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/json/method_call_no_params.json b/tests/gold_tests/jsonrpc/json/method_call_no_params.json new file mode 100644 index 00000000000..21c6d3a6c03 --- /dev/null +++ b/tests/gold_tests/jsonrpc/json/method_call_no_params.json @@ -0,0 +1,5 @@ +{ + "id":"0ea2672f-b369-4d99-8d46-98c3fadc152d", + "jsonrpc":"2.0", + "method":"$method" +} \ No newline at end of file diff --git a/tests/gold_tests/jsonrpc/jsonrpc_api_schema.test.py b/tests/gold_tests/jsonrpc/jsonrpc_api_schema.test.py new file mode 100644 index 00000000000..8d6eda0a282 --- /dev/null +++ b/tests/gold_tests/jsonrpc/jsonrpc_api_schema.test.py @@ -0,0 +1,200 @@ +''' +JSONRPC Schema test. This test will run a basic request/response schema validation. +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import sys +import tempfile +from string import Template + +Test.Summary = 'Test jsonrpc admin API' + + +# set the schema folder. +schema_folder = os.path.join(Test.TestDirectory, '..', '..', '..', "mgmt2", "rpc", "schema") + + +def substitute_context_in_file(process, file, context): + ''' + Perform substitution based on the passed context dict. This function will return a new path for the substituted file. + ''' + if os.path.isdir(file): + raise ValueError(f"Mapping substitution not supported for directories.") + + with open(os.path.join(process.TestDirectory, file), 'r') as req_file: + req_template = Template(req_file.read()) + req_content = req_template.substitute(context) + tf = tempfile.NamedTemporaryFile(delete=False, dir=process.RunDirectory, suffix=f"_{os.path.basename(file)}") + file = tf.name + with open(file, "w") as new_req_file: + new_req_file.write(req_content) + + return file + + +def add_testrun_for_jsonrpc_request( + test_description, + request_file_name, + params_schema_file_name=None, + result_schema_file_name=None, + context=None): + ''' + Simple wrapper around the AddJsonRPCClientRequest method. + + context: + This can be used if a template substitution is needed in the request file. + + stdout_testers: + Testers to be run on the output stream. + + params_schema_file_name: + Schema file to validate the request 'params' field. + + result_schema_file_name: + Schema file to validate the response 'result' field. + ''' + tr = Test.AddTestRun(test_description) + tr.Setup.Copy(request_file_name) + + if context: + request_file_name = substitute_context_in_file(tr, request_file_name, context) + + request_schema_file_name = os.path.join(schema_folder, "jsonrpc_request_schema.json") + tr.AddJsonRPCClientRequest( + ts, + file=os.path.join( + ts.RunDirectory, + os.path.basename(request_file_name)), + schema_file_name=request_schema_file_name, + params_field_schema_file_name=params_schema_file_name) + + tr.Processes.Default.ReturnCode = 0 + + response_schema_file_name = os.path.join(schema_folder, "jsonrpc_response_schema.json") + tr.Processes.Default.Streams.stdout = Testers.JSONRPCResponseSchemaValidator( + schema_file_name=response_schema_file_name, result_field_schema_file_name=result_schema_file_name) + + tr.StillRunningAfter = ts + return tr + + +ts = Test.MakeATSProcess('ts', enable_cache=True, dump_runroot=True) +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + +Test.testName = 'Basic JSONRPC API test' + +ts.Disk.records_config.update({ + 'proxy.config.diags.debug.enabled': 1, + 'proxy.config.diags.debug.tags': 'rpc|filemanager|http|cache', + 'proxy.config.jsonrpc.filename': "jsonrpc.yaml", # We will be using this record to tests some RPC API. +}) + +# One of the API's will be checking the storage. Need this to get a response with content. +storage_path = os.path.join(Test.RunDirectory, "ts", "storage") +ts.Disk.storage_config.AddLine(f"{storage_path} 512M") + + +# The following tests will only validate the jsonrpc message, it will not run any validation on the content of the 'result' or 'params' +# of the jsonrpc message. This should be added once the schemas are avilable. + +# jsonrpc 2.0 schema file. This will not check the param fields. + +success_schema_file_name_name = os.path.join(schema_folder, "success_response_schema.json") +# admin_lookup_records + + +params_schema_file_name = os.path.join(schema_folder, "admin_lookup_records_params_schema.json") +first = add_testrun_for_jsonrpc_request("Test admin_lookup_records", + request_file_name='json/admin_lookup_records_req_1.json', + params_schema_file_name=params_schema_file_name, + context={'record_name': 'proxy.config.jsonrpc.filename'}) +first.Processes.Default.StartBefore(ts) + +add_testrun_for_jsonrpc_request("Test admin_lookup_records w/error", + request_file_name='json/admin_lookup_records_req_invalid_rec.json') + +add_testrun_for_jsonrpc_request("Test admin_lookup_records", + request_file_name='json/admin_lookup_records_req_1.json', + context={'record_name': 'proxy.config.jsonrpc.filename'}) + +add_testrun_for_jsonrpc_request("Test admin_lookup_records w/error", + request_file_name='json/admin_lookup_records_req_invalid_rec.json') + +add_testrun_for_jsonrpc_request("Test admin_lookup_records w/error", + request_file_name='json/admin_lookup_records_req_multiple.json', + context={'record_name': 'proxy.config.jsonrpc.filename'}) + +add_testrun_for_jsonrpc_request("Test admin_lookup_records w/error", + request_file_name='json/admin_lookup_records_req_metric.json', + context={'record_name_regex': 'proxy.process.http.total_client_connections_ipv4*'}) + + +# admin_config_set_records +add_testrun_for_jsonrpc_request("Test admin_lookup_records w/error", request_file_name='json/admin_config_set_records_req.json', + context={'record_name': 'proxy.config.jsonrpc.filename', 'record_value': 'test_jsonrpc.yaml'}) + +# admin_config_reload +add_testrun_for_jsonrpc_request("Test admin_config_reload", request_file_name='json/admin_config_reload_req.json', + result_schema_file_name=success_schema_file_name_name) + +# admin_clear_metrics_records +add_testrun_for_jsonrpc_request("Clear admin_clear_metrics_records", request_file_name='json/admin_clear_metrics_records_req.json', + context={'record_name': 'proxy.process.http.404_responses'}) + +# admin_host_set_status +add_testrun_for_jsonrpc_request("Test admin_host_set_status", request_file_name='json/admin_host_set_status_req.json', + context={'operation': 'up', 'host': 'my.test.host.trafficserver.com'}) + +# admin_host_set_status +add_testrun_for_jsonrpc_request("Test admin_host_set_status", request_file_name='json/admin_host_set_status_req.json', + context={'operation': 'down', 'host': 'my.test.host.trafficserver.com'}) + + +# admin_server_start_drain +add_testrun_for_jsonrpc_request("Test admin_server_start_drain", request_file_name='json/method_call_no_params.json', + context={'method': 'admin_server_start_drain'}) + +add_testrun_for_jsonrpc_request("Test admin_server_start_drain", + request_file_name='json/method_call_no_params.json', + context={'method': 'admin_server_start_drain'}) + +# admin_server_stop_drain +add_testrun_for_jsonrpc_request("Test admin_server_stop_drain", request_file_name='json/method_call_no_params.json', + context={'method': 'admin_server_stop_drain'}) + +# admin_storage_get_device_status +add_testrun_for_jsonrpc_request( + "Test admin_storage_get_device_status", + request_file_name='json/admin_storage_x_device_status_req.json', + context={ + 'method': 'admin_storage_get_device_status', + 'device': f'{storage_path}/cache.db'}) + +# admin_storage_set_device_offline +add_testrun_for_jsonrpc_request( + "Test admin_storage_set_device_offline", + request_file_name='json/admin_storage_x_device_status_req.json', + context={ + 'method': 'admin_storage_set_device_offline', + 'device': f'{storage_path}/cache.db'}) + +# admin_plugin_send_basic_msg +add_testrun_for_jsonrpc_request("Test admin_plugin_send_basic_msg", request_file_name='json/admin_plugin_send_basic_msg_req.json', + result_schema_file_name=success_schema_file_name_name) diff --git a/tests/gold_tests/logging/log_retention.test.py b/tests/gold_tests/logging/log_retention.test.py index 605bfd84a26..5eb559d62f2 100644 --- a/tests/gold_tests/logging/log_retention.test.py +++ b/tests/gold_tests/logging/log_retention.test.py @@ -50,7 +50,7 @@ class TestLogRetention: __ts_counter = 0 __server_is_started = False - def __init__(self, records_config, run_description, command="traffic_manager"): + def __init__(self, records_config, run_description, command="traffic_server"): """ Create a TestLogRetention instance. """ @@ -101,7 +101,7 @@ def __create_server(cls): cls.__server = server return cls.__server - def __create_ts(self, records_config, command="traffic_manager"): + def __create_ts(self, records_config, command="traffic_server"): """ Create an ATS process. @@ -110,7 +110,7 @@ def __create_ts(self, records_config, command="traffic_manager"): """ ts_name = "ts{counter}".format(counter=TestLogRetention.__ts_counter) TestLogRetention.__ts_counter += 1 - self.ts = Test.MakeATSProcess(ts_name, command=command) + self.ts = Test.MakeATSProcess(ts_name, command=command, dump_runroot=True) combined_records_config = TestLogRetention.__base_records_config.copy() combined_records_config.update(records_config) @@ -526,6 +526,9 @@ def get_command_to_rotate_thrice(self): test.tr.StillRunningAfter = test.ts test.tr.StillRunningAfter = test.server +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +test.ts.SetRunRootEnv() + tr = Test.AddTestRun("Perform a config reload") tr.Processes.Default.Command = "traffic_ctl config reload" tr.Processes.Default.Env = test.ts.Env diff --git a/tests/gold_tests/pluginTest/cert_update/cert_update.test.py b/tests/gold_tests/pluginTest/cert_update/cert_update.test.py index dbda8bd335b..69bf0ba68d1 100644 --- a/tests/gold_tests/pluginTest/cert_update/cert_update.test.py +++ b/tests/gold_tests/pluginTest/cert_update/cert_update.test.py @@ -36,7 +36,7 @@ server.addResponse("sessionlog.json", request_header, response_header) # Set up ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager", enable_tls=1) +ts = Test.MakeATSProcess("ts", command="traffic_server", enable_tls=1, dump_runroot=True) # Set up ssl files ts.addSSLfile("ssl/server1.pem") @@ -44,6 +44,9 @@ ts.addSSLfile("ssl/client1.pem") ts.addSSLfile("ssl/client2.pem") +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + # reserve port, attach it to 'ts' so it is released later ports.get_port(ts, 's_server_port') diff --git a/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py b/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py index 40b9c86fe3e..a3ae401b1c0 100644 --- a/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py +++ b/tests/gold_tests/pluginTest/client_context_dump/client_context_dump.test.py @@ -25,7 +25,7 @@ Test.SkipUnless(Condition.PluginExists('client_context_dump.so')) # Set up ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, enable_tls=True, dump_runroot=True) # Set up ssl files ts.addSSLfile("ssl/one.com.pem") @@ -73,6 +73,6 @@ tr = Test.AddTestRun() tr.Processes.Default.Env = ts.Env tr.Processes.Default.Command = ( - '{0}/traffic_ctl plugin msg client_context_dump.t 1'.format(ts.Variables.BINDIR) + '{0}/traffic_ctl plugin msg client_context_dump.t 1 --run-root {1}'.format(ts.Variables.BINDIR, ts.Disk.runroot_yaml.Name) ) tr.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py b/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py index 47dc09b6b01..75fc97d2f2c 100644 --- a/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py +++ b/tests/gold_tests/pluginTest/lua/lua_states_stats.test.py @@ -30,10 +30,9 @@ # It is necessary to redirect stderr to a file so it will be available for examination by a test run. ts = Test.MakeATSProcess( - "ts", command="traffic_manager 2> " + Test.RunDirectory + "/ts.stderr.txt", select_ports=True + "ts", command="traffic_server 2> " + Test.RunDirectory + "/ts.stderr.txt", select_ports=True, dump_runroot=True ) -# For unknown reasons, traffic_manager returns 2 instead of 0 on exit with stderr redirect here. -ts.ReturnCode = 2 +ts.ReturnCode = 0 Test.testName = "Lua states and stats" @@ -65,6 +64,9 @@ 'proxy.config.plugin.lua.max_states': 4, }) +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + curl_and_args = 'curl -s -D /dev/stdout -o /dev/stderr -x localhost:{} '.format(ts.Variables.port) # 0 Test - Check for configured lua states diff --git a/tests/gold_tests/pluginTest/regex_revalidate/metrics.sh b/tests/gold_tests/pluginTest/regex_revalidate/metrics.sh index 4365a0e6e02..7616bf28398 100755 --- a/tests/gold_tests/pluginTest/regex_revalidate/metrics.sh +++ b/tests/gold_tests/pluginTest/regex_revalidate/metrics.sh @@ -18,7 +18,7 @@ N=60 while (( N > 0 )) do rm -f metrics.out - traffic_ctl metric match regex_revalidate > metrics.out + traffic_ctl metric match regex_revalidate --run-root $1 > metrics.out sleep 1 if diff metrics.out ${AUTEST_TEST_DIR}/gold/metrics.gold then diff --git a/tests/gold_tests/pluginTest/regex_revalidate/metrics_miss.sh b/tests/gold_tests/pluginTest/regex_revalidate/metrics_miss.sh index 164a4e45277..0102961ae7a 100755 --- a/tests/gold_tests/pluginTest/regex_revalidate/metrics_miss.sh +++ b/tests/gold_tests/pluginTest/regex_revalidate/metrics_miss.sh @@ -18,7 +18,7 @@ N=60 while (( N > 0 )) do rm -f metrics.out - traffic_ctl metric match regex_revalidate > metrics.out + traffic_ctl metric match regex_revalidate --run-root $1 > metrics.out sleep 1 if diff metrics.out ${AUTEST_TEST_DIR}/gold/metrics_miss.gold then diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py index 73e4b50107b..0e2082f57dd 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate.test.py @@ -18,6 +18,8 @@ import os import time +from jsonrpc import Request + Test.Summary = ''' Test a basic regex_revalidate ''' @@ -42,7 +44,7 @@ server = Test.MakeOriginServer("server") # Define ATS and configure -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) Test.testName = "regex_revalidate" Test.Setup.Copy("metrics.sh") @@ -194,9 +196,7 @@ ]) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'traffic_ctl config reload' -# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket -tr.Processes.Default.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 tr.TimeOut = 5 @@ -228,9 +228,7 @@ ]) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'traffic_ctl config reload' -# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket -tr.Processes.Default.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 tr.TimeOut = 5 @@ -265,9 +263,7 @@ ]) tr.StillRunningAfter = ts tr.StillRunningAfter = server -tr.Processes.Default.Command = 'traffic_ctl config reload' -# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket -tr.Processes.Default.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 tr.TimeOut = 5 @@ -283,7 +279,7 @@ # 12 Stats check tr = Test.AddTestRun("Check stats") tr.DelayStart = 5 -tr.Processes.Default.Command = "bash -c ./metrics.sh" +tr.Processes.Default.Command = f"bash -c './metrics.sh {ts.Disk.runroot_yaml.Name}'" tr.Processes.Default.Env = ts.Env tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py index ea28b503809..1499e5157eb 100644 --- a/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py +++ b/tests/gold_tests/pluginTest/regex_revalidate/regex_revalidate_miss.test.py @@ -18,6 +18,8 @@ import os import time +from jsonrpc import Request + Test.Summary = ''' regex_revalidate plugin test, MISS (refetch) functionality ''' @@ -36,7 +38,7 @@ server = Test.MakeOriginServer("server") # Define ATS and configure -ts = Test.MakeATSProcess("ts", command="traffic_manager") +ts = Test.MakeATSProcess("ts", command="traffic_server", dump_runroot=True) Test.testName = "regex_revalidate_miss" Test.Setup.Copy("metrics_miss.sh") @@ -139,9 +141,7 @@ tr.Disk.File(regex_revalidate_conf_path + "_tr2", typename="ats:config").AddLine(path1_rule + ' MISS') tr.StillRunningAfter = ts tr.StillRunningAfter = server -ps.Command = 'traffic_ctl config reload' -# Need to copy over the environment so traffic_ctl knows where to find the unix domain socket -ps.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) ps.ReturnCode = 0 ps.TimeOut = 5 tr.TimeOut = 5 @@ -170,8 +170,7 @@ tr.Disk.File(regex_revalidate_conf_path + "_tr5", typename="ats:config").AddLine(path1_rule + ' STALE') tr.StillRunningAfter = ts tr.StillRunningAfter = server -ps.Command = 'traffic_ctl config reload' -ps.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) ps.ReturnCode = 0 ps.TimeOut = 5 tr.TimeOut = 5 @@ -192,8 +191,7 @@ tr.Disk.File(regex_revalidate_conf_path + "_tr7", typename="ats:config").AddLine(path1_rule + ' MISS') tr.StillRunningAfter = ts tr.StillRunningAfter = server -ps.Command = 'traffic_ctl config reload' -ps.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) ps.ReturnCode = 0 ps.TimeOut = 5 tr.TimeOut = 5 @@ -214,8 +212,7 @@ tr.Disk.File(regex_revalidate_conf_path + "_tr9", typename="ats:config").AddLine(path1_rule + ' MISSSTALE') tr.StillRunningAfter = ts tr.StillRunningAfter = server -ps.Command = 'traffic_ctl config reload' -ps.Env = ts.Env +tr.AddJsonRPCClientRequest(ts, Request.admin_config_reload()) ps.ReturnCode = 0 ps.TimeOut = 5 tr.TimeOut = 5 @@ -232,7 +229,7 @@ # 11 Stats check tr = Test.AddTestRun("Check stats") tr.DelayStart = 5 -tr.Processes.Default.Command = "bash -c ./metrics_miss.sh" +tr.Processes.Default.Command = f"bash -c './metrics_miss.sh {ts.Disk.runroot_yaml.Name}'" tr.Processes.Default.Env = ts.Env tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = ts diff --git a/tests/gold_tests/pluginTest/remap_stats/remap_stats.test.py b/tests/gold_tests/pluginTest/remap_stats/remap_stats.test.py index 727ab9abe8f..3dafdb98d43 100644 --- a/tests/gold_tests/pluginTest/remap_stats/remap_stats.test.py +++ b/tests/gold_tests/pluginTest/remap_stats/remap_stats.test.py @@ -29,7 +29,7 @@ "timestamp": "1469733493.993", "body": ""} server.addResponse("sessionlog.json", request_header, response_header) -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) ts.Disk.plugin_config.AddLine('remap_stats.so') @@ -64,7 +64,7 @@ # 2 Test - Gather output tr = Test.AddTestRun("analyze stats") -tr.Processes.Default.Command = r'traffic_ctl metric match \.\*remap_stats\*' +tr.Processes.Default.Command = r'traffic_ctl metric match \.\*remap_stats\* {0}'.format(ts.Disk.runroot_yaml.Name) tr.Processes.Default.Env = ts.Env tr.Processes.Default.ReturnCode = 0 tr.Processes.Default.TimeOut = 5 diff --git a/tests/gold_tests/remap/conf_remap_float.py b/tests/gold_tests/remap/conf_remap_float.py index 805f24e01f7..92449b5c012 100644 --- a/tests/gold_tests/remap/conf_remap_float.py +++ b/tests/gold_tests/remap/conf_remap_float.py @@ -24,7 +24,10 @@ ''' Test.testName = 'Float in conf_remap Config Test' -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) + +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() # Add dummy remap rule ts.Disk.remap_config.AddLine( @@ -47,6 +50,6 @@ tr.StillRunningAfter = ts p = tr.Processes.Default -p.Command = "traffic_ctl config describe proxy.config.http.background_fill_completed_threshold" +p.Command = f"traffic_ctl config describe proxy.config.http.background_fill_completed_threshold --run-root {ts.Disk.runroot_yaml.Name}" p.ReturnCode = 0 p.StartBefore(Test.Processes.ts, ready=When.FileExists(os.path.join(tr.RunDirectory, 'ts/log/diags.log'))) diff --git a/tests/gold_tests/tls/tls_check_cert_select_plugin.test.py b/tests/gold_tests/tls/tls_check_cert_select_plugin.test.py index b189060d1ac..3cf99a01aea 100644 --- a/tests/gold_tests/tls/tls_check_cert_select_plugin.test.py +++ b/tests/gold_tests/tls/tls_check_cert_select_plugin.test.py @@ -23,7 +23,8 @@ ''' # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, enable_tls=True, dump_runroot=True) +ts.SetRunRootEnv() server = Test.MakeOriginServer("server", ssl=True) dns = Test.MakeDNServer("dns") diff --git a/tests/gold_tests/tls/tls_check_cert_selection_reload.test.py b/tests/gold_tests/tls/tls_check_cert_selection_reload.test.py index edd18e7df55..75227b7f020 100644 --- a/tests/gold_tests/tls/tls_check_cert_selection_reload.test.py +++ b/tests/gold_tests/tls/tls_check_cert_selection_reload.test.py @@ -21,7 +21,8 @@ ''' # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager", enable_tls=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", enable_tls=True, dump_runroot=True) +ts.SetRunRootEnv() server = Test.MakeOriginServer("server", ssl=True) server3 = Test.MakeOriginServer("server3", ssl=True) diff --git a/tests/gold_tests/tls/tls_client_cert.test.py b/tests/gold_tests/tls/tls_client_cert.test.py index 7a2c5789b09..67e2fbd70df 100644 --- a/tests/gold_tests/tls/tls_client_cert.test.py +++ b/tests/gold_tests/tls/tls_client_cert.test.py @@ -21,7 +21,7 @@ Test different combinations of TLS handshake hooks to ensure they are applied consistently. ''' -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) # --clientverify: "" empty string because microserver does store_true for argparse, but options is a dictionary @@ -117,6 +117,7 @@ '''.split("\n") ) + # Should succeed tr = Test.AddTestRun("Connect with first client cert to first server") tr.Processes.Default.StartBefore(Test.Processes.ts) @@ -189,6 +190,9 @@ tr2.Processes.Default.Env = ts.Env tr2.Processes.Default.ReturnCode = 0 +# Set TS_RUNROOT, traffic_ctl needs it to find the socket. +ts.SetRunRootEnv() + tr2reload = Test.AddTestRun("Reload config") tr2reload.StillRunningAfter = ts tr2reload.StillRunningAfter = server diff --git a/tests/gold_tests/tls/tls_client_cert_override.test.py b/tests/gold_tests/tls/tls_client_cert_override.test.py index e1fb030d06f..f50550c2fac 100644 --- a/tests/gold_tests/tls/tls_client_cert_override.test.py +++ b/tests/gold_tests/tls/tls_client_cert_override.test.py @@ -21,7 +21,7 @@ Test conf_remp to specify different client certificates to offer to the origin ''' -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) server = Test.MakeOriginServer("server", diff --git a/tests/gold_tests/tls/tls_client_cert_override_plugin.test.py b/tests/gold_tests/tls/tls_client_cert_override_plugin.test.py index 1aa46fa48d1..bbf7e2c3eae 100644 --- a/tests/gold_tests/tls/tls_client_cert_override_plugin.test.py +++ b/tests/gold_tests/tls/tls_client_cert_override_plugin.test.py @@ -21,7 +21,8 @@ ''' -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) +ts.SetRunRootEnv() cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) server = Test.MakeOriginServer("server", diff --git a/tests/gold_tests/tls/tls_client_cert_plugin.test.py b/tests/gold_tests/tls/tls_client_cert_plugin.test.py index e493b97a65f..504add979ea 100644 --- a/tests/gold_tests/tls/tls_client_cert_plugin.test.py +++ b/tests/gold_tests/tls/tls_client_cert_plugin.test.py @@ -24,7 +24,8 @@ Test offering client cert to origin, but using plugin for cert loading ''' -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, dump_runroot=True) +ts.SetRunRootEnv() cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) # --clientverify: "" empty string because microserver does store_true for argparse, but options is a dictionary diff --git a/tests/gold_tests/tls/tls_client_verify.test.py b/tests/gold_tests/tls/tls_client_verify.test.py index baaf04788f6..cda82eeb861 100644 --- a/tests/gold_tests/tls/tls_client_verify.test.py +++ b/tests/gold_tests/tls/tls_client_verify.test.py @@ -22,7 +22,7 @@ Test various options for requiring certificate from client for mutual authentication TLS ''' -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, enable_tls=True) cafile = "{0}/signer.pem".format(Test.RunDirectory) cafile2 = "{0}/signer2.pem".format(Test.RunDirectory) server = Test.MakeOriginServer("server") diff --git a/tests/gold_tests/tls/tls_tunnel.test.py b/tests/gold_tests/tls/tls_tunnel.test.py index bcf7cc94681..940f91344ba 100644 --- a/tests/gold_tests/tls/tls_tunnel.test.py +++ b/tests/gold_tests/tls/tls_tunnel.test.py @@ -21,7 +21,7 @@ ''' # Define default ATS -ts = Test.MakeATSProcess("ts", command="traffic_manager", select_ports=True, enable_tls=True) +ts = Test.MakeATSProcess("ts", command="traffic_server", select_ports=True, enable_tls=True, dump_runroot=True) server_foo = Test.MakeOriginServer("server_foo", ssl=True) server_bar = Test.MakeOriginServer("server_bar", ssl=True) server2 = Test.MakeOriginServer("server2") @@ -184,7 +184,7 @@ trreload.StillRunningAfter = ts trreload.StillRunningAfter = server_foo trreload.StillRunningAfter = server_bar -trreload.Processes.Default.Command = 'traffic_ctl config reload' +trreload.Processes.Default.Command = 'traffic_ctl config reload --run-root {}'.format(ts.Disk.runroot_yaml.Name) # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket trreload.Processes.Default.Env = ts.Env trreload.Processes.Default.ReturnCode = 0 diff --git a/tests/gold_tests/traffic_ctl/remap_inc/remap_inc.test.py b/tests/gold_tests/traffic_ctl/remap_inc/remap_inc.test.py index 852b0f95ec8..83a51d2bf81 100644 --- a/tests/gold_tests/traffic_ctl/remap_inc/remap_inc.test.py +++ b/tests/gold_tests/traffic_ctl/remap_inc/remap_inc.test.py @@ -24,7 +24,7 @@ Test.Setup.Copy("wait_reload.sh") # Define ATS and configure -ts = Test.MakeATSProcess("ts", command="traffic_manager", enable_cache=False) +ts = Test.MakeATSProcess("ts", command="traffic_server", enable_cache=False, dump_runroot=True) ts.Disk.File(ts.Variables.CONFIGDIR + "/test.inc", id="test_cfg", typename="ats:config") ts.Disk.test_cfg.AddLine( @@ -60,7 +60,7 @@ tr = Test.AddTestRun("Reload config") tr.StillRunningAfter = ts -tr.Processes.Default.Command = 'traffic_ctl config reload' +tr.Processes.Default.Command = f'traffic_ctl config reload --run-root {ts.Disk.runroot_yaml.Name}' # Need to copy over the environment so traffic_ctl knows where to find the unix domain socket tr.Processes.Default.Env = ts.Env tr.Processes.Default.ReturnCode = 0 @@ -76,7 +76,3 @@ ) tr.Processes.Default.ReturnCode = 0 tr.StillRunningAfter = ts - -ts.Disk.manager_log.Content += Testers.ExcludesExpression( - "needs restart", - "Ensure that extra msg reported in issue #7530 does not reappear")