diff --git a/build/hiredis.m4 b/build/hiredis.m4 index 1a9a18c0273..b2f09179be0 100644 --- a/build/hiredis.m4 +++ b/build/hiredis.m4 @@ -34,7 +34,7 @@ AC_ARG_WITH(hiredis, [AS_HELP_STRING([--with-hiredis=DIR],[use a specific hiredi case "$hiredis_base_dir" in *":"*) - hidredis_include="`echo $hiredis_base_dir |sed -e 's/:.*$//'`" + hiredis_include="`echo $hiredis_base_dir |sed -e 's/:.*$//'`" hiredis_ldflags="`echo $hiredis_base_dir |sed -e 's/^.*://'`" AC_MSG_CHECKING(for hiredis includes in $hiredis_include libs in $hiredis_ldflags ) ;; diff --git a/build/nuraft.m4 b/build/nuraft.m4 new file mode 100644 index 00000000000..dca9e96a9bb --- /dev/null +++ b/build/nuraft.m4 @@ -0,0 +1,85 @@ +dnl -------------------------------------------------------- -*- autoconf -*- +dnl Licensed to the Apache Software Foundation (ASF) under one or more +dnl contributor license agreements. See the NOTICE file distributed with +dnl this work for additional information regarding copyright ownership. +dnl The ASF licenses this file to You under the Apache License, Version 2.0 +dnl (the "License"); you may not use this file except in compliance with +dnl the License. You may obtain a copy of the License at +dnl +dnl http://www.apache.org/licenses/LICENSE-2.0 +dnl +dnl Unless required by applicable law or agreed to in writing, software +dnl distributed under the License is distributed on an "AS IS" BASIS, +dnl WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +dnl See the License for the specific language governing permissions and +dnl limitations under the License. + +dnl +dnl nuraft.m4: Trafficserver's nuraft autoconf macros +dnl + +dnl +dnl TS_CHECK_NURAFT: look for nuraft libraries and headers +dnl + +AC_DEFUN([TS_CHECK_NURAFT], [ +has_nuraft=no +AC_ARG_WITH(nuraft, [AC_HELP_STRING([--with-nuraft=DIR], [use a specific nuraft library])], +[ + if test "x$withval" != "xyes" && test "x$withval" != "x"; then + nuraft_base_dir="$withval" + if test "$withval" != "no"; then + has_nuraft=yes + case "$withval" in + *":"*) + nuraft_include="`echo $withval | sed -e 's/:.*$//'`" + nuraft_ldflags="`echo $withval | sed -e 's/^.*://'`" + AC_MSG_CHECKING(for nuraft includes in $nuraft_include libs in $nuraft_ldflags) + ;; + *) + nuraft_include="$withval/include" + nuraft_ldflags="$withval/lib" + nuraft_base_dir="$withval" + AC_MSG_CHECKING(for nuraft includes in $nuraft_include libs in $nuraft_ldflags) + ;; + esac + fi + fi + + if test -d $nuraft_include && test -d $nuraft_ldflags && test -f $nuraft_include/libnuraft/nuraft.hxx; then + AC_MSG_RESULT([ok]) + else + AC_MSG_RESULT([not found]) + fi + +if test "$has_nuraft" != "no"; then + saved_ldflags=$LDFLAGS + saved_cppflags=$CPPFLAGS + + NURAFT_LIBS=-lnuraft + if test "$nuraft_base_dir" != "/usr"; then + NURAFT_INCLUDES=-I${nuraft_include} + NURAFT_LDFLAGS=-L${nuraft_ldflags} + + TS_ADDTO(CPPFLAGS, [${NURAFT_INCLUDES}]) + TS_ADDTO(LDFLAGS, [${NURAFT_LDFLAGS}]) + TS_ADDTO_RPATH(${nuraft_ldflags}) + fi + + if test "$nuraft_include" != "0"; then + NURAFT_INCLUDES=-I${nuraft_include} + else + has_nuraft=no + CPPFLAGS=$saved_cppflags + LDFLAGS=$saved_ldflags + fi +fi +], +[ + has_nuraft=no +]) + +AC_SUBST([NURAFT_INCLUDES]) +AC_SUBST([NURAFT_LIBS]) +AC_SUBST([NURAFT_LDFLAGS]) +]) diff --git a/configure.ac b/configure.ac index ddc905945e2..1745358ed5b 100644 --- a/configure.ac +++ b/configure.ac @@ -1469,9 +1469,12 @@ TS_CHECK_BORINGOCSP # Check for optional hiredis library TS_CHECK_HIREDIS - AM_CONDITIONAL([BUILD_SSL_SESSION_REUSE_PLUGIN], [test ! -z "${LIB_HIREDIS}" -a "x${has_hiredis}" = "x1" ]) +# Check for optional nuraft library +TS_CHECK_NURAFT +AM_CONDITIONAL([BUILD_STEK_SHARE_PLUGIN], [test x"$has_nuraft" = x"yes"]) + # Check for backtrace() support has_backtrace=0 AC_CHECK_HEADERS([execinfo.h], [has_backtrace=1],[]) @@ -2330,13 +2333,15 @@ AC_MSG_NOTICE([Build option summary: CXXFLAGS: $CXXFLAGS CPPFLAGS: $CPPFLAGS LDFLAGS: $LDFLAGS - AM@&t@_CFLAGS: $AM_CFLAGS - AM@&t@_CXXFLAGS: $AM_CXXFLAGS - AM@&t@_CPPFLAGS: $AM_CPPFLAGS - AM@&t@_LDFLAGS: $AM_LDFLAGS + AM@&t@_CFLAGS: $AM_CFLAGS + AM@&t@_CXXFLAGS: $AM_CXXFLAGS + AM@&t@_CPPFLAGS: $AM_CPPFLAGS + AM@&t@_LDFLAGS: $AM_LDFLAGS TS_INCLUDES: $TS_INCLUDES OPENSSL_LDFLAGS: $OPENSSL_LDFLAGS OPENSSL_INCLUDES: $OPENSSL_INCLUDES YAMLCPP_LDFLAGS: $YAMLCPP_LDFLAGS YAMLCPP_INCLUDES: $YAMLCPP_INCLUDES + NURAFT_LDFLAGS: $NURAFT_LDFLAGS + NURAFT_INCLUDES: $NURAFT_INCLUDES ]) diff --git a/doc/admin-guide/plugins/index.en.rst b/doc/admin-guide/plugins/index.en.rst index a848475732b..12ff6ad50e5 100644 --- a/doc/admin-guide/plugins/index.en.rst +++ b/doc/admin-guide/plugins/index.en.rst @@ -172,6 +172,7 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi Slice SSL Headers SSL Session Reuse + STEK Share System Statistics Traffic Dump WebP Transform @@ -265,6 +266,9 @@ directory of the |TS| source tree. Experimental plugins can be compiled by passi :doc:`SSL Headers ` Populate request headers with SSL session information. +:doc:`STEK Share ` + Coordinates STEK (Session Ticket Encryption Key) between ATS instances running in a group. + :doc:`System Stats ` Inserts system statistics in to the stats list diff --git a/doc/admin-guide/plugins/stek_share.en.rst b/doc/admin-guide/plugins/stek_share.en.rst new file mode 100644 index 00000000000..43ac28c6786 --- /dev/null +++ b/doc/admin-guide/plugins/stek_share.en.rst @@ -0,0 +1,105 @@ +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + +.. include:: ../../common.defs + +.. _admin-plugins-stek_share: + + +STEK Share Plugin +***************** + +This plugin coordinates STEK (Session Ticket Encryption Key) between ATS instances running in a group. +As the ID based session resumption rate continue to decrease, this new plugin will replace the +:ref:`admin-plugins-ssl_session_reuse` plugin. + + +How It Works +============ + +This plugin implements the `Raft consensus algorithm ` to decide on a leader. The leader will +periodically create a new STEK key and share it with all other ATS boxes in the group. When the plugin starts up, it +will automatically join the cluster of all other ATS boxes in the group, which will also automatically elect a leader. +The plugin uses the `TSSslTicketKeyUpdate` call to update ATS with the latest two STEK's it has received. + +All communication are encrypted. All the ATS boxes participating in the STEK sharing must have access to the cert/key pair. + +Note that since the this plugin only updates STEK every few hours, all Raft related stuff are kept in memory, and some code is +borrowed from the examples from `NuRaft library ` that is used in this plugin. + + +Building +======== + +This plugin uses `NuRaft library ` for leader election and communication. +The NuRaft library must be installed for this plugin to build. It can be specified by the `--with-nuraft` argument to configure. + +This plugin also uses `YAML-CPP library ` for reading the configuration file. +The YAML-CPP library must be installed for this plugin to build. It can be specified by the `--with-yaml-cpp` argument to configure. + +As part of the experimental plugs, the `--enable-experimental-plugins` option must also be given to configure to build this plugin. + + +Config File +=========== + +STEK Share is a global plugin. Its configuration file uses YAML, and is given as an argument to the plugin in :file:`plugin.config`. + +:: + stek_share.so etc/trafficserver/example_server_conf.yaml + +Available options: + +* server_id - An unique ID for the server. +* address - Hostname or IP address of the server. +* port - Port number for communication. +* asio_thread_pool_size - [Optional] Thread pool size for `ASIO library `. Default size is 4. +* heart_beat_interval - [Optional] Heart beat interval of Raft leader, must be less than "election_timeout_lower_bound". Default value is 100 ms. +* election_timeout_lower_bound - [Optional] Lower bound of Raft leader election timeout. Default value is 200 ms. +* election_timeout_upper_bound - [Optional] Upper bound of Raft leader election timeout. Default value is 400 ms. +* reserved_log_items - [Optional] The maximum number of logs preserved ahead the last snapshot. Default value is 5. +* snapshot_distance - [Optional] The number of log appends for each snapshot. Default value is 5. +* client_req_timeout - [Optional] Client request timeout. Default value is 3000 ms. +* key_update_interval - The interval between STEK update. +* server_list_file - Path to a file containing information of all the servers that's supposed to be in the Raft cluster. +* root_cert_file - Path to the root ca file. +* server_cert_file - Path to the cert file. +* server_key_file - Path to the key file. +* cert_verify_str - SSL verification string, for example "/C=US/ST=IL/O=Yahoo/OU=Edge/CN=localhost" + + +Example Config File +=================== + +.. literalinclude:: ../../../plugins/experimental/stek_share/example_server_conf.yaml + + +Server List File +================ + +Server list file as mentioned above, also in YAML. + +* server_id - ID of the server. +* address - Hostname or IP address of the server. +* port - Port number of the server. + + +Example Server List File +======================== + +.. literalinclude:: ../../../plugins/experimental/stek_share/example_server_list.yaml diff --git a/plugins/Makefile.am b/plugins/Makefile.am index 75876ae3eba..fff6fdbd667 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -107,6 +107,10 @@ if BUILD_SSL_SESSION_REUSE_PLUGIN include experimental/ssl_session_reuse/Makefile.inc endif +if BUILD_STEK_SHARE_PLUGIN +include experimental/stek_share/Makefile.inc +endif + if HAS_KYOTOCABINET include experimental/cache_key_genid/Makefile.inc endif diff --git a/plugins/experimental/stek_share/Makefile.inc b/plugins/experimental/stek_share/Makefile.inc new file mode 100644 index 00000000000..462916d3374 --- /dev/null +++ b/plugins/experimental/stek_share/Makefile.inc @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Only build if NURAFT_LIBS is set to a non-empty value +pkglib_LTLIBRARIES += experimental/stek_share/stek_share.la + +experimental_stek_share_stek_share_la_SOURCES = \ + experimental/stek_share/common.cc \ + experimental/stek_share/common.h \ + experimental/stek_share/log_store.cc \ + experimental/stek_share/log_store.h \ + experimental/stek_share/state_machine.h \ + experimental/stek_share/state_manager.h \ + experimental/stek_share/stek_share.cc \ + experimental/stek_share/stek_share.h \ + experimental/stek_share/stek_utils.cc \ + experimental/stek_share/stek_utils.h + +experimental_stek_share_stek_share_la_LDFLAGS = $(AM_LDFLAGS) @YAMLCPP_LDFLAGS@ + +AM_CPPFLAGS += @NURAFT_INCLUDES@ @YAMLCPP_INCLUDES@ + +experimental_stek_share_stek_share_la_LIBADD = @NURAFT_LIBS@ @YAMLCPP_LIBS@ diff --git a/plugins/experimental/stek_share/common.cc b/plugins/experimental/stek_share/common.cc new file mode 100644 index 00000000000..b9af195e701 --- /dev/null +++ b/plugins/experimental/stek_share/common.cc @@ -0,0 +1,46 @@ +/** @file + + common.cc - Some common functions everyone needs + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include +#include + +#include +#include + +#include "common.h" + +const unsigned char hex_chars[] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + +std::string +hex_str(std::string const &str) +{ + std::string hex_str; + hex_str.reserve(str.size() * 2); + for (unsigned long int i = 0; i < str.size(); ++i) { + unsigned char c = str.at(i); + hex_str[i * 2] = hex_chars[(c & 0xF0) >> 4]; + hex_str[i * 2 + 1] = hex_chars[(c & 0x0F)]; + } + return hex_str; +} diff --git a/plugins/experimental/stek_share/common.h b/plugins/experimental/stek_share/common.h new file mode 100644 index 00000000000..312ad557d0c --- /dev/null +++ b/plugins/experimental/stek_share/common.h @@ -0,0 +1,72 @@ +/** @file + + common.h - Things that need to be everywhere + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#define PLUGIN "stek_share" + +class PluginThreads +{ +public: + void + store(const pthread_t &th) + { + std::lock_guard lock(threads_mutex); + threads_queue.push_back(th); + } + + void + terminate() + { + shut_down = true; + + std::lock_guard lock(threads_mutex); + while (!threads_queue.empty()) { + pthread_t th = threads_queue.front(); + ::pthread_join(th, nullptr); + threads_queue.pop_front(); + } + } + + bool + is_shut_down() + { + return shut_down; + } + +private: + bool shut_down = false; + std::deque threads_queue; + std::mutex threads_mutex; +}; + +std::string hex_str(std::string const &str); diff --git a/plugins/experimental/stek_share/example_server_conf.yaml b/plugins/experimental/stek_share/example_server_conf.yaml new file mode 100644 index 00000000000..6b6e65fa735 --- /dev/null +++ b/plugins/experimental/stek_share/example_server_conf.yaml @@ -0,0 +1,16 @@ +server_id: 1 +address: 127.0.0.1 +port: 10001 +asio_thread_pool_size: 4 +heart_beat_interval: 100 +election_timeout_lower_bound: 200 +election_timeout_upper_bound: 400 +reserved_log_items: 5 +snapshot_distance: 5 +client_req_timeout: 3000 # this is in milliseconds +key_update_interval: 3600 # this is in seconds +server_list_file: /abs/path/to/server_list.yaml +root_cert_file: /abs/path/to/ca.pem +server_cert_file: /abs/path/to/server.pem +server_key_file: /abs/path/to/server.key +cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=localhost diff --git a/plugins/experimental/stek_share/example_server_list.yaml b/plugins/experimental/stek_share/example_server_list.yaml new file mode 100644 index 00000000000..77d815f0aed --- /dev/null +++ b/plugins/experimental/stek_share/example_server_list.yaml @@ -0,0 +1,15 @@ +- server_id: 1 + address: 127.0.0.1 + port: 10001 +- server_id: 2 + address: 127.0.0.1 + port: 10002 +- server_id: 3 + address: 127.0.0.1 + port: 10003 +- server_id: 4 + address: 127.0.0.1 + port: 10004 +- server_id: 5 + address: 127.0.0.1 + port: 10005 diff --git a/plugins/experimental/stek_share/log_store.cc b/plugins/experimental/stek_share/log_store.cc new file mode 100644 index 00000000000..0bd31673a1f --- /dev/null +++ b/plugins/experimental/stek_share/log_store.cc @@ -0,0 +1,263 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples + +#include + +#include + +#include "log_store.h" + +STEKShareLogStore::STEKShareLogStore() : start_idx_(1) +{ + // Dummy entry for index 0. + nuraft::ptr buf = nuraft::buffer::alloc(sizeof(uint64_t)); + logs_[0] = nuraft::cs_new(0, buf); +} + +STEKShareLogStore::~STEKShareLogStore() {} + +nuraft::ptr +STEKShareLogStore::make_clone(const nuraft::ptr &entry) +{ + nuraft::ptr clone = + nuraft::cs_new(entry->get_term(), nuraft::buffer::clone(entry->get_buf()), entry->get_val_type()); + return clone; +} + +uint64_t +STEKShareLogStore::next_slot() const +{ + std::lock_guard l(logs_lock_); + + // Exclude the dummy entry. + return start_idx_ + logs_.size() - 1; +} + +uint64_t +STEKShareLogStore::start_index() const +{ + return start_idx_; +} + +nuraft::ptr +STEKShareLogStore::last_entry() const +{ + uint64_t next_idx = next_slot(); + std::lock_guard l(logs_lock_); + auto entry = logs_.find(next_idx - 1); + if (entry == logs_.end()) { + entry = logs_.find(0); + } + + return make_clone(entry->second); +} + +uint64_t +STEKShareLogStore::append(nuraft::ptr &entry) +{ + nuraft::ptr clone = make_clone(entry); + + std::lock_guard l(logs_lock_); + size_t idx = start_idx_ + logs_.size() - 1; + logs_[idx] = clone; + return idx; +} + +void +STEKShareLogStore::write_at(uint64_t index, nuraft::ptr &entry) +{ + nuraft::ptr clone = make_clone(entry); + + // Discard all logs equal to or greater than "index". + std::lock_guard l(logs_lock_); + auto itr = logs_.lower_bound(index); + while (itr != logs_.end()) { + itr = logs_.erase(itr); + } + logs_[index] = clone; +} + +nuraft::ptr>> +STEKShareLogStore::log_entries(uint64_t start, uint64_t end) +{ + nuraft::ptr>> ret = nuraft::cs_new>>(); + + ret->resize(end - start); + uint64_t cc = 0; + for (uint64_t i = start; i < end; ++i) { + nuraft::ptr src = nullptr; + { + std::lock_guard l(logs_lock_); + auto entry = logs_.find(i); + if (entry == logs_.end()) { + entry = logs_.find(0); + assert(0); + } + src = entry->second; + } + (*ret)[cc++] = make_clone(src); + } + return ret; +} + +nuraft::ptr>> +STEKShareLogStore::log_entries_ext(uint64_t start, uint64_t end, int64_t batch_size_hint_in_bytes) +{ + nuraft::ptr>> ret = nuraft::cs_new>>(); + + if (batch_size_hint_in_bytes < 0) { + return ret; + } + + size_t accum_size = 0; + for (uint64_t i = start; i < end; ++i) { + nuraft::ptr src = nullptr; + { + std::lock_guard l(logs_lock_); + auto entry = logs_.find(i); + if (entry == logs_.end()) { + entry = logs_.find(0); + assert(0); + } + src = entry->second; + } + ret->push_back(make_clone(src)); + accum_size += src->get_buf().size(); + if (batch_size_hint_in_bytes && accum_size >= (uint64_t)batch_size_hint_in_bytes) { + break; + } + } + return ret; +} + +nuraft::ptr +STEKShareLogStore::entry_at(uint64_t index) +{ + nuraft::ptr src = nullptr; + { + std::lock_guard l(logs_lock_); + auto entry = logs_.find(index); + if (entry == logs_.end()) { + entry = logs_.find(0); + } + src = entry->second; + } + return make_clone(src); +} + +uint64_t +STEKShareLogStore::term_at(uint64_t index) +{ + uint64_t term = 0; + { + std::lock_guard l(logs_lock_); + auto entry = logs_.find(index); + if (entry == logs_.end()) { + entry = logs_.find(0); + } + term = entry->second->get_term(); + } + return term; +} + +nuraft::ptr +STEKShareLogStore::pack(uint64_t index, int32_t cnt) +{ + std::vector> logs; + + size_t size_total = 0; + for (uint64_t i = index; i < index + cnt; ++i) { + nuraft::ptr le = nullptr; + { + std::lock_guard l(logs_lock_); + le = logs_[i]; + } + assert(le.get()); + nuraft::ptr buf = le->serialize(); + size_total += buf->size(); + logs.push_back(buf); + } + + nuraft::ptr buf_out = nuraft::buffer::alloc(sizeof(int32_t) + cnt * sizeof(int32_t) + size_total); + buf_out->pos(0); + buf_out->put((int32_t)cnt); + + for (auto &entry : logs) { + nuraft::ptr &bb = entry; + buf_out->put((int32_t)bb->size()); + buf_out->put(*bb); + } + return buf_out; +} + +void +STEKShareLogStore::apply_pack(uint64_t index, nuraft::buffer &pack) +{ + pack.pos(0); + int32_t num_logs = pack.get_int(); + + for (int32_t i = 0; i < num_logs; ++i) { + uint64_t cur_idx = index + i; + int32_t buf_size = pack.get_int(); + + nuraft::ptr buf_local = nuraft::buffer::alloc(buf_size); + pack.get(buf_local); + + nuraft::ptr le = nuraft::log_entry::deserialize(*buf_local); + { + std::lock_guard l(logs_lock_); + logs_[cur_idx] = le; + } + } + + { + std::lock_guard l(logs_lock_); + auto entry = logs_.upper_bound(0); + if (entry != logs_.end()) { + start_idx_ = entry->first; + } else { + start_idx_ = 1; + } + } +} + +bool +STEKShareLogStore::compact(uint64_t last_log_index) +{ + std::lock_guard l(logs_lock_); + for (uint64_t i = start_idx_; i <= last_log_index; ++i) { + auto entry = logs_.find(i); + if (entry != logs_.end()) { + logs_.erase(entry); + } + } + + // WARNING: + // Even though nothing has been erased, we should set "start_idx_" to new index. + if (start_idx_ <= last_log_index) { + start_idx_ = last_log_index + 1; + } + + return true; +} + +void +STEKShareLogStore::close() +{ +} diff --git a/plugins/experimental/stek_share/log_store.h b/plugins/experimental/stek_share/log_store.h new file mode 100644 index 00000000000..6f6659cdee4 --- /dev/null +++ b/plugins/experimental/stek_share/log_store.h @@ -0,0 +1,76 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples + +#pragma once + +#include +#include +#include + +#include + +class STEKShareLogStore : public nuraft::log_store +{ +public: + STEKShareLogStore(); + + ~STEKShareLogStore(); + + __nocopy__(STEKShareLogStore); + + uint64_t next_slot() const; + + uint64_t start_index() const; + + nuraft::ptr last_entry() const; + + uint64_t append(nuraft::ptr &entry); + + void write_at(uint64_t index, nuraft::ptr &entry); + + nuraft::ptr>> log_entries(uint64_t start, uint64_t end); + + nuraft::ptr>> log_entries_ext(uint64_t start, uint64_t end, + int64_t batch_size_hint_in_bytes = 0); + + nuraft::ptr entry_at(uint64_t index); + + uint64_t term_at(uint64_t index); + + nuraft::ptr pack(uint64_t index, int32_t cnt); + + void apply_pack(uint64_t index, nuraft::buffer &pack); + + bool compact(uint64_t last_log_index); + + bool + flush() + { + return true; + } + + void close(); + +private: + static nuraft::ptr make_clone(const nuraft::ptr &entry); + + std::map> logs_; + mutable std::mutex logs_lock_; + std::atomic start_idx_; +}; diff --git a/plugins/experimental/stek_share/state_machine.h b/plugins/experimental/stek_share/state_machine.h new file mode 100644 index 00000000000..26a3995e851 --- /dev/null +++ b/plugins/experimental/stek_share/state_machine.h @@ -0,0 +1,237 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples + +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "stek_utils.h" + +class STEKShareSM : public nuraft::state_machine +{ +public: + STEKShareSM() : last_committed_idx_(0) {} + + ~STEKShareSM() {} + + nuraft::ptr + pre_commit(const uint64_t log_idx, nuraft::buffer &data) + { + return nullptr; + } + + nuraft::ptr + commit(const uint64_t log_idx, nuraft::buffer &data) + { + // Extract bytes from "data". + size_t len = 0; + nuraft::buffer_serializer bs_data(data); + void *byte_array = bs_data.get_bytes(len); + // TSDebug(PLUGIN, "commit %lu: %s", log_idx, hex_str(std::string(reinterpret_cast(byte_array), len)).c_str()); + + assert(len == SSL_TICKET_KEY_SIZE); + + { + std::lock_guard l(stek_lock_); + std::memcpy(&stek_, byte_array, len); + received_stek_ = true; + } + + // Update last committed index number. + last_committed_idx_ = log_idx; + + nuraft::ptr ret = nuraft::buffer::alloc(sizeof(log_idx)); + nuraft::buffer_serializer bs_ret(ret); + bs_ret.put_u64(log_idx); + + return ret; + } + + bool + received_stek(ssl_ticket_key_t *curr_stek) + { + std::lock_guard l(stek_lock_); + if (!received_stek_) { + return false; + } + + received_stek_ = false; + + if (std::memcmp(curr_stek, &stek_, SSL_TICKET_KEY_SIZE != 0)) { + std::memcpy(curr_stek, &stek_, SSL_TICKET_KEY_SIZE); + return true; + } + + return false; + } + + void + commit_config(const uint64_t log_idx, nuraft::ptr &new_conf) + { + // Nothing to do with configuration change. Just update committed index. + last_committed_idx_ = log_idx; + } + + void + rollback(const uint64_t log_idx, nuraft::buffer &data) + { + // Nothing to do here since we don't have pre-commit. + } + + int + read_logical_snp_obj(nuraft::snapshot &s, void *&user_snp_ctx, uint64_t obj_id, nuraft::ptr &data_out, + bool &is_last_obj) + { + // TSDebug(PLUGIN, "read snapshot %lu term %lu object ID %lu", s.get_last_log_idx(), s.get_last_log_term(), obj_id); + + is_last_obj = true; + + { + std::lock_guard l(snapshot_lock_); + if (snapshot_ == nullptr || snapshot_->snapshot_->get_last_log_idx() != s.get_last_log_idx()) { + data_out = nullptr; + return -1; + } else { + data_out = nuraft::buffer::alloc(sizeof(int) + SSL_TICKET_KEY_SIZE); + nuraft::buffer_serializer bs(data_out); + bs.put_bytes(reinterpret_cast(&snapshot_->stek_), SSL_TICKET_KEY_SIZE); + return 0; + } + } + } + + void + save_logical_snp_obj(nuraft::snapshot &s, uint64_t &obj_id, nuraft::buffer &data, bool is_first_obj, bool is_last_obj) + { + // TSDebug(PLUGIN, "save snapshot %lu term %lu object ID %lu", s.get_last_log_idx(), s.get_last_log_term(), obj_id); + + size_t len = 0; + nuraft::buffer_serializer bs_data(data); + void *byte_array = bs_data.get_bytes(len); + + assert(len == SSL_TICKET_KEY_SIZE); + + ssl_ticket_key_t local_stek; + std::memcpy(&local_stek, byte_array, len); + + nuraft::ptr snp_buf = s.serialize(); + nuraft::ptr ss = nuraft::snapshot::deserialize(*snp_buf); + nuraft::ptr ctx = nuraft::cs_new(ss, local_stek); + + { + std::lock_guard l(snapshot_lock_); + snapshot_ = ctx; + } + + obj_id++; + } + + bool + apply_snapshot(nuraft::snapshot &s) + { + // TSDebug(PLUGIN, "apply snapshot %lu term %lu", s.get_last_log_idx(), s.get_last_log_term()); + + { + std::lock_guard l(snapshot_lock_); + if (snapshot_ != nullptr) { + std::lock_guard ll(stek_lock_); + std::memcpy(&stek_, &snapshot_->stek_, SSL_TICKET_KEY_SIZE); + received_stek_ = true; + return true; + } else { + return false; + } + } + } + + void + free_user_snp_ctx(void *&user_snp_ctx) + { + } + + nuraft::ptr + last_snapshot() + { + // Just return the latest snapshot. + std::lock_guard l(snapshot_lock_); + if (snapshot_ != nullptr) { + return snapshot_->snapshot_; + } + return nullptr; + } + + uint64_t + last_commit_index() + { + return last_committed_idx_; + } + + void + create_snapshot(nuraft::snapshot &s, nuraft::async_result::handler_type &when_done) + { + // TSDebug(PLUGIN, "create snapshot %lu term %lu", s.get_last_log_idx(), s.get_last_log_term()); + + ssl_ticket_key_t local_stek; + { + std::lock_guard l(stek_lock_); + std::memcpy(&local_stek, &stek_, SSL_TICKET_KEY_SIZE); + } + + nuraft::ptr snp_buf = s.serialize(); + nuraft::ptr ss = nuraft::snapshot::deserialize(*snp_buf); + nuraft::ptr ctx = nuraft::cs_new(ss, local_stek); + + { + std::lock_guard l(snapshot_lock_); + snapshot_ = ctx; + } + + nuraft::ptr except(nullptr); + bool ret = true; + when_done(ret, except); + } + +private: + struct snapshot_ctx { + snapshot_ctx(nuraft::ptr &s, ssl_ticket_key_t key) : snapshot_(s), stek_(key) {} + nuraft::ptr snapshot_; + ssl_ticket_key_t stek_; + }; + + // Last committed Raft log number. + std::atomic last_committed_idx_; + + nuraft::ptr snapshot_; + + // Mutex for snapshot. + std::mutex snapshot_lock_; + + bool received_stek_ = false; + + ssl_ticket_key_t stek_; + + std::mutex stek_lock_; +}; diff --git a/plugins/experimental/stek_share/state_manager.h b/plugins/experimental/stek_share/state_manager.h new file mode 100644 index 00000000000..63e16249edb --- /dev/null +++ b/plugins/experimental/stek_share/state_manager.h @@ -0,0 +1,102 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples + +#pragma once + +#include + +#include "log_store.h" + +class STEKShareSMGR : public nuraft::state_mgr +{ +public: + STEKShareSMGR(int srv_id, const std::string &endpoint, std::map server_list) + : my_id_(srv_id), my_endpoint_(endpoint), cur_log_store_(nuraft::cs_new()) + { + my_srv_config_ = nuraft::cs_new(srv_id, endpoint); + + // Initial cluster config, read from the list loaded from the configuration file. + saved_config_ = nuraft::cs_new(); + for (auto const &s : server_list) { + int server_id = s.first; + std::string endpoint = s.second; + nuraft::ptr new_server = nuraft::cs_new(server_id, endpoint); + saved_config_->get_servers().push_back(new_server); + } + } + + ~STEKShareSMGR() {} + + nuraft::ptr + load_config() + { + return saved_config_; + } + + void + save_config(const nuraft::cluster_config &config) + { + nuraft::ptr buf = config.serialize(); + saved_config_ = nuraft::cluster_config::deserialize(*buf); + } + + void + save_state(const nuraft::srv_state &state) + { + nuraft::ptr buf = state.serialize(); + saved_state_ = nuraft::srv_state::deserialize(*buf); + } + + nuraft::ptr + read_state() + { + return saved_state_; + } + + nuraft::ptr + load_log_store() + { + return cur_log_store_; + } + + int32_t + server_id() + { + return my_id_; + } + + void + system_exit(const int exit_code) + { + } + + nuraft::ptr + get_srv_config() const + { + return my_srv_config_; + } + +private: + int my_id_; + std::string my_endpoint_; + nuraft::ptr cur_log_store_; + nuraft::ptr my_srv_config_; + nuraft::ptr saved_config_; + nuraft::ptr saved_state_; +}; diff --git a/plugins/experimental/stek_share/stek_share.cc b/plugins/experimental/stek_share/stek_share.cc new file mode 100644 index 00000000000..810c4f75ba9 --- /dev/null +++ b/plugins/experimental/stek_share/stek_share.cc @@ -0,0 +1,445 @@ +/** @file + + stek_share.cc + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "state_machine.h" +#include "state_manager.h" +#include "stek_share.h" +#include "stek_utils.h" +#include "common.h" + +using raft_result = nuraft::cmd_result>; + +PluginThreads plugin_threads; + +static STEKShareServer stek_share_server; +static const nuraft::raft_params::return_method_type CALL_TYPE = nuraft::raft_params::blocking; +// static const nuraft::raft_params::return_method_type CALL_TYPE = nuraft::raft_params::async_handler; + +static int +shutdown_handler(TSCont contp, TSEvent event, void *edata) +{ + if (event == TS_EVENT_LIFECYCLE_SHUTDOWN) { + plugin_threads.terminate(); + stek_share_server.launcher_.shutdown(); + } + return 0; +} + +bool +cert_verification(const std::string &sn) +{ + if (sn.compare(stek_share_server.cert_verify_str_) != 0) { + TSDebug(PLUGIN, "Cert incorrect, expecting: %s, got: %s", stek_share_server.cert_verify_str_.c_str(), sn.c_str()); + return false; + } + return true; +} + +int +init_raft(nuraft::ptr sm_instance) +{ + // State machine. + stek_share_server.smgr_ = + nuraft::cs_new(stek_share_server.server_id_, stek_share_server.endpoint_, stek_share_server.server_list_); + + // State manager. + stek_share_server.sm_ = sm_instance; + + // ASIO options. + nuraft::asio_service::options asio_opts; + asio_opts.thread_pool_size_ = stek_share_server.asio_thread_pool_size_; + asio_opts.enable_ssl_ = true; + asio_opts.verify_sn_ = cert_verification; + asio_opts.root_cert_file_ = stek_share_server.root_cert_file_; + asio_opts.server_cert_file_ = stek_share_server.server_cert_file_; + asio_opts.server_key_file_ = stek_share_server.server_key_file_; + + // Raft parameters. + nuraft::raft_params params; + params.heart_beat_interval_ = stek_share_server.heart_beat_interval_; + params.election_timeout_lower_bound_ = stek_share_server.election_timeout_lower_bound_; + params.election_timeout_upper_bound_ = stek_share_server.election_timeout_upper_bound_; + params.reserved_log_items_ = stek_share_server.reserved_log_items_; + params.snapshot_distance_ = stek_share_server.snapshot_distance_; + params.client_req_timeout_ = stek_share_server.client_req_timeout_; + + // According to this method, "append_log" function should be handled differently. + params.return_method_ = CALL_TYPE; + + // Initialize Raft server. + stek_share_server.raft_instance_ = stek_share_server.launcher_.init(stek_share_server.sm_, stek_share_server.smgr_, nullptr, + stek_share_server.port_, asio_opts, params); + + if (!stek_share_server.raft_instance_) { + TSDebug(PLUGIN, "Failed to initialize launcher."); + return -1; + } + + TSDebug(PLUGIN, "Raft instance initialization done."); + return 0; +} + +int +set_server_info(int argc, const char *argv[]) +{ + // Get server ID. + YAML::Node server_conf; + try { + server_conf = YAML::LoadFile(argv[1]); + } catch (YAML::BadFile &e) { + TSEmergency("[%s] Cannot load configuration file: %s.", PLUGIN, e.what()); + } catch (std::exception &e) { + TSEmergency("[%s] Unknown error while loading configuration file: %s.", PLUGIN, e.what()); + } + + if (server_conf["server_id"]) { + stek_share_server.server_id_ = server_conf["server_id"].as(); + if (stek_share_server.server_id_ < 1) { + TSDebug(PLUGIN, "Wrong server id (must be >= 1): %d", stek_share_server.server_id_); + return -1; + } + } else { + TSDebug(PLUGIN, "Must specify server id in the configuration file."); + return -1; + } + + // Get server address and port. + if (server_conf["address"]) { + stek_share_server.addr_ = server_conf["address"].as(); + } else { + TSDebug(PLUGIN, "Must specify server address in the configuration file."); + return -1; + } + + if (server_conf["port"]) { + stek_share_server.port_ = server_conf["port"].as(); + } else { + TSDebug(PLUGIN, "Must specify server port in the configuration file."); + return -1; + } + + stek_share_server.endpoint_ = stek_share_server.addr_ + ":" + std::to_string(stek_share_server.port_); + + if (server_conf["asio_thread_pool_size"]) { + stek_share_server.asio_thread_pool_size_ = server_conf["asio_thread_pool_size"].as(); + } + + if (server_conf["heart_beat_interval"]) { + stek_share_server.heart_beat_interval_ = server_conf["heart_beat_interval"].as(); + } + + if (server_conf["election_timeout_lower_bound"]) { + stek_share_server.election_timeout_lower_bound_ = server_conf["election_timeout_lower_bound"].as(); + } + + if (server_conf["election_timeout_upper_bound"]) { + stek_share_server.election_timeout_upper_bound_ = server_conf["election_timeout_upper_bound"].as(); + } + + if (server_conf["reserved_log_items"]) { + stek_share_server.reserved_log_items_ = server_conf["reserved_log_items"].as(); + } + + if (server_conf["snapshot_distance"]) { + stek_share_server.snapshot_distance_ = server_conf["snapshot_distance"].as(); + } + + if (server_conf["client_req_timeout"]) { + stek_share_server.client_req_timeout_ = server_conf["client_req_timeout"].as(); + } + + if (server_conf["key_update_interval"]) { + stek_share_server.key_update_interval_ = server_conf["key_update_interval"].as(); + } else { + TSDebug(PLUGIN, "Must specify server key update interval in the configuration file."); + return -1; + } + + if (server_conf["server_list_file"]) { + YAML::Node server_list; + try { + server_list = YAML::LoadFile(server_conf["server_list_file"].as()); + } catch (YAML::BadFile &e) { + TSEmergency("[%s] Cannot load server list file: %s.", PLUGIN, e.what()); + } catch (std::exception &e) { + TSEmergency("[%s] Unknown error while loading server list file: %s.", PLUGIN, e.what()); + } + + std::string cluster_list_str = ""; + cluster_list_str += "\nSTEK Share Cluster Server List:"; + for (auto it = server_list.begin(); it != server_list.end(); ++it) { + YAML::Node server_info = it->as(); + if (server_info["server_id"] && server_info["address"] && server_info["port"]) { + int server_id = server_info["server_id"].as(); + std::string address = server_info["address"].as(); + int port = server_info["port"].as(); + std::string endpoint = address + ":" + std::to_string(port); + stek_share_server.server_list_[server_id] = endpoint; + cluster_list_str += "\n " + std::to_string(server_id) + ", " + endpoint; + } else { + TSDebug(PLUGIN, "Wrong server list format."); + return -1; + } + } + TSDebug(PLUGIN, "%s", cluster_list_str.c_str()); + } else { + TSDebug(PLUGIN, "Must specify server list file in the configuration file."); + return -1; + } + + // TODO: check cert and key files exist + if (server_conf["root_cert_file"]) { + stek_share_server.root_cert_file_ = server_conf["root_cert_file"].as(); + } else { + TSDebug(PLUGIN, "Must specify root ca file in the configuration file."); + return -1; + } + + if (server_conf["server_cert_file"]) { + stek_share_server.server_cert_file_ = server_conf["server_cert_file"].as(); + } else { + TSDebug(PLUGIN, "Must specify server cert file in the configuration file."); + return -1; + } + + if (server_conf["server_key_file"]) { + stek_share_server.server_key_file_ = server_conf["server_key_file"].as(); + } else { + TSDebug(PLUGIN, "Must specify server key file in the configuration file."); + return -1; + } + + if (server_conf["cert_verify_str"]) { + stek_share_server.cert_verify_str_ = server_conf["cert_verify_str"].as(); + } else { + TSDebug(PLUGIN, "Must specify cert verify string in the configuration file."); + return -1; + } + + return 0; +} + +void +handle_result(raft_result &result, nuraft::ptr &err) +{ + if (result.get_result_code() != nuraft::cmd_result_code::OK) { + // Something went wrong. + // This means committing this log failed, but the log itself is still in the log store. + TSDebug(PLUGIN, "Replication failed: %d", result.get_result_code()); + return; + } + TSDebug(PLUGIN, "Replication succeeded."); +} + +void +append_log(const void *data, int data_len) +{ + // Create a new log which will contain 4-byte length and string data. + nuraft::ptr new_log = nuraft::buffer::alloc(sizeof(int) + data_len); + nuraft::buffer_serializer bs(new_log); + bs.put_bytes(data, data_len); + + // Do append. + nuraft::ptr ret = stek_share_server.raft_instance_->append_entries({new_log}); + + if (!ret->get_accepted()) { + // Log append rejected, usually because this node is not a leader. + TSDebug(PLUGIN, "Replication failed: %d", ret->get_result_code()); + return; + } + + // Log append accepted, but that doesn't mean the log is committed. + // Commit result can be obtained below. + if (CALL_TYPE == nuraft::raft_params::blocking) { + // Blocking mode: + // "append_entries" returns after getting a consensus, so that "ret" already has the result from state machine. + nuraft::ptr err(nullptr); + handle_result(*ret, err); + } else if (CALL_TYPE == nuraft::raft_params::async_handler) { + // Async mode: + // "append_entries" returns immediately. "handle_result" will be invoked asynchronously, after getting a consensus. + ret->when_ready(std::bind(handle_result, std::placeholders::_1, std::placeholders::_2)); + } else { + assert(0); + } +} + +void +print_status() +{ + // For debugging + nuraft::ptr ls = stek_share_server.smgr_->load_log_store(); + std::string status_str = ""; + status_str += "\n Server ID: " + std::to_string(stek_share_server.server_id_); + status_str += "\n Leader ID: " + std::to_string(stek_share_server.raft_instance_->get_leader()); + status_str += "\n Raft log range: " + std::to_string(ls->start_index()) + " - " + std::to_string((ls->next_slot() - 1)); + status_str += "\n Last committed index: " + std::to_string(stek_share_server.raft_instance_->get_committed_log_idx()); + TSDebug(PLUGIN, "%s", status_str.c_str()); +} + +static void * +stek_updater(void *arg) +{ + plugin_threads.store(::pthread_self()); + ::pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, nullptr); + ::pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + + ssl_ticket_key_t curr_stek; + time_t init_key_time = 0; + + // Initial key to use before syncing up. + TSDebug(PLUGIN, "Generating initial STEK..."); + if (generate_new_stek(&curr_stek, 0 /* fast start */) == 0) { + TSDebug(PLUGIN, "Generate initial STEK succeeded: %s", + hex_str(std::string(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str()); + + std::memcpy(&stek_share_server.ticket_keys_[0], &curr_stek, SSL_TICKET_KEY_SIZE); + + TSDebug(PLUGIN, "Updating SSL Ticket Key..."); + if (TSSslTicketKeyUpdate(reinterpret_cast(stek_share_server.ticket_keys_), SSL_TICKET_KEY_SIZE) == TS_ERROR) { + TSDebug(PLUGIN, "Update SSL Ticket Key failed."); + } else { + TSDebug(PLUGIN, "Update SSL Ticket Key succeeded."); + init_key_time = time(nullptr); + } + } else { + TSFatal("Generate initial STEK failed."); + } + + // Since we're using a pre-configured cluster, we need to have >= 3 nodes in the clust + // to initialize. Busy check before that. + while (!plugin_threads.is_shut_down()) { + if (!stek_share_server.raft_instance_->is_initialized()) { + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + continue; + } + + if (stek_share_server.raft_instance_->is_leader()) { + // We only need to generate new STEK if this server is the leader. + // Otherwise we wake up every 10 seconds to see whether a new STEK has been received. + if (init_key_time != 0 && time(nullptr) - init_key_time < stek_share_server.key_update_interval_) { + // If we got here after starting up, that means the initial key is still valid and we can send it to everyone else. + stek_share_server.last_updated_ = init_key_time; + TSDebug(PLUGIN, "Using initial STEK: %s", + hex_str(std::string(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str()); + append_log(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE); + + } else if (time(nullptr) - stek_share_server.last_updated_ >= stek_share_server.key_update_interval_) { + // Generate a new key as the last one has expired. + // Move the old key from ticket_keys_[0] to ticket_keys_[1], then put the new key in ticket_keys_[0]. + TSDebug(PLUGIN, "Generating new STEK..."); + if (generate_new_stek(&curr_stek, 1) == 0) { + TSDebug(PLUGIN, "Generate new STEK succeeded: %s", + hex_str(std::string(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str()); + + std::memcpy(&stek_share_server.ticket_keys_[1], &stek_share_server.ticket_keys_[0], SSL_TICKET_KEY_SIZE); + std::memcpy(&stek_share_server.ticket_keys_[0], &curr_stek, SSL_TICKET_KEY_SIZE); + + TSDebug(PLUGIN, "Updating SSL Ticket Key..."); + if (TSSslTicketKeyUpdate(reinterpret_cast(stek_share_server.ticket_keys_), SSL_TICKET_KEY_SIZE * 2) == TS_ERROR) { + TSDebug(PLUGIN, "Update SSL Ticket Key failed."); + } else { + stek_share_server.last_updated_ = time(nullptr); + TSDebug(PLUGIN, "Update SSL Ticket Key succeeded."); + TSDebug(PLUGIN, "Using new STEK: %s", + hex_str(std::string(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str()); + append_log(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE); + } + } else { + TSFatal("Generate new STEK failed."); + } + } + init_key_time = 0; + + } else { + init_key_time = 0; + auto sm = dynamic_cast(stek_share_server.sm_.get()); + + // Check whether we received a new key. + // TODO: retry updating STEK when failed + if (sm->received_stek(&curr_stek)) { + TSDebug(PLUGIN, "Received new STEK: %s", + hex_str(std::string(reinterpret_cast(&curr_stek), SSL_TICKET_KEY_SIZE)).c_str()); + + // Move the old key from ticket_keys_[0] to ticket_keys_[1], then put the new key in ticket_keys_[0]. + std::memcpy(&stek_share_server.ticket_keys_[1], &stek_share_server.ticket_keys_[0], SSL_TICKET_KEY_SIZE); + std::memcpy(&stek_share_server.ticket_keys_[0], &curr_stek, SSL_TICKET_KEY_SIZE); + + TSDebug(PLUGIN, "Updating SSL Ticket Key..."); + if (TSSslTicketKeyUpdate(reinterpret_cast(stek_share_server.ticket_keys_), SSL_TICKET_KEY_SIZE * 2) == TS_ERROR) { + TSDebug(PLUGIN, "Update SSL Ticket Key failed."); + } else { + stek_share_server.last_updated_ = time(nullptr); + TSDebug(PLUGIN, "Update SSL Ticket Key succeeded."); + } + } + } + + // Wakeup every 10 seconds to check whether there is a new key to use. + // We do this because if a server is lagging behind, either by losing connection or joining late, + // that server might receive multiple keys (the ones it missed) when it reconnects. Since we only need the + // most recent one, and to save time, we check back every 10 seconds in hope that the barrage of incoming + // keys has finished, and if not the second time around it'll definitely has. + std::this_thread::sleep_for(std::chrono::seconds(10)); + } + + return nullptr; +} + +void +TSPluginInit(int argc, const char *argv[]) +{ + TSPluginRegistrationInfo info; + + info.plugin_name = (char *)("stek_share"); + info.vendor_name = (char *)("ats"); + info.support_email = (char *)("ats-devel@yahooinc.com"); + + TSLifecycleHookAdd(TS_LIFECYCLE_SHUTDOWN_HOOK, TSContCreate(shutdown_handler, nullptr)); + + if (TSPluginRegister(&info) != TS_SUCCESS) { + TSError("Plugin registration failed."); + return; + } + + if (argc < 2) { + TSError("Must specify config file."); + } else if (set_server_info(argc, argv) == 0 && init_raft(nuraft::cs_new()) == 0) { + TSDebug(PLUGIN, "Server ID: %d, Endpoint: %s", stek_share_server.server_id_, stek_share_server.endpoint_.c_str()); + TSThreadCreate(stek_updater, nullptr); + } else { + TSError("Raft initialization failed."); + } +} diff --git a/plugins/experimental/stek_share/stek_share.h b/plugins/experimental/stek_share/stek_share.h new file mode 100644 index 00000000000..14a8579bb46 --- /dev/null +++ b/plugins/experimental/stek_share/stek_share.h @@ -0,0 +1,123 @@ +/************************************************************************ +Copyright 2017-2019 eBay Inc. +Author/Developer(s): Jung-Sang Ahn + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +**************************************************************************/ + +// This file is based on the example code from https://github.com/eBay/NuRaft/tree/master/examples + +#pragma once + +#include +#include +#include + +#include + +#include "stek_utils.h" + +class STEKShareServer +{ +public: + STEKShareServer() : server_id_(1), addr_("localhost"), port_(25000), sm_(nullptr), smgr_(nullptr), raft_instance_(nullptr) + { + last_updated_ = 0; + current_log_idx_ = 0; + + // Default ASIO thread pool size: 4. + asio_thread_pool_size_ = 4; + + // Default heart beat interval: 100 ms. + heart_beat_interval_ = 100; + + // Default election timeout: 200~400 ms. + election_timeout_lower_bound_ = 200; + election_timeout_upper_bound_ = 400; + + // Up to 5 logs will be preserved ahead the last snapshot. + reserved_log_items_ = 5; + + // Snapshot will be created for every 5 log appends. + snapshot_distance_ = 5; + + // Client timeout: 3000 ms. + client_req_timeout_ = 3000; + + std::memset(ticket_keys_, 0, SSL_TICKET_KEY_SIZE * 2); + } + + void + reset() + { + sm_.reset(); + smgr_.reset(); + raft_instance_.reset(); + } + + // Server ID. + int server_id_; + + // Server address. + std::string addr_; + + // Server port. + int port_; + + // Endpoint: ":". + std::string endpoint_; + + // State machine. + nuraft::ptr sm_; + + // State manager. + nuraft::ptr smgr_; + + // Raft launcher. + nuraft::raft_launcher launcher_; + + // Raft server instance. + nuraft::ptr raft_instance_; + + // List of servers to auto add. + std::map server_list_; + + // STEK update interval. + int key_update_interval_; + + // When was STEK last updated. + time_t last_updated_; + + uint64_t current_log_idx_; + + size_t asio_thread_pool_size_; + + int heart_beat_interval_; + + int election_timeout_lower_bound_; + int election_timeout_upper_bound_; + + int reserved_log_items_; + + int snapshot_distance_; + + int client_req_timeout_; + + // TLS related stuff. + std::string root_cert_file_; + std::string server_cert_file_; + std::string server_key_file_; + std::string cert_verify_str_; + + ssl_ticket_key_t ticket_keys_[2]; +}; diff --git a/plugins/experimental/stek_share/stek_utils.cc b/plugins/experimental/stek_share/stek_utils.cc new file mode 100644 index 00000000000..89a3884b766 --- /dev/null +++ b/plugins/experimental/stek_share/stek_utils.cc @@ -0,0 +1,82 @@ +/** @file + + stek_utils.cc - Deal with STEK + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "stek_utils.h" +#include "common.h" + +int +get_good_random(char *buffer, int size, int need_good_entropy) +{ + FILE *fp; + int numread = 0; + char *rand_file_name; + + /* /dev/random blocks until good entropy and can take up to 2 seconds per byte on idle machines */ + /* /dev/urandom does not have entropy check, and is very quick. + * Caller decides quality needed */ + rand_file_name = const_cast((need_good_entropy) ? /* Good & slow */ "/dev/random" : /*Fast*/ "/dev/urandom"); + + if (nullptr == (fp = fopen(rand_file_name, "r"))) { + return -1; /* failure */ + } + numread = static_cast(fread(buffer, 1, size, fp)); + fclose(fp); + + return ((numread == size) ? 0 /* success*/ : -1 /*failure*/); +} + +int +generate_new_stek(ssl_ticket_key_t *return_stek, int entropy_ensured) +{ + /* Generate a new Session-Ticket-Encryption-Key and places it into buffer + * provided return -1 on failure, 0 on success. + * if boolean global_key is set (inidcating it's the global key space), + will get global key lock before setting */ + + ssl_ticket_key_t new_key; // tmp local buffer + + /* We create key in local buff to minimize lock time on global, + * because entropy ensuring can take a very long time e.g. 2 seconds per byte of entropy*/ + if ((get_good_random(reinterpret_cast(&(new_key.aes_key)), SSL_KEY_LEN, (entropy_ensured) ? 1 : 0) != 0) || + (get_good_random(reinterpret_cast(&(new_key.hmac_secret)), SSL_KEY_LEN, (entropy_ensured) ? 1 : 0) != 0) || + (get_good_random(reinterpret_cast(&(new_key.key_name)), SSL_KEY_LEN, 0) != 0)) { + return -1; /* couldn't generate new STEK */ + } + + std::memcpy(return_stek, &new_key, SSL_TICKET_KEY_SIZE); + std::memset(&new_key, 0, SSL_TICKET_KEY_SIZE); // keep our stack clean + + return 0; /* success */ +} diff --git a/plugins/experimental/stek_share/stek_utils.h b/plugins/experimental/stek_share/stek_utils.h new file mode 100644 index 00000000000..c51a73bbbd8 --- /dev/null +++ b/plugins/experimental/stek_share/stek_utils.h @@ -0,0 +1,43 @@ +/** @file + + stek_utils.h + + @section license License + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + */ + +#pragma once + +/* STEK - Session Ticket Encryption Key stuff */ +#define STEK_MAX_LIFETIME 86400 // 24 hours max - should rotate STEK +#define STEK_NOT_CHANGED_WARNING_INTERVAL (2 * STEK_MAX_LIFETIME) // warn on non-stek rotate every X secs. +#define STEK_MAX_ENC_SIZE 512 + +#define SSL_KEY_LEN 16 + +typedef struct ssl_ticket_key // an STEK +{ + unsigned char key_name[SSL_KEY_LEN]; // tickets use this name to identify who encrypted + unsigned char hmac_secret[SSL_KEY_LEN]; + unsigned char aes_key[SSL_KEY_LEN]; +} ssl_ticket_key_t; + +#define SSL_TICKET_KEY_SIZE sizeof(ssl_ticket_key_t) + +int generate_new_stek(ssl_ticket_key_t *return_stek, int entropy_ensured); diff --git a/tests/gold_tests/pluginTest/stek_share/server_list.yaml b/tests/gold_tests/pluginTest/stek_share/server_list.yaml new file mode 100644 index 00000000000..77d815f0aed --- /dev/null +++ b/tests/gold_tests/pluginTest/stek_share/server_list.yaml @@ -0,0 +1,15 @@ +- server_id: 1 + address: 127.0.0.1 + port: 10001 +- server_id: 2 + address: 127.0.0.1 + port: 10002 +- server_id: 3 + address: 127.0.0.1 + port: 10003 +- server_id: 4 + address: 127.0.0.1 + port: 10004 +- server_id: 5 + address: 127.0.0.1 + port: 10005 diff --git a/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.crt b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.crt new file mode 100644 index 00000000000..e90438c777a --- /dev/null +++ b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.crt @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFfTCCA2WgAwIBAgIUEH/jBPLg2k3sO2+PKgnU+Rht2y4wDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAklMMQ4wDAYDVQQKDAVZYWhvbzEN +MAsGA1UECwwERWRnZTETMBEGA1UEAwwKc3Rlay1zaGFyZTAeFw0yMjAyMTAxNjIy +MTdaFw0zMjAyMDgxNjIyMTdaME4xCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJJTDEO +MAwGA1UECgwFWWFob28xDTALBgNVBAsMBEVkZ2UxEzARBgNVBAMMCnN0ZWstc2hh +cmUwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDOJLcQ0P6yNQoYANlU +231rpXK7tgiL0QXOv3vw7AosShkZj7NRPEIFRSFrzyAqJLfmeYBz0JPVjOraJ3CV +8D9/2BUNdrbPGTgESEiDzyB4D3DSi8RHy11dwRozwi1YzDzcNoSii3xmE70bt0sT +tzQ1L2lns+B1NoNBNP0F9VhYf0tvNsy5rRtgxFjZDpSiLXeqkf3IYEs0pEK+uk6+ +eXQ0uSTiAkfkQabw65NoFXaN9OxLyWDu4c+lxq7LKPRQ9O9lI05YaScA3d79ob/k +qb0v4khS7cuYi19h7D5b0QZo1T/EA0HykMYvxfHHH3HviqQItxBYPnVxbwo2D6mF +WS3Uupoh57QkR2HkX3MReZcHwgBDd7IciAyKB6a0VoKqlyLN82Mod4svHo8LGX1d +zWWeOV89pa3txZ5gzilpMtoCwgGVaKn0u1mqEBx4azY8bJkm7OtSYC8GXtDbB7fo +tptcXn3JWGgvtUAmZlaaSzlNocjORmjUIXr85kiaWNLmJpHNS5w3Ncf9Y7swxfwn +BQmNiU0TdXsMHDDlGXoy+0Osw+GJmQijOIF2pQGnKsSozVtBIP4/ESKgAxh8+5Q1 +d0D0yG0OC/pcnam9Yx/qFRwvSUZ5X+hSam8ljwT1NzC74ATMvt9DU6AFSGK56Qqe +/Y/QQShsUg1uIqa9L2hvamIOfwIDAQABo1MwUTAdBgNVHQ4EFgQU0G+dbyjOmCbn +ay+x1u/lM9OY/fAwHwYDVR0jBBgwFoAU0G+dbyjOmCbnay+x1u/lM9OY/fAwDwYD +VR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAgEAq9zQnurNYUCsDjRXNU+h +72i2zZCnWu5oKe6lCGn9hW3nGmRFAjfKb+fE3sNJxikRF7Cas+3mx1g7Z6riHjq9 +WMPKfg6H+ONXaxlfBbHCVrAkrflU1EVyPCXHVkapq4+h1HQAfEVpA13CXFnztTXu +/tldQf2qDsloRdh9CNp/rNAzee2ouzy7SaDdTeA5DgNWhLwUQ4qCj+llD8me10lu +YqyBWWdRG28Tt7GyeK3mJH9vwHsIA/hadOiDJ6ReS5Rm+MyxoRQY8QnnNS3h5UjZ +wR0o3DrweyX0gWf4CI3Ycj5+gZvjI9kPd3deqCAh5YUj9yEYLZI1++Aud0zUSjPK +R/Z9fLyjmADN0FV71aP3g6e/A9doyQYpNVg+spxCQnTCDGRuRhJv8Ubvehy9bCuw +d+iMZXybqii8xsEdTZp/2L8ZarAe888gC/JEQw3JKo/NTaW0P6FS7LYAMLmya2ez +5I4rT9BTTdwoxKSVbEro3f36U4hChS4YMYIsZ1sLjNljAQiUAAJeoJnVS/vCDw44 +YAQtgAIaIeMLqFgikf0U688r+78lZYIxhAuB2hWSUt7JDHGEq6+GZqh/3GLmSZmi +Pqk/jV/esxEBBmxyP6hiwDKanfEQfpk6n5kxyftIRLGSg5EMtD00AA3Ono9uyr7e +znK//SeO390xcNVQsi+8Ppw= +-----END CERTIFICATE----- diff --git a/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.key b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.key new file mode 100644 index 00000000000..bbd6bbb746d --- /dev/null +++ b/tests/gold_tests/pluginTest/stek_share/ssl/self_signed.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDOJLcQ0P6yNQoY +ANlU231rpXK7tgiL0QXOv3vw7AosShkZj7NRPEIFRSFrzyAqJLfmeYBz0JPVjOra +J3CV8D9/2BUNdrbPGTgESEiDzyB4D3DSi8RHy11dwRozwi1YzDzcNoSii3xmE70b +t0sTtzQ1L2lns+B1NoNBNP0F9VhYf0tvNsy5rRtgxFjZDpSiLXeqkf3IYEs0pEK+ +uk6+eXQ0uSTiAkfkQabw65NoFXaN9OxLyWDu4c+lxq7LKPRQ9O9lI05YaScA3d79 +ob/kqb0v4khS7cuYi19h7D5b0QZo1T/EA0HykMYvxfHHH3HviqQItxBYPnVxbwo2 +D6mFWS3Uupoh57QkR2HkX3MReZcHwgBDd7IciAyKB6a0VoKqlyLN82Mod4svHo8L +GX1dzWWeOV89pa3txZ5gzilpMtoCwgGVaKn0u1mqEBx4azY8bJkm7OtSYC8GXtDb +B7fotptcXn3JWGgvtUAmZlaaSzlNocjORmjUIXr85kiaWNLmJpHNS5w3Ncf9Y7sw +xfwnBQmNiU0TdXsMHDDlGXoy+0Osw+GJmQijOIF2pQGnKsSozVtBIP4/ESKgAxh8 ++5Q1d0D0yG0OC/pcnam9Yx/qFRwvSUZ5X+hSam8ljwT1NzC74ATMvt9DU6AFSGK5 +6Qqe/Y/QQShsUg1uIqa9L2hvamIOfwIDAQABAoICAD5j2FAroNpYuSxYnW5UW9pH +obj0OBPw+DwBskZReia7amtVFaWBgk3MBXh2oLqAkHQd0+W5e/THCJFsHGQU6XMM ++BoyEtQNQunw4pmaB66upavjh01fXGuytPZzT3wvnD/d9Dip1MWkNbj8ualG6nMq +XVF4nHd9Py5uFiJGhi2KoU8Qm9eab83Svz06b3vCHRSvyMprcneFO3o0Mv7tDWGj +o2kP3ahUwmzqL5vx2wbN2PJ7CW5jQ5Bd2Ks+Qut5pjbK/7w8XwShIgtLeCOBx/OF +HfSTaepKTFz7vkfVtIXn/LubbMs4S/NLioiEmNwx7sGAfl7m0G67d7Cy/tCQFBFi +JsdeZ0ekLH+nH+9QJzSFQzxDUrl5oGxuvtUa8HYbRUtkfaTzrHX/W3EO0Cgscs9y +Pz6ARWunyanmttmWQEXorstjiRxOXP+QX+B3Uwjl18tDHs6dLUgpPv7Kl5nJ63KF +B30ITVOK6e3sLi9y9AGVT8EVNQ7Zd7XLtXJ9lPOKjDec1TsSFvxfdAjigVFy1gfN +nsmdyL0IeeZWi00qRQmTel0+mc9sqV4zczbxcHfDsSgrMvlDhiRxyusUyTcG1XHc +20f/wqfTKDMkWYtWHB9Ty4BxmJUJZlixioim6eAk26L9STA1dB3092kYLk0/62Iv +Fms1UjNn4DDERe/cUsaxAoIBAQDsqFbZ7R+SLBD/0HiQBWpwHbOjJ5AtTglNIrTZ +XewLWwtYEvUT0bTfQDJJ+PBybR6RE4otFxKEfuR/WXQ9szQN6SUvIabaky/vRh0I +q07NCBgpu3hFEw+qR4eXoXNFpYjSmAFyc1ViwwVfFvgqhywd45Rj46VkTv/x1UlS +dOhuCUPTwuKVsDlRtPwBqi1dLCgdR0cEyjAlqe2DNrvtBmBlXqti4oFXDFPMzln8 +Xk2dyqDBeyKLnJU3W51x0J3hOl+axeMxPF1qWKgB5hvIADEKKywverMlyYqiWT+P +QikUOocmiwna+sf2zEQrhGfa6Y8gWGMOcGy2NFRGAAyWkN/XAoIBAQDe/esmXmkq +gKWSxqOzeJfUHVF4ygmvS7mKbDNMY3GCvZgfD+hpZeZARTNOu9JrDWiUcxT8Wj+j +ZtYruQtCOxt1eiBoJOnYwaamm3d+HgvaE19+WxRGjcTGsZ6VreKzZlbgbAiCw/pd +rDAqi+Bs7bNG4dOdSb9ctuj4YkBmpM6u5TQUCEKAIrr90gIJli/B9+x0PIgrovlm +YCjlmrQlw+2DSFPrRxuqpXxNOyFyFsj992O/LFKAwYCyhtkqiUUqJbREIlyR1oL6 +X38Gbsp/jgrEUQ3N3773eV+bltz2nazEb67ps7nemjmZvjyluDjwfJdJJogWJsKK +0jGC21EGqRGZAoIBAQCZoO5Aql5EVbbzWjHpzJo8Dgv/bj96KZ6AJHeiZAZHmOLU +Wfoe05PHGbWLr77niU6+fyDEBKZQvM84nKmJJDw2i5NH9WCLo+EKQ0m1xv9wukB1 +Vu3MaYNR1v1+waBDJiKcE3FdCuHzKwbho9eWRAmvnX1HGxDS/TXJl9vxW1NHm1wc +q/bLlYqgMA0oR4ELaw7fctX3lgmLabR16aI1TF5nb/1yQ/gSuj3sRkjEO7PHKzMQ +Zw8V8qArP54FtJfJDkvh/XRvEfDSiJsIIIkIXJd5Mm2MpOqHLT6CBc3tAdYI+7Wg +n8HWFdaZsCDQ3zNMOTJgnQAw72qjHXVXu9BwLbwhAoIBAE/PUXpKEBnGMXx22+BA +KRch5yb0KMM0txNz5mhQry+769YyO1x9vAsEuXhUgNsP0X5QMhKfumchR0Ye1Ii/ +3vQM4cxkac3KgXrf6cSZvGQwytzOfFNEKklzCO9JbPoIhs+L2v/yZIliN1sC9TAH +Y0LbUIHbA0KLtJYxlBsooVC3eAwzaJmz1HlD0LbdqfoiYd64S4RSsDCT+g8zb4aU +uU1jdaWfradF01dQ8oeC4C0Ffg3OLzkmCInc+ZzfxIFxPTOlmLwZqocx5qTGwnMk +w3XADNDCY/bu2ek19Z/Ojyc/UbsTOFMTn8oG7G3joX1xGjR0NgC3nqlQ0aekFzvr +BwECggEBAKqlqm/yiPVViqHpQPrHbMWihaDEDF95L3jvvvByWS/96D7KUGGTHDJf +Vszm8slfvdIrMtqdGH9DvhP2X1HDMLwl6zMFKgSwkf1bT0+A3a7TldJ4ezcoeAOd +27YujmF5tk5yV2ufu9l0M+6bhXZuwdx1ktYo+GiR5G5lwllMT6ztQLcQA1BrSXgy +1hr7OsKBBS6DGhjBxxULbp9DgADbemp8xXOXD5EhBIhu7TIioHoPDaXesjnsClda +hRzyWycj5SDy1/kFAEYO94/wMFY5zYsmzUG3+AZ3eymdULP3mg6hkYJrht84R/hg +YAxBUEWWyJC/Uhh1KnlWUPX3fcl0VyU= +-----END PRIVATE KEY----- diff --git a/tests/gold_tests/pluginTest/stek_share/stek_share.test.py b/tests/gold_tests/pluginTest/stek_share/stek_share.test.py new file mode 100644 index 00000000000..50f2010022e --- /dev/null +++ b/tests/gold_tests/pluginTest/stek_share/stek_share.test.py @@ -0,0 +1,254 @@ +''' +''' +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re + +Test.Summary = 'Test the STEK Share plugin' +Test.testName = "stek_share" + +Test.SkipUnless(Condition.PluginExists('stek_share.so')) + +server = Test.MakeOriginServer('server') + +ts1 = Test.MakeATSProcess("ts1", select_ports=True, enable_tls=True) +ts2 = Test.MakeATSProcess("ts2", select_ports=True, enable_tls=True) +ts3 = Test.MakeATSProcess("ts3", select_ports=True, enable_tls=True) +ts4 = Test.MakeATSProcess("ts4", select_ports=True, enable_tls=True) +ts5 = Test.MakeATSProcess("ts5", select_ports=True, enable_tls=True) + +Test.Setup.Copy('ssl/self_signed.crt') +Test.Setup.Copy('ssl/self_signed.key') +Test.Setup.Copy('server_list.yaml') + +cert_path = os.path.join(Test.RunDirectory, 'self_signed.crt') +key_path = os.path.join(Test.RunDirectory, 'self_signed.key') +server_list_path = os.path.join(Test.RunDirectory, 'server_list.yaml') + +request_header1 = { + 'headers': 'GET / HTTP/1.1\r\nHost: www.example.com\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': '' +} +response_header1 = { + 'headers': 'HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n', + 'timestamp': '1469733493.993', + 'body': 'curl test' +} +server.addResponse('sessionlog.json', request_header1, response_header1) + +stek_share_conf_path_1 = os.path.join(ts1.Variables.CONFIGDIR, 'stek_share_conf.yaml') +stek_share_conf_path_2 = os.path.join(ts2.Variables.CONFIGDIR, 'stek_share_conf.yaml') +stek_share_conf_path_3 = os.path.join(ts3.Variables.CONFIGDIR, 'stek_share_conf.yaml') +stek_share_conf_path_4 = os.path.join(ts4.Variables.CONFIGDIR, 'stek_share_conf.yaml') +stek_share_conf_path_5 = os.path.join(ts5.Variables.CONFIGDIR, 'stek_share_conf.yaml') + +ts1.Disk.File(stek_share_conf_path_1, id="stek_share_conf_1", typename="ats:config") +ts2.Disk.File(stek_share_conf_path_2, id="stek_share_conf_2", typename="ats:config") +ts3.Disk.File(stek_share_conf_path_3, id="stek_share_conf_3", typename="ats:config") +ts4.Disk.File(stek_share_conf_path_4, id="stek_share_conf_4", typename="ats:config") +ts5.Disk.File(stek_share_conf_path_5, id="stek_share_conf_5", typename="ats:config") + +ts1.Disk.stek_share_conf_1.AddLines([ + 'server_id: 1', + 'address: 127.0.0.1', + 'port: 10001', + 'asio_thread_pool_size: 4', + 'heart_beat_interval: 100', + 'election_timeout_lower_bound: 200', + 'election_timeout_upper_bound: 400', + 'reserved_log_items: 5', + 'snapshot_distance: 5', + 'client_req_timeout: 3000', # this is in milliseconds + 'key_update_interval: 3600', # this is in seconds + 'server_list_file: {0}'.format(server_list_path), + 'root_cert_file: {0}'.format(cert_path), + 'server_cert_file: {0}'.format(cert_path), + 'server_key_file: {0}'.format(key_path), + 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share' +]) + +ts2.Disk.stek_share_conf_2.AddLines([ + 'server_id: 2', + 'address: 127.0.0.1', + 'port: 10002', + 'asio_thread_pool_size: 4', + 'heart_beat_interval: 100', + 'election_timeout_lower_bound: 200', + 'election_timeout_upper_bound: 400', + 'reserved_log_items: 5', + 'snapshot_distance: 5', + 'client_req_timeout: 3000', # this is in milliseconds + 'key_update_interval: 3600', # this is in seconds + 'server_list_file: {0}'.format(server_list_path), + 'root_cert_file: {0}'.format(cert_path), + 'server_cert_file: {0}'.format(cert_path), + 'server_key_file: {0}'.format(key_path), + 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share' +]) + +ts3.Disk.stek_share_conf_3.AddLines([ + 'server_id: 3', + 'address: 127.0.0.1', + 'port: 10003', + 'asio_thread_pool_size: 4', + 'heart_beat_interval: 100', + 'election_timeout_lower_bound: 200', + 'election_timeout_upper_bound: 400', + 'reserved_log_items: 5', + 'snapshot_distance: 5', + 'client_req_timeout: 3000', # this is in milliseconds + 'key_update_interval: 3600', # this is in seconds + 'server_list_file: {0}'.format(server_list_path), + 'root_cert_file: {0}'.format(cert_path), + 'server_cert_file: {0}'.format(cert_path), + 'server_key_file: {0}'.format(key_path), + 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share' +]) + +ts4.Disk.stek_share_conf_4.AddLines([ + 'server_id: 4', + 'address: 127.0.0.1', + 'port: 10004', + 'asio_thread_pool_size: 4', + 'heart_beat_interval: 100', + 'election_timeout_lower_bound: 200', + 'election_timeout_upper_bound: 400', + 'reserved_log_items: 5', + 'snapshot_distance: 5', + 'client_req_timeout: 3000', # this is in milliseconds + 'key_update_interval: 3600', # this is in seconds + 'server_list_file: {0}'.format(server_list_path), + 'root_cert_file: {0}'.format(cert_path), + 'server_cert_file: {0}'.format(cert_path), + 'server_key_file: {0}'.format(key_path), + 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share' +]) + +ts5.Disk.stek_share_conf_5.AddLines([ + 'server_id: 5', + 'address: 127.0.0.1', + 'port: 10005', + 'asio_thread_pool_size: 4', + 'heart_beat_interval: 100', + 'election_timeout_lower_bound: 200', + 'election_timeout_upper_bound: 400', + 'reserved_log_items: 5', + 'snapshot_distance: 5', + 'client_req_timeout: 3000', # this is in milliseconds + 'key_update_interval: 3600', # this is in seconds + 'server_list_file: {0}'.format(server_list_path), + 'root_cert_file: {0}'.format(cert_path), + 'server_cert_file: {0}'.format(cert_path), + 'server_key_file: {0}'.format(key_path), + 'cert_verify_str: /C=US/ST=IL/O=Yahoo/OU=Edge/CN=stek-share' +]) + +ts1.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl.session_cache.num_buckets': 16, 'proxy.config.ssl.server.session_ticket.enable': 1, 'proxy.config.ssl.server.cipher_suite': + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'}) +ts1.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_1)) +ts1.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key') +ts1.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts2.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl.session_cache.num_buckets': 16, 'proxy.config.ssl.server.session_ticket.enable': 1, 'proxy.config.ssl.server.cipher_suite': + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'}) +ts2.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_2)) +ts2.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key') +ts2.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts3.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl.session_cache.num_buckets': 16, 'proxy.config.ssl.server.session_ticket.enable': 1, 'proxy.config.ssl.server.cipher_suite': + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'}) +ts3.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_3)) +ts3.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key') +ts3.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts4.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl.session_cache.num_buckets': 16, 'proxy.config.ssl.server.session_ticket.enable': 1, 'proxy.config.ssl.server.cipher_suite': + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'}) +ts4.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_4)) +ts4.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key') +ts4.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + +ts5.Disk.records_config.update({'proxy.config.diags.debug.enabled': 1, 'proxy.config.diags.debug.tags': 'stek_share', 'proxy.config.exec_thread.autoconfig': 0, 'proxy.config.exec_thread.limit': 4, 'proxy.config.ssl.server.cert.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.server.private_key.path': '{0}'.format(Test.RunDirectory), 'proxy.config.ssl.session_cache': 2, 'proxy.config.ssl.session_cache.size': 1024, 'proxy.config.ssl.session_cache.timeout': 7200, 'proxy.config.ssl.session_cache.num_buckets': 16, 'proxy.config.ssl.server.session_ticket.enable': 1, 'proxy.config.ssl.server.cipher_suite': + 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-DSS-AES256-SHA:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'}) +ts5.Disk.plugin_config.AddLine('stek_share.so {0}'.format(stek_share_conf_path_5)) +ts5.Disk.ssl_multicert_config.AddLine('dest_ip=* ssl_cert_name=self_signed.crt ssl_key_name=self_signed.key') +ts5.Disk.remap_config.AddLine('map / http://127.0.0.1:{0}'.format(server.Variables.Port)) + + +def check_session(ev, test): + retval = False + f = open(test.GetContent(ev), 'r') + err = "Session ids match" + if not f: + err = "Failed to open {0}".format(openssl_output) + return (retval, "Check that session ids match", err) + + content = f.read() + match = re.findall('Session-ID: ([0-9A-F]+)', content) + + if match: + if all(i == j for i, j in zip(match, match[1:])): + err = "{0} reused successfully {1} times".format(match[0], len(match) - 1) + retval = True + else: + err = "Session is not being reused as expected" + else: + err = "Didn't find session id" + return (retval, "Check that session ids match", err) + + +tr1 = Test.AddTestRun('Basic Curl test, and give it enough time for all ATS to start up and sync STEK') +tr1.Processes.Default.Command = 'sleep 10 && curl https://127.0.0.1:{0} -k'.format(ts1.Variables.ssl_port) +tr1.Processes.Default.ReturnCode = 0 +tr1.Processes.Default.StartBefore(server) +tr1.Processes.Default.StartBefore(ts1) +tr1.Processes.Default.StartBefore(ts2) +tr1.Processes.Default.StartBefore(ts3) +tr1.Processes.Default.StartBefore(ts4) +tr1.Processes.Default.StartBefore(ts5) +tr1.Processes.Default.Streams.All = Testers.ContainsExpression('curl test', 'Making sure the basics still work') +ts1.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed') +ts2.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed') +ts3.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed') +ts4.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed') +ts5.Streams.All = Testers.ContainsExpression('Generate initial STEK succeeded', 'should succeed') +tr1.StillRunningAfter = server +tr1.StillRunningAfter += ts1 +tr1.StillRunningAfter += ts2 +tr1.StillRunningAfter += ts3 +tr1.StillRunningAfter += ts4 +tr1.StillRunningAfter += ts5 + +tr2 = Test.AddTestRun("TLSv1.2 Session Ticket") +tr2.Command = \ + 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_out {5} && ' \ + 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{0} -sess_in {5} && ' \ + 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{1} -sess_in {5} && ' \ + 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{2} -sess_in {5} && ' \ + 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{3} -sess_in {5} && ' \ + 'echo -e "GET / HTTP/1.1\r\n" | openssl s_client -tls1_2 -connect 127.0.0.1:{4} -sess_in {5}' \ + .format( + ts1.Variables.ssl_port, + ts2.Variables.ssl_port, + ts3.Variables.ssl_port, + ts4.Variables.ssl_port, + ts5.Variables.ssl_port, + os.path.join(Test.RunDirectory, 'sess.dat') + ) +tr2.ReturnCode = 0 +tr2.Processes.Default.Streams.All.Content = Testers.Lambda(check_session)